{"schema":1,"snapshot":"20261004","notes":{"task":"single-turn structured answer (<=128 tokens) scored by a deterministic verifier with 12 reward channels","core":"cross-validated core score channel (GDPO priority 3.0)","holdout":"50 fixed probe prompts scored on a later, frozen evaluation slice; mean over valid answers","selection":"checkpoint with the best train-side core score on the same probe"},"environment_a":[{"id":"dapo-lora-500","label":"DAPO \u00b7 LoRA \u00b7 500 prompts","run_ids":["1790661651","1790738119"],"loss":"DAPO","adapter":"LoRA","prompts":500,"repo_r":false,"holdout_by_checkpoint":[{"step":0,"holdout":-1.3664,"train_core":-0.9836,"valid_rate":0.9},{"step":100,"holdout":-0.9112,"train_core":-0.506,"valid_rate":0.86},{"step":200,"holdout":-0.5,"train_core":-0.4151,"valid_rate":0.92},{"step":300,"holdout":-0.5874,"train_core":-0.3048,"valid_rate":0.94},{"step":400,"holdout":-0.6258,"train_core":-0.362,"valid_rate":0.96},{"step":500,"holdout":-0.5551,"train_core":-0.336,"valid_rate":0.96},{"step":600,"holdout":-0.4742,"train_core":-0.2953,"valid_rate":0.96}],"selected_step":600,"selected_holdout":-0.4742,"train_steps":[{"step":1,"reward":-2.4800984859466553,"core":-0.7827860713005066,"cost":-0.09962290525436401,"kl":0.0,"entropy":0.5109328404068947},{"step":2,"reward":-3.3529109954833984,"core":-0.9996337890625,"cost":-0.2529066801071167,"kl":0.0003398795015527867,"entropy":0.4769136384129524},{"step":3,"reward":-3.1347711086273193,"core":-0.9240076541900635,"cost":-0.28424111008644104,"kl":0.00019672626149258576,"entropy":0.49911557883024216},{"step":4,"reward":-3.299381971359253,"core":-0.9858458042144775,"cost":-0.2978191673755646,"kl":0.0004884729132754728,"entropy":0.5490728318691254},{"step":5,"reward":-5.238131999969482,"core":-1.5039620399475098,"cost":-0.6675043106079102,"kl":0.0009607649990357459,"entropy":0.5068085342645645},{"step":6,"reward":-2.667231559753418,"core":-0.7936592102050781,"cost":-0.23005366325378418,"kl":0.00055294694902841,"entropy":0.47096067667007446},{"step":7,"reward":-4.201741695404053,"core":-1.2255959510803223,"cost":-0.47694286704063416,"kl":0.0007880648190621287,"entropy":0.4891737923026085},{"step":8,"reward":-3.385573387145996,"core":-0.9751376509666443,"cost":-0.32098689675331116,"kl":0.001420965272700414,"entropy":0.5092106014490128},{"step":9,"reward":-1.7967100143432617,"core":-0.5510198473930359,"cost":-0.058272834867239,"kl":0.001570823456859216,"entropy":0.5184860676527023},{"step":10,"reward":-2.5293428897857666,"core":-0.7519988417625427,"cost":-0.1716056615114212,"kl":0.0011799749918282032,"entropy":0.45426998287439346},{"step":11,"reward":-2.368854284286499,"core":-0.7379443049430847,"cost":-0.06857822835445404,"kl":0.0010980860679410398,"entropy":0.46594522148370743},{"step":12,"reward":-2.9439468383789062,"core":-0.8285266757011414,"cost":-0.28169703483581543,"kl":0.0016231494373641908,"entropy":0.4420052468776703},{"step":13,"reward":-2.1769301891326904,"core":-0.6652453541755676,"cost":-0.11480730772018433,"kl":0.0018227719701826572,"entropy":0.4056234657764435},{"step":14,"reward":-2.678600549697876,"core":-0.7486753463745117,"cost":-0.2399551123380661,"kl":0.0018651789869181812,"entropy":0.4783995896577835},{"step":15,"reward":-3.2450473308563232,"core":-0.9813854694366455,"cost":-0.23066578805446625,"kl":0.0030047839391045272,"entropy":0.524692602455616},{"step":16,"reward":-2.5300023555755615,"core":-0.7465260624885559,"cost":-0.1769241839647293,"kl":0.0033521668519824743,"entropy":0.4818916693329811},{"step":17,"reward":-4.144521236419678,"core":-1.2641972303390503,"cost":-0.26392391324043274,"kl":0.002913717762567103,"entropy":0.4215548560023308},{"step":18,"reward":-3.0707848072052,"core":-0.9248175621032715,"cost":-0.23011143505573273,"kl":0.0056348638609051704,"entropy":0.5909754186868668},{"step":19,"reward":-2.365257740020752,"core":-0.6844921708106995,"cost":-0.18328285217285156,"kl":0.004642658575903624,"entropy":0.5015170723199844},{"step":20,"reward":-2.1913740634918213,"core":-0.6625267863273621,"cost":-0.07142599672079086,"kl":0.00437116960529238,"entropy":0.4595140367746353},{"step":21,"reward":-2.2065179347991943,"core":-0.6686447262763977,"cost":-0.13708150386810303,"kl":0.004527828888967633,"entropy":0.4890234023332596},{"step":22,"reward":-3.3832485675811768,"core":-1.0317124128341675,"cost":-0.2049540877342224,"kl":0.004547262156847864,"entropy":0.5166595876216888},{"step":23,"reward":-2.592552900314331,"core":-0.7557744383811951,"cost":-0.16046421229839325,"kl":0.006475151050835848,"entropy":0.47420743852853775},{"step":24,"reward":-1.9900225400924683,"core":-0.6363727450370789,"cost":-0.017757540568709373,"kl":0.004741964163258672,"entropy":0.4742291197180748},{"step":25,"reward":-4.157835006713867,"core":-1.2293349504470825,"cost":-0.4323602616786957,"kl":0.007116415887139738,"entropy":0.4278884306550026},{"step":26,"reward":-1.9261088371276855,"core":-0.5307184457778931,"cost":-0.0946003794670105,"kl":0.006696309894323349,"entropy":0.4538767784833908},{"step":27,"reward":-2.089970111846924,"core":-0.5876926779747009,"cost":-0.19874556362628937,"kl":0.007214726880192757,"entropy":0.44982582330703735},{"step":28,"reward":-2.437753915786743,"core":-0.7023116946220398,"cost":-0.18839307129383087,"kl":0.007665549288503826,"entropy":0.4853874146938324},{"step":29,"reward":-1.3372637033462524,"core":-0.39761975407600403,"cost":-0.03874296322464943,"kl":0.018040232127532363,"entropy":0.5813458189368248},{"step":30,"reward":-1.9992233514785767,"core":-0.6021098494529724,"cost":-0.015771782025694847,"kl":0.006001574918627739,"entropy":0.3616034910082817},{"step":31,"reward":-3.6613452434539795,"core":-1.0787452459335327,"cost":-0.3541708290576935,"kl":0.009503945126198232,"entropy":0.4586072415113449},{"step":32,"reward":-1.974791169166565,"core":-0.6343578696250916,"cost":-0.038442835211753845,"kl":0.01006741973105818,"entropy":0.5564479380846024},{"step":33,"reward":-2.445833206176758,"core":-0.7433250546455383,"cost":-0.1236071065068245,"kl":0.01281022047623992,"entropy":0.5742675215005875},{"step":34,"reward":-2.0755348205566406,"core":-0.6090640425682068,"cost":-0.067402184009552,"kl":0.014855657005682588,"entropy":0.5670893043279648},{"step":35,"reward":-2.058314800262451,"core":-0.6136383414268494,"cost":-0.078800268471241,"kl":0.015550426673144102,"entropy":0.5938212126493454},{"step":36,"reward":-2.967348098754883,"core":-0.892519474029541,"cost":-0.22520245611667633,"kl":0.008660079562105238,"entropy":0.4065868705511093},{"step":37,"reward":-2.4136712551116943,"core":-0.691363513469696,"cost":-0.21701538562774658,"kl":0.013537541963160038,"entropy":0.4376407265663147},{"step":38,"reward":-1.6332712173461914,"core":-0.47590944170951843,"cost":-0.06074463948607445,"kl":0.014487896114587784,"entropy":0.479805052280426},{"step":39,"reward":-1.6416527032852173,"core":-0.47307589650154114,"cost":-0.09824972599744797,"kl":0.017738936468958855,"entropy":0.42898598313331604},{"step":40,"reward":-1.268860936164856,"core":-0.3590371906757355,"cost":-0.03400961682200432,"kl":0.017529786797240376,"entropy":0.4475024864077568},{"step":41,"reward":-2.5107364654541016,"core":-0.753931999206543,"cost":-0.15159304440021515,"kl":0.013015237404033542,"entropy":0.43749668449163437},{"step":42,"reward":-2.0024378299713135,"core":-0.5681090354919434,"cost":-0.21871910989284515,"kl":0.011088311905041337,"entropy":0.425168476998806},{"step":43,"reward":-2.255131959915161,"core":-0.6336845755577087,"cost":-0.18966144323349,"kl":0.026257898192852736,"entropy":0.4737432226538658},{"step":44,"reward":-2.2853400707244873,"core":-0.6988380551338196,"cost":-0.05512276664376259,"kl":0.012770242523401976,"entropy":0.4572109133005142},{"step":45,"reward":-2.3776473999023438,"core":-0.7332146167755127,"cost":-0.11499973386526108,"kl":0.01901169284246862,"entropy":0.48241761326789856},{"step":46,"reward":-2.461258888244629,"core":-0.7344159483909607,"cost":-0.10531911998987198,"kl":0.0268824421800673,"entropy":0.5653451383113861},{"step":47,"reward":-1.8739053010940552,"core":-0.5733045339584351,"cost":-0.05866182968020439,"kl":0.017313159303739667,"entropy":0.48870352655649185},{"step":48,"reward":-2.7676632404327393,"core":-0.8366269469261169,"cost":-0.1664595752954483,"kl":0.02049514278769493,"entropy":0.4613654091954231},{"step":49,"reward":-1.4066184759140015,"core":-0.3905436098575592,"cost":-0.054780393838882446,"kl":0.02956387773156166,"entropy":0.46183912456035614},{"step":50,"reward":-1.5699189901351929,"core":-0.432910293340683,"cost":-0.1013861820101738,"kl":0.02443428849801421,"entropy":0.5111003071069717},{"step":51,"reward":-2.7787485122680664,"core":-0.8620988726615906,"cost":-0.10673908144235611,"kl":0.033702582120895386,"entropy":0.5136755034327507},{"step":52,"reward":-2.408777952194214,"core":-0.7166282534599304,"cost":-0.16040010750293732,"kl":0.03546975878998637,"entropy":0.47548476606607437},{"step":53,"reward":-1.820522665977478,"core":-0.5788542032241821,"cost":-0.032152723520994186,"kl":0.032245917711406946,"entropy":0.4790041595697403},{"step":54,"reward":-2.2941579818725586,"core":-0.6404721140861511,"cost":-0.15953370928764343,"kl":0.04519974486902356,"entropy":0.5685155540704727},{"step":55,"reward":-1.7260551452636719,"core":-0.5215153694152832,"cost":-0.06523091346025467,"kl":0.0408881432376802,"entropy":0.5400892347097397},{"step":56,"reward":-2.1389553546905518,"core":-0.6381815075874329,"cost":-0.17291505634784698,"kl":0.03414488071575761,"entropy":0.4596310257911682},{"step":57,"reward":-1.8329354524612427,"core":-0.5893383026123047,"cost":-0.0008177816052921116,"kl":0.036741726100444794,"entropy":0.5190303102135658},{"step":58,"reward":-1.9731416702270508,"core":-0.6134824752807617,"cost":-0.025105023756623268,"kl":0.048117691185325384,"entropy":0.5101988539099693},{"step":59,"reward":-1.5218545198440552,"core":-0.46583375334739685,"cost":-0.006030741613358259,"kl":0.03699167910963297,"entropy":0.5109082087874413},{"step":60,"reward":-2.088196039199829,"core":-0.6166635155677795,"cost":-0.07969379425048828,"kl":0.05103110242635012,"entropy":0.5295429676771164},{"step":61,"reward":-1.8284406661987305,"core":-0.5756561756134033,"cost":-0.022183353081345558,"kl":0.030772658064961433,"entropy":0.40581393241882324},{"step":62,"reward":-2.1889708042144775,"core":-0.6566048264503479,"cost":-0.12440600246191025,"kl":0.11735896486788988,"entropy":0.4774172529578209},{"step":63,"reward":-2.4658803939819336,"core":-0.7631964683532715,"cost":-0.08432936668395996,"kl":0.04684815742075443,"entropy":0.45916783809661865},{"step":64,"reward":-1.8301254510879517,"core":-0.543353259563446,"cost":-0.030087193474173546,"kl":0.04040777124464512,"entropy":0.47117384523153305},{"step":65,"reward":-1.6901775598526,"core":-0.4818948805332184,"cost":-0.020194923505187035,"kl":0.06945784203708172,"entropy":0.530610516667366},{"step":66,"reward":-1.4356235265731812,"core":-0.4307873249053955,"cost":0.0,"kl":0.05400410108268261,"entropy":0.49635642766952515},{"step":67,"reward":-2.0108988285064697,"core":-0.6409069299697876,"cost":-0.047302763909101486,"kl":0.06248745694756508,"entropy":0.596562996506691},{"step":68,"reward":-1.6063183546066284,"core":-0.4735325276851654,"cost":-0.0107730096206069,"kl":0.05680401436984539,"entropy":0.5293212980031967},{"step":69,"reward":-2.011298894882202,"core":-0.6294655799865723,"cost":-0.024464895948767662,"kl":0.05574311129748821,"entropy":0.5194585397839546},{"step":70,"reward":-2.759658098220825,"core":-0.8687778115272522,"cost":-0.05840140953660011,"kl":0.05594557151198387,"entropy":0.536455973982811},{"step":71,"reward":-2.959805727005005,"core":-0.8779711723327637,"cost":-0.1561231166124344,"kl":0.045760720036923885,"entropy":0.49289049953222275},{"step":72,"reward":-1.4700207710266113,"core":-0.44832643866539,"cost":-0.02148538827896118,"kl":0.04233530070632696,"entropy":0.5059888660907745},{"step":73,"reward":-2.0534658432006836,"core":-0.6176475286483765,"cost":-0.07992216944694519,"kl":0.03449504356831312,"entropy":0.45779549330472946},{"step":74,"reward":-1.6525088548660278,"core":-0.4787823259830475,"cost":-0.0069372281432151794,"kl":0.04366347845643759,"entropy":0.5615854859352112},{"step":75,"reward":-3.0148556232452393,"core":-0.910184383392334,"cost":-0.21733975410461426,"kl":0.02721633529290557,"entropy":0.4196028485894203},{"step":76,"reward":-2.666032552719116,"core":-0.8132349848747253,"cost":-0.10863834619522095,"kl":0.037542513106018305,"entropy":0.4674869105219841},{"step":77,"reward":-2.3995187282562256,"core":-0.7369987964630127,"cost":-0.10897303372621536,"kl":0.024600678589195013,"entropy":0.49705297499895096},{"step":78,"reward":-2.132715940475464,"core":-0.6290438175201416,"cost":-0.12136252969503403,"kl":0.024847211316227913,"entropy":0.4986134395003319},{"step":79,"reward":-2.2699127197265625,"core":-0.6601402163505554,"cost":-0.16210150718688965,"kl":0.0390932853333652,"entropy":0.5857090055942535},{"step":80,"reward":-3.134894609451294,"core":-0.8649547696113586,"cost":-0.360095351934433,"kl":0.024668992962688208,"entropy":0.48527785390615463},{"step":81,"reward":-2.346100091934204,"core":-0.6976132392883301,"cost":-0.07249720394611359,"kl":0.03000060562044382,"entropy":0.4817960858345032},{"step":82,"reward":-2.453749895095825,"core":-0.7572700381278992,"cost":-0.09772968292236328,"kl":0.03902307944372296,"entropy":0.5297085642814636},{"step":83,"reward":-3.468796968460083,"core":-1.0271751880645752,"cost":-0.31189393997192383,"kl":0.023799461778253317,"entropy":0.5065068230032921},{"step":84,"reward":-2.015090227127075,"core":-0.6137590408325195,"cost":-0.025302434340119362,"kl":0.036721726413816214,"entropy":0.5074105337262154},{"step":85,"reward":-1.630044937133789,"core":-0.4463748037815094,"cost":-0.051711272448301315,"kl":0.027413911651819944,"entropy":0.5472077801823616},{"step":86,"reward":-2.6175925731658936,"core":-0.8004941940307617,"cost":-0.13339141011238098,"kl":0.026480764150619507,"entropy":0.5343486815690994},{"step":87,"reward":-1.909304141998291,"core":-0.5680750012397766,"cost":-0.06388868391513824,"kl":0.025554317515343428,"entropy":0.5643064081668854},{"step":88,"reward":-3.2914321422576904,"core":-0.9591779112815857,"cost":-0.31583479046821594,"kl":0.029609124176204205,"entropy":0.5267953649163246},{"step":89,"reward":-1.8668688535690308,"core":-0.5745541453361511,"cost":0.0,"kl":0.020212937146425247,"entropy":0.478524774312973},{"step":90,"reward":-2.1659634113311768,"core":-0.6475015878677368,"cost":-0.06274288892745972,"kl":0.028049379587173462,"entropy":0.6079447418451309},{"step":91,"reward":-1.9135518074035645,"core":-0.5732420682907104,"cost":-0.04897111654281616,"kl":0.0286382045596838,"entropy":0.4470741003751755},{"step":92,"reward":-2.1677486896514893,"core":-0.6259585022926331,"cost":-0.0713035985827446,"kl":0.02251226920634508,"entropy":0.4447069764137268},{"step":93,"reward":-2.830355405807495,"core":-0.8387710452079773,"cost":-0.2535947561264038,"kl":0.009974109474569559,"entropy":0.43419886380434036},{"step":94,"reward":-2.4025051593780518,"core":-0.6935297846794128,"cost":-0.1175224557518959,"kl":0.01982612069696188,"entropy":0.45372524857521057},{"step":95,"reward":-1.7600067853927612,"core":-0.497867226600647,"cost":-0.10768598318099976,"kl":0.028596813790500164,"entropy":0.47629714012145996},{"step":96,"reward":-1.5564823150634766,"core":-0.4323953092098236,"cost":-0.057591598480939865,"kl":0.027451214846223593,"entropy":0.4929938390851021},{"step":97,"reward":-3.805638551712036,"core":-1.100554347038269,"cost":-0.45148277282714844,"kl":0.020004968158900738,"entropy":0.5122217759490013},{"step":98,"reward":-1.8578535318374634,"core":-0.43098434805870056,"cost":-0.14041782915592194,"kl":0.022519266698509455,"entropy":0.3955655097961426},{"step":99,"reward":-3.347597122192383,"core":-0.9817214608192444,"cost":-0.3036840260028839,"kl":0.02687334269285202,"entropy":0.37051838636398315},{"step":100,"reward":-1.937149167060852,"core":-0.5547595024108887,"cost":-0.11423555761575699,"kl":0.02006896259263158,"entropy":0.4852400943636894},{"step":101,"reward":-1.3719252347946167,"core":-0.3964090347290039,"cost":-0.006252942606806755,"kl":0.02802477404475212,"entropy":0.48931682109832764},{"step":102,"reward":-3.345137357711792,"core":-1.0142086744308472,"cost":-0.18811364471912384,"kl":0.0230998924234882,"entropy":0.4703226685523987},{"step":103,"reward":-1.241875171661377,"core":-0.37358906865119934,"cost":-0.019343996420502663,"kl":0.02309793373569846,"entropy":0.5202175378799438},{"step":104,"reward":-1.518389344215393,"core":-0.43714675307273865,"cost":-0.008735910058021545,"kl":0.031918845139443874,"entropy":0.5477901250123978},{"step":105,"reward":-2.0016708374023438,"core":-0.6178271174430847,"cost":-0.006801603361964226,"kl":0.020024676341563463,"entropy":0.4644717276096344},{"step":106,"reward":-2.883582830429077,"core":-0.8866665959358215,"cost":-0.14807206392288208,"kl":0.03926357626914978,"entropy":0.4500170722603798},{"step":107,"reward":-2.0771090984344482,"core":-0.6203872561454773,"cost":-0.07896670699119568,"kl":0.03695158241316676,"entropy":0.41423869132995605},{"step":108,"reward":-3.4810221195220947,"core":-1.0354176759719849,"cost":-0.27597832679748535,"kl":0.026913468725979328,"entropy":0.4761153906583786},{"step":109,"reward":-2.996633529663086,"core":-0.9057472348213196,"cost":-0.18236656486988068,"kl":0.02757600648328662,"entropy":0.47489268332719803},{"step":110,"reward":-2.294341802597046,"core":-0.5772371888160706,"cost":-0.23086600005626678,"kl":0.05623798258602619,"entropy":0.5298794880509377},{"step":111,"reward":-1.9168672561645508,"core":-0.537541925907135,"cost":-0.05913431569933891,"kl":0.04043242568150163,"entropy":0.46622102707624435},{"step":112,"reward":-1.8682159185409546,"core":-0.5663362145423889,"cost":-0.0442948155105114,"kl":0.04196652723476291,"entropy":0.5272673666477203},{"step":113,"reward":-2.095184803009033,"core":-0.6230822205543518,"cost":-0.13310925662517548,"kl":0.03371364017948508,"entropy":0.5125789865851402},{"step":114,"reward":-2.5286760330200195,"core":-0.7390997409820557,"cost":-0.19272179901599884,"kl":0.028853117488324642,"entropy":0.372362345457077},{"step":115,"reward":-1.2518173456192017,"core":-0.32718580961227417,"cost":-0.023984432220458984,"kl":0.01918660756200552,"entropy":0.4364074543118477},{"step":116,"reward":-0.23817306756973267,"core":0.04859141632914543,"cost":0.0,"kl":0.02785341627895832,"entropy":0.42962877452373505},{"step":117,"reward":-2.5396568775177,"core":-0.7563709616661072,"cost":-0.1540936529636383,"kl":0.04485970549285412,"entropy":0.4263489544391632},{"step":118,"reward":-2.870110273361206,"core":-0.8541045188903809,"cost":-0.2182660549879074,"kl":0.04658907279372215,"entropy":0.4703413397073746},{"step":119,"reward":-0.9024950861930847,"core":-0.20066122710704803,"cost":-0.0006882695015519857,"kl":0.030540451873093843,"entropy":0.41534270346164703},{"step":120,"reward":-2.366199493408203,"core":-0.6661303639411926,"cost":-0.18561434745788574,"kl":0.03486949857324362,"entropy":0.46186476945877075},{"step":121,"reward":-2.666168212890625,"core":-0.7403804659843445,"cost":-0.2214653491973877,"kl":0.020124954637140036,"entropy":0.3832852765917778},{"step":122,"reward":-1.334679126739502,"core":-0.28672340512275696,"cost":-0.058588847517967224,"kl":0.04239023011177778,"entropy":0.476081982254982},{"step":123,"reward":-1.7143951654434204,"core":-0.49408158659935,"cost":-0.04607511684298515,"kl":0.031892110127955675,"entropy":0.4749254286289215},{"step":124,"reward":-1.802504062652588,"core":-0.5617725253105164,"cost":-0.001581844873726368,"kl":0.03313151700422168,"entropy":0.4724032059311867},{"step":125,"reward":-3.1387603282928467,"core":-0.7935298085212708,"cost":-0.4356558322906494,"kl":0.03158772876486182,"entropy":0.4502989426255226},{"step":126,"reward":-2.3966064453125,"core":-0.7319110035896301,"cost":-0.09421149641275406,"kl":0.033161133993417025,"entropy":0.4924248531460762},{"step":127,"reward":-1.459202766418457,"core":-0.3737965524196625,"cost":-0.07200853526592255,"kl":0.032350340858101845,"entropy":0.38863223046064377},{"step":128,"reward":-1.0358606576919556,"core":-0.2769400477409363,"cost":-0.006875855382531881,"kl":0.03547431482002139,"entropy":0.4685356691479683},{"step":129,"reward":-3.08855938911438,"core":-0.9010340571403503,"cost":-0.2951997220516205,"kl":0.04250665847212076,"entropy":0.4966145306825638},{"step":130,"reward":-0.12073960155248642,"core":0.03966611251235008,"cost":-0.004807741846889257,"kl":0.035931576043367386,"entropy":0.44583092629909515},{"step":131,"reward":-4.433073997497559,"core":-1.307878017425537,"cost":-0.4442845582962036,"kl":0.03477930277585983,"entropy":0.46417880058288574},{"step":132,"reward":-3.2273166179656982,"core":-0.9073357582092285,"cost":-0.35376879572868347,"kl":0.03694336162880063,"entropy":0.48142585158348083},{"step":133,"reward":-1.0601469278335571,"core":-0.2889353036880493,"cost":-0.025359762832522392,"kl":0.05218574032187462,"entropy":0.4246600791811943},{"step":134,"reward":-2.3340749740600586,"core":-0.7310572266578674,"cost":-0.027073077857494354,"kl":0.04027447756379843,"entropy":0.43801149725914},{"step":135,"reward":-2.8781650066375732,"core":-0.8563321232795715,"cost":-0.20783424377441406,"kl":0.03480476001277566,"entropy":0.4617690220475197},{"step":136,"reward":-2.730536699295044,"core":-0.671941339969635,"cost":-0.36918509006500244,"kl":0.04034475237131119,"entropy":0.49283066391944885},{"step":137,"reward":-0.5974223017692566,"core":-0.10124386101961136,"cost":-0.01615038327872753,"kl":0.04656388983130455,"entropy":0.5011814460158348},{"step":138,"reward":-1.9992069005966187,"core":-0.5520565509796143,"cost":-0.14180679619312286,"kl":0.041793144308030605,"entropy":0.4190758317708969},{"step":139,"reward":-1.5250552892684937,"core":-0.43377116322517395,"cost":-0.04097753390669823,"kl":0.034070031717419624,"entropy":0.538079097867012},{"step":140,"reward":-1.2880676984786987,"core":-0.39077281951904297,"cost":-0.02185751497745514,"kl":0.07799261622130871,"entropy":0.5392422825098038},{"step":141,"reward":-2.097123384475708,"core":-0.6424384713172913,"cost":-0.06584268808364868,"kl":0.054825649596750736,"entropy":0.567357674241066},{"step":142,"reward":-2.3801021575927734,"core":-0.682230532169342,"cost":-0.17390620708465576,"kl":0.05018319748342037,"entropy":0.45144813507795334},{"step":143,"reward":-0.6055018901824951,"core":-0.15709196031093597,"cost":-0.001252349466085434,"kl":0.03329092264175415,"entropy":0.453620009124279},{"step":144,"reward":-1.4644664525985718,"core":-0.31838592886924744,"cost":-0.10455095767974854,"kl":0.06592551246285439,"entropy":0.4960903897881508},{"step":145,"reward":-2.790651559829712,"core":-0.8287734389305115,"cost":-0.19037775695323944,"kl":0.027366421185433865,"entropy":0.4621838554739952},{"step":146,"reward":-1.7753854990005493,"core":-0.45556876063346863,"cost":-0.15258125960826874,"kl":0.055804021656513214,"entropy":0.5009386539459229},{"step":147,"reward":-0.9208073616027832,"core":-0.24243034422397614,"cost":-0.01199283730238676,"kl":0.03757974458858371,"entropy":0.4485558196902275},{"step":148,"reward":-2.0611279010772705,"core":-0.5667583346366882,"cost":-0.15326786041259766,"kl":0.036448195576667786,"entropy":0.46030960977077484},{"step":149,"reward":-1.9271364212036133,"core":-0.5945044755935669,"cost":-0.02520575560629368,"kl":0.04203294590115547,"entropy":0.5386362448334694},{"step":150,"reward":-1.6921682357788086,"core":-0.513460636138916,"cost":-0.026223434135317802,"kl":0.03577421698719263,"entropy":0.4860406741499901},{"step":151,"reward":-2.045006513595581,"core":-0.6032252907752991,"cost":-0.11253374814987183,"kl":0.04405286628752947,"entropy":0.5562239736318588},{"step":152,"reward":-2.2082927227020264,"core":-0.5477713942527771,"cost":-0.25230109691619873,"kl":0.05673951003700495,"entropy":0.556455671787262},{"step":153,"reward":-1.71401846408844,"core":-0.47217658162117004,"cost":-0.11368910223245621,"kl":0.04443845618516207,"entropy":0.4704775959253311},{"step":154,"reward":-1.1774762868881226,"core":-0.3132030665874481,"cost":-0.004978967364877462,"kl":0.03627471812069416,"entropy":0.5129215568304062},{"step":155,"reward":-1.2672206163406372,"core":-0.36734339594841003,"cost":-0.031056882813572884,"kl":0.02618896448984742,"entropy":0.450632780790329},{"step":156,"reward":-1.819854736328125,"core":-0.5284361243247986,"cost":-0.10314170271158218,"kl":0.04823139728978276,"entropy":0.4758630692958832},{"step":157,"reward":-2.691610336303711,"core":-0.808158814907074,"cost":-0.14235670864582062,"kl":0.04205882363021374,"entropy":0.4434872195124626},{"step":158,"reward":-2.216756582260132,"core":-0.6024342179298401,"cost":-0.20196633040905,"kl":0.03938256809487939,"entropy":0.5531943887472153},{"step":159,"reward":-1.036450743675232,"core":-0.2576574385166168,"cost":-0.06967779994010925,"kl":0.07504043728113174,"entropy":0.5292960703372955},{"step":160,"reward":-1.4810810089111328,"core":-0.40737757086753845,"cost":-0.025991341099143028,"kl":0.03539997199550271,"entropy":0.4802926704287529},{"step":161,"reward":-1.6091009378433228,"core":-0.4700871407985687,"cost":-0.052749764174222946,"kl":0.02919907821342349,"entropy":0.4231031462550163},{"step":162,"reward":-0.7456437945365906,"core":-0.15366031229496002,"cost":-0.007123393472284079,"kl":0.031317823100835085,"entropy":0.5460691154003143},{"step":163,"reward":-2.375396251678467,"core":-0.730151891708374,"cost":-0.0674036368727684,"kl":0.03522633062675595,"entropy":0.5533442050218582},{"step":164,"reward":-1.7785167694091797,"core":-0.5490439534187317,"cost":-0.060707349330186844,"kl":0.04548656661063433,"entropy":0.4395672082901001},{"step":165,"reward":-1.220090389251709,"core":-0.32410892844200134,"cost":-0.005041855853050947,"kl":0.04256645496934652,"entropy":0.5564462542533875},{"step":166,"reward":-1.7387045621871948,"core":-0.5321893095970154,"cost":-0.08463522046804428,"kl":0.0397948008030653,"entropy":0.48454172164201736},{"step":167,"reward":-1.422903060913086,"core":-0.43330708146095276,"cost":-0.0459168441593647,"kl":0.026840523816645145,"entropy":0.4320089668035507},{"step":168,"reward":-1.9114452600479126,"core":-0.5476529002189636,"cost":-0.05100097134709358,"kl":0.04131976142525673,"entropy":0.512296050786972},{"step":169,"reward":-1.4027668237686157,"core":-0.4344519376754761,"cost":0.0,"kl":0.03130568517372012,"entropy":0.4989214837551117},{"step":170,"reward":-2.8050014972686768,"core":-0.8479993939399719,"cost":-0.12354723364114761,"kl":0.03168346546590328,"entropy":0.5138235837221146},{"step":171,"reward":-2.4585354328155518,"core":-0.7728075981140137,"cost":-0.07319509238004684,"kl":0.04025254584848881,"entropy":0.5375296473503113},{"step":172,"reward":-1.8138364553451538,"core":-0.519984781742096,"cost":-0.0349782295525074,"kl":0.027371076866984367,"entropy":0.47169607132673264},{"step":173,"reward":-2.081848382949829,"core":-0.6246026158332825,"cost":-0.10991159826517105,"kl":0.035221054684370756,"entropy":0.492868609726429},{"step":174,"reward":-1.6186065673828125,"core":-0.46565818786621094,"cost":-0.07925855368375778,"kl":0.04032964026555419,"entropy":0.4355178326368332},{"step":175,"reward":-2.6719284057617188,"core":-0.8497582077980042,"cost":-0.06659670919179916,"kl":0.03723800275474787,"entropy":0.4779684990644455},{"step":176,"reward":-1.6677179336547852,"core":-0.49010732769966125,"cost":-0.054798055440187454,"kl":0.029952597804367542,"entropy":0.4873868450522423},{"step":177,"reward":-1.7802133560180664,"core":-0.4866790771484375,"cost":-0.08072706311941147,"kl":0.03437752649188042,"entropy":0.5872653871774673},{"step":178,"reward":-1.76779305934906,"core":-0.4937962591648102,"cost":-0.12383658438920975,"kl":0.039913034066557884,"entropy":0.48688289523124695},{"step":179,"reward":-2.09769344329834,"core":-0.6110026240348816,"cost":-0.15858830511569977,"kl":0.03783585736528039,"entropy":0.5608621388673782},{"step":180,"reward":-1.93289053440094,"core":-0.5645472407341003,"cost":-0.059328917413949966,"kl":0.04740254767239094,"entropy":0.45829828828573227},{"step":181,"reward":-1.5379527807235718,"core":-0.3796713650226593,"cost":-0.10428047925233841,"kl":0.02773209335282445,"entropy":0.47617995738983154},{"step":182,"reward":-1.4504307508468628,"core":-0.4192063808441162,"cost":-0.032426346093416214,"kl":0.02449404774233699,"entropy":0.4389878585934639},{"step":183,"reward":-1.2027887105941772,"core":-0.36730512976646423,"cost":0.0,"kl":0.031018961686640978,"entropy":0.4823528751730919},{"step":184,"reward":-1.7121505737304688,"core":-0.5008276700973511,"cost":-0.0783076286315918,"kl":0.043635676614940166,"entropy":0.5616630762815475},{"step":185,"reward":-1.7732343673706055,"core":-0.5155909657478333,"cost":-0.062210798263549805,"kl":0.033278589602559805,"entropy":0.49908165633678436},{"step":186,"reward":-2.4641830921173096,"core":-0.7314499020576477,"cost":-0.1445768028497696,"kl":0.03372355177998543,"entropy":0.35190100967884064},{"step":187,"reward":-0.6694967746734619,"core":-0.10057145357131958,"cost":-0.02336397022008896,"kl":0.0547263715416193,"entropy":0.5100136324763298},{"step":188,"reward":-0.7161757349967957,"core":-0.15390081703662872,"cost":0.0,"kl":0.030652498360723257,"entropy":0.45907068252563477},{"step":189,"reward":-1.651412844657898,"core":-0.5066428780555725,"cost":-0.020021511241793633,"kl":0.029771601781249046,"entropy":0.5835612118244171},{"step":190,"reward":-2.29744029045105,"core":-0.697070300579071,"cost":-0.10584231466054916,"kl":0.023814002983272076,"entropy":0.4381798133254051},{"step":191,"reward":-2.654632091522217,"core":-0.8341339230537415,"cost":-0.024349212646484375,"kl":0.02340984856709838,"entropy":0.3943141996860504},{"step":192,"reward":-0.7935647964477539,"core":-0.1464395374059677,"cost":0.0,"kl":0.02708719251677394,"entropy":0.5167761519551277},{"step":193,"reward":-0.5000048875808716,"core":-0.037287984043359756,"cost":-0.02302701771259308,"kl":0.04454295802861452,"entropy":0.47213731706142426},{"step":194,"reward":-1.986714482307434,"core":-0.5813704133033752,"cost":-0.11633185297250748,"kl":0.05313301179558039,"entropy":0.500447042286396},{"step":195,"reward":-1.1799311637878418,"core":-0.2571229636669159,"cost":-0.033233169466257095,"kl":0.046866013668477535,"entropy":0.432698093354702},{"step":196,"reward":-1.558031678199768,"core":-0.4635566174983978,"cost":-0.02400326170027256,"kl":0.04446485824882984,"entropy":0.43895580619573593},{"step":197,"reward":-0.9687283635139465,"core":-0.21744263172149658,"cost":-0.03296215459704399,"kl":0.024721969850361347,"entropy":0.4230239912867546},{"step":198,"reward":-1.9922860860824585,"core":-0.5631420016288757,"cost":-0.0772269070148468,"kl":0.02324773045256734,"entropy":0.3965666592121124},{"step":199,"reward":-0.666378915309906,"core":-0.13694192469120026,"cost":0.0,"kl":0.04158607963472605,"entropy":0.4597154036164284},{"step":200,"reward":-1.8496507406234741,"core":-0.5197243094444275,"cost":-0.07103829830884933,"kl":0.030921743251383305,"entropy":0.44826652854681015},{"step":201,"reward":-2.051950216293335,"core":-0.6111276745796204,"cost":-0.023529043421149254,"kl":0.04738902021199465,"entropy":0.5251339301466942},{"step":202,"reward":-1.9205193519592285,"core":-0.5423125624656677,"cost":-0.127332866191864,"kl":0.03395934123545885,"entropy":0.4808323234319687},{"step":203,"reward":-1.6265054941177368,"core":-0.47599053382873535,"cost":-0.03389841690659523,"kl":0.046194033697247505,"entropy":0.5053495988249779},{"step":204,"reward":-1.958590030670166,"core":-0.5867967009544373,"cost":-0.07490023225545883,"kl":0.04295419855043292,"entropy":0.435610294342041},{"step":205,"reward":-1.345889925956726,"core":-0.35625267028808594,"cost":-0.028135672211647034,"kl":0.03892820281907916,"entropy":0.49585143476724625},{"step":206,"reward":-1.1883045434951782,"core":-0.3209795653820038,"cost":-0.09618327021598816,"kl":0.049549649469554424,"entropy":0.5579586327075958},{"step":207,"reward":-1.9460786581039429,"core":-0.5633491277694702,"cost":-0.1335323452949524,"kl":0.036482210736721754,"entropy":0.5001472309231758},{"step":208,"reward":-1.3047642707824707,"core":-0.4101113975048065,"cost":-0.028188256546854973,"kl":0.03604761464521289,"entropy":0.5117781013250351},{"step":209,"reward":-2.492446184158325,"core":-0.7315356135368347,"cost":-0.18601460754871368,"kl":0.02959994040429592,"entropy":0.40879253298044205},{"step":210,"reward":-1.560436725616455,"core":-0.44165778160095215,"cost":-0.025341197848320007,"kl":0.04579395428299904,"entropy":0.4735047221183777},{"step":211,"reward":-1.9026333093643188,"core":-0.6114562153816223,"cost":0.0,"kl":0.043941838666796684,"entropy":0.5302275568246841},{"step":212,"reward":-1.7188129425048828,"core":-0.45171037316322327,"cost":-0.1039019525051117,"kl":0.07461753115057945,"entropy":0.4187764525413513},{"step":213,"reward":-1.2200418710708618,"core":-0.35128697752952576,"cost":-0.006171732675284147,"kl":0.04224791517481208,"entropy":0.47903063148260117},{"step":214,"reward":-0.9167702198028564,"core":-0.22263894975185394,"cost":-0.009322778321802616,"kl":0.04508906789124012,"entropy":0.5200869292020798},{"step":215,"reward":-1.2794322967529297,"core":-0.38545623421669006,"cost":-0.05264595150947571,"kl":0.05080180708318949,"entropy":0.5045078471302986},{"step":216,"reward":-2.1766579151153564,"core":-0.6335697174072266,"cost":-0.1696218103170395,"kl":0.03941371198743582,"entropy":0.5125104039907455},{"step":217,"reward":-2.8182432651519775,"core":-0.8576071262359619,"cost":-0.14391180872917175,"kl":0.027416559401899576,"entropy":0.36575692892074585},{"step":218,"reward":-2.2061784267425537,"core":-0.640225350856781,"cost":-0.127157062292099,"kl":0.06287282053381205,"entropy":0.500908337533474},{"step":219,"reward":-1.935872197151184,"core":-0.5271640419960022,"cost":-0.10586690902709961,"kl":0.05036862939596176,"entropy":0.5170655846595764},{"step":220,"reward":-3.1107990741729736,"core":-0.9217929244041443,"cost":-0.24345733225345612,"kl":0.028312419541180134,"entropy":0.43863947689533234},{"step":221,"reward":-2.2634246349334717,"core":-0.6693703532218933,"cost":-0.09309092164039612,"kl":0.042639292776584625,"entropy":0.45144931226968765},{"step":222,"reward":-1.5094267129898071,"core":-0.451081782579422,"cost":-0.06859137862920761,"kl":0.04847069550305605,"entropy":0.5279631614685059},{"step":223,"reward":-2.549809455871582,"core":-0.7638681530952454,"cost":-0.1426544040441513,"kl":0.053557357750833035,"entropy":0.4323234409093857},{"step":224,"reward":-1.9422391653060913,"core":-0.6007647514343262,"cost":-0.06661536544561386,"kl":0.046851896680891514,"entropy":0.45726825296878815},{"step":225,"reward":-1.1181386709213257,"core":-0.2979971766471863,"cost":-0.0056627974845469,"kl":0.03768374491482973,"entropy":0.4929715320467949},{"step":226,"reward":-3.0864505767822266,"core":-0.9172770380973816,"cost":-0.19412358105182648,"kl":0.031816094648092985,"entropy":0.3675955906510353},{"step":227,"reward":-1.4705867767333984,"core":-0.419274240732193,"cost":-0.06043046712875366,"kl":0.05334600526839495,"entropy":0.5438427627086639},{"step":228,"reward":-2.0056817531585693,"core":-0.5669438242912292,"cost":-0.05960698425769806,"kl":0.050374927930533886,"entropy":0.4168771803379059},{"step":229,"reward":-1.2279378175735474,"core":-0.2697964012622833,"cost":-0.0668981745839119,"kl":0.05901752691715956,"entropy":0.5114161968231201},{"step":230,"reward":-1.4650425910949707,"core":-0.38336387276649475,"cost":-0.05603785440325737,"kl":0.0490577882155776,"entropy":0.5099918246269226},{"step":231,"reward":-0.6722910404205322,"core":-0.17082057893276215,"cost":0.0,"kl":0.04253681655973196,"entropy":0.43083488941192627},{"step":232,"reward":-2.0441086292266846,"core":-0.5923446416854858,"cost":-0.10445376485586166,"kl":0.051900146529078484,"entropy":0.4588749036192894},{"step":233,"reward":-3.103036880493164,"core":-0.9168221950531006,"cost":-0.2591830790042877,"kl":0.052203587256371975,"entropy":0.5100400373339653},{"step":234,"reward":-3.1767947673797607,"core":-0.9976296424865723,"cost":-0.1186593696475029,"kl":0.055052878335118294,"entropy":0.5239097625017166},{"step":235,"reward":-1.880520224571228,"core":-0.5593385100364685,"cost":-0.08585712313652039,"kl":0.046826968900859356,"entropy":0.49256838113069534},{"step":236,"reward":-2.6931211948394775,"core":-0.7861868739128113,"cost":-0.27411481738090515,"kl":0.06672079209238291,"entropy":0.5384384244680405},{"step":237,"reward":-0.49225592613220215,"core":-0.10113563388586044,"cost":-0.007283790502697229,"kl":0.08511119894683361,"entropy":0.5311165601015091},{"step":238,"reward":-0.4031655788421631,"core":0.023154446855187416,"cost":0.0,"kl":0.051772051490843296,"entropy":0.46369941532611847},{"step":239,"reward":-2.6402690410614014,"core":-0.7951690554618835,"cost":-0.14890794456005096,"kl":0.05697371065616608,"entropy":0.4296174719929695},{"step":240,"reward":-1.0789562463760376,"core":-0.2787538766860962,"cost":-0.047263339161872864,"kl":0.03533994033932686,"entropy":0.4308491423726082},{"step":241,"reward":-0.8254073262214661,"core":-0.16583049297332764,"cost":-0.005368141923099756,"kl":0.07448568940162659,"entropy":0.4980719983577728},{"step":242,"reward":-1.1246137619018555,"core":-0.30899155139923096,"cost":-0.004426129627972841,"kl":0.0665891133248806,"entropy":0.5241402685642242},{"step":243,"reward":-1.8072538375854492,"core":-0.5459403395652771,"cost":-0.0378122441470623,"kl":0.06391307059675455,"entropy":0.4250495806336403},{"step":244,"reward":-2.960278272628784,"core":-0.916824996471405,"cost":-0.14457736909389496,"kl":0.06009416375309229,"entropy":0.45267798006534576},{"step":245,"reward":-0.67409747838974,"core":-0.1608019471168518,"cost":-0.015865756198763847,"kl":0.05696870293468237,"entropy":0.4382988438010216},{"step":246,"reward":-1.4117072820663452,"core":-0.3891889154911041,"cost":-0.007559212390333414,"kl":0.04642812814563513,"entropy":0.40067947655916214},{"step":247,"reward":-1.7198642492294312,"core":-0.5018365979194641,"cost":-0.012188319116830826,"kl":0.040010550525039434,"entropy":0.49141256511211395},{"step":248,"reward":-2.160161256790161,"core":-0.5816591382026672,"cost":-0.1359221190214157,"kl":0.08517420943826437,"entropy":0.48846329003572464},{"step":249,"reward":-2.1688344478607178,"core":-0.5897452235221863,"cost":-0.16638465225696564,"kl":0.09710025787353516,"entropy":0.6147425770759583},{"step":250,"reward":-1.2625385522842407,"core":-0.3478606641292572,"cost":-0.05594765767455101,"kl":0.047474998049438,"entropy":0.48416832834482193},{"step":251,"reward":-1.8883399963378906,"core":-0.5429142117500305,"cost":-0.15722407400608063,"kl":0.12305713444948196,"entropy":0.5370851829648018},{"step":252,"reward":-1.2378567457199097,"core":-0.32962891459465027,"cost":-0.05131205916404724,"kl":0.06547683756798506,"entropy":0.43977879732847214},{"step":253,"reward":-2.2359278202056885,"core":-0.6694723963737488,"cost":-0.08724824339151382,"kl":0.04196629673242569,"entropy":0.4501051530241966},{"step":254,"reward":-1.2894703149795532,"core":-0.34152325987815857,"cost":-0.0993824228644371,"kl":0.06705165654420853,"entropy":0.5336062461137772},{"step":255,"reward":-1.9027894735336304,"core":-0.5219376683235168,"cost":-0.066855289041996,"kl":0.06789220403879881,"entropy":0.4915686622262001},{"step":256,"reward":-2.53794002532959,"core":-0.6995396614074707,"cost":-0.24679066240787506,"kl":0.06649971567094326,"entropy":0.5374817997217178},{"step":257,"reward":-1.6660178899765015,"core":-0.46494463086128235,"cost":-0.11141311377286911,"kl":0.06181193282827735,"entropy":0.4508693143725395},{"step":258,"reward":-1.267612338066101,"core":-0.3545961380004883,"cost":-0.06261097639799118,"kl":0.07178179547190666,"entropy":0.5381389260292053},{"step":259,"reward":-2.171499490737915,"core":-0.6641564965248108,"cost":-0.1183629035949707,"kl":0.05085176322609186,"entropy":0.5379015058279037},{"step":260,"reward":-2.538874387741089,"core":-0.7635443806648254,"cost":-0.14175663888454437,"kl":0.04210210870951414,"entropy":0.4365225210785866},{"step":261,"reward":-2.429692029953003,"core":-0.7622893452644348,"cost":-0.07747229188680649,"kl":0.07333346270024776,"entropy":0.4567064046859741},{"step":262,"reward":-2.4681003093719482,"core":-0.7287287712097168,"cost":-0.14255546033382416,"kl":0.05756524857133627,"entropy":0.497140109539032},{"step":263,"reward":-0.5924819111824036,"core":-0.11259619146585464,"cost":-0.030011633411049843,"kl":0.07265002653002739,"entropy":0.5466247946023941},{"step":264,"reward":-1.1857181787490845,"core":-0.2867170572280884,"cost":-0.06472586840391159,"kl":0.07268789038062096,"entropy":0.4874114990234375},{"step":265,"reward":-2.478090524673462,"core":-0.7593040466308594,"cost":-0.10941710323095322,"kl":0.04596382100135088,"entropy":0.4961842969059944},{"step":266,"reward":-1.8665457963943481,"core":-0.562845766544342,"cost":-0.060335922986269,"kl":0.06215763185173273,"entropy":0.4871991351246834},{"step":267,"reward":-2.0787298679351807,"core":-0.5692480206489563,"cost":-0.1683206558227539,"kl":0.06254930701106787,"entropy":0.5679266452789307},{"step":268,"reward":-2.2677276134490967,"core":-0.6379678845405579,"cost":-0.16604973375797272,"kl":0.049351347610354424,"entropy":0.5480923801660538},{"step":269,"reward":-1.1194430589675903,"core":-0.3170899450778961,"cost":0.0,"kl":0.05174052529036999,"entropy":0.5070660412311554},{"step":270,"reward":-1.4361282587051392,"core":-0.4118439853191376,"cost":-0.032669294625520706,"kl":0.04271886218339205,"entropy":0.4836215078830719},{"step":271,"reward":-1.521685004234314,"core":-0.42887893319129944,"cost":-0.07096060365438461,"kl":0.09299513790756464,"entropy":0.44422314316034317},{"step":272,"reward":-0.6488382816314697,"core":-0.052463650703430176,"cost":-0.014232742600142956,"kl":0.06550496816635132,"entropy":0.48297179490327835},{"step":273,"reward":-2.7787282466888428,"core":-0.7837563157081604,"cost":-0.28306877613067627,"kl":0.04484897945076227,"entropy":0.41061586141586304},{"step":274,"reward":-0.8348515033721924,"core":-0.2043752819299698,"cost":0.0,"kl":0.05073011014610529,"entropy":0.4475657418370247},{"step":275,"reward":-2.1957619190216064,"core":-0.6847003102302551,"cost":-0.04475430026650429,"kl":0.048601602204144,"entropy":0.5320816189050674},{"step":276,"reward":-1.2080875635147095,"core":-0.2920457124710083,"cost":-0.04651275649666786,"kl":0.026416488457471132,"entropy":0.40715719014406204},{"step":277,"reward":-2.980109453201294,"core":-0.8976235389709473,"cost":-0.15059104561805725,"kl":0.06402718368917704,"entropy":0.43553125858306885},{"step":278,"reward":-1.465173363685608,"core":-0.41586318612098694,"cost":-0.07461447268724442,"kl":0.06061189342290163,"entropy":0.5083194449543953},{"step":279,"reward":-1.5736265182495117,"core":-0.407639741897583,"cost":-0.1872115582227707,"kl":0.06176465656608343,"entropy":0.5709725320339203},{"step":280,"reward":-1.297745704650879,"core":-0.3277602791786194,"cost":-0.024167010560631752,"kl":0.0708583714440465,"entropy":0.5309371650218964},{"step":281,"reward":-2.0223562717437744,"core":-0.5578075051307678,"cost":-0.0863051638007164,"kl":0.06949823629111052,"entropy":0.5782329887151718},{"step":282,"reward":-1.7998195886611938,"core":-0.5165557265281677,"cost":-0.11730726808309555,"kl":0.039778691716492176,"entropy":0.4541880860924721},{"step":283,"reward":-1.8128350973129272,"core":-0.501594066619873,"cost":-0.03685769811272621,"kl":0.05183510389178991,"entropy":0.500906728208065},{"step":284,"reward":-2.2444493770599365,"core":-0.697699248790741,"cost":-0.05775247886776924,"kl":0.034726346377283335,"entropy":0.47768816351890564},{"step":285,"reward":-1.772576928138733,"core":-0.4998030662536621,"cost":-0.06099382042884827,"kl":0.05133209889754653,"entropy":0.47060298174619675},{"step":286,"reward":-1.874606966972351,"core":-0.469390869140625,"cost":-0.13090823590755463,"kl":0.042042167857289314,"entropy":0.47247033566236496},{"step":287,"reward":-2.0099871158599854,"core":-0.5251875519752502,"cost":-0.13369043171405792,"kl":0.0424575568176806,"entropy":0.4906645268201828},{"step":288,"reward":-2.0992891788482666,"core":-0.6226547956466675,"cost":-0.059096988290548325,"kl":0.06120136193931103,"entropy":0.5805776566267014},{"step":289,"reward":-1.301047921180725,"core":-0.3724222183227539,"cost":-0.018626488745212555,"kl":0.04243824351578951,"entropy":0.4820858985185623},{"step":290,"reward":-1.7395519018173218,"core":-0.460540771484375,"cost":-0.0894840732216835,"kl":0.05656887963414192,"entropy":0.4375137463212013},{"step":291,"reward":-0.8855708241462708,"core":-0.21599610149860382,"cost":0.0,"kl":0.064090964384377,"entropy":0.48646266758441925},{"step":292,"reward":-1.5746475458145142,"core":-0.43425431847572327,"cost":-0.027792977169156075,"kl":0.05150845926254988,"entropy":0.5169715955853462},{"step":293,"reward":-2.278419256210327,"core":-0.6558372974395752,"cost":-0.1572420746088028,"kl":0.056455028243362904,"entropy":0.5637415498495102},{"step":294,"reward":-1.4854621887207031,"core":-0.4205828607082367,"cost":-0.05573849007487297,"kl":0.07137884479016066,"entropy":0.4247393384575844},{"step":295,"reward":-0.4860287606716156,"core":-0.07211670279502869,"cost":-0.0008804491371847689,"kl":0.04636729508638382,"entropy":0.44640781730413437},{"step":296,"reward":-1.2606439590454102,"core":-0.3221410810947418,"cost":-0.04166724905371666,"kl":0.06583661399781704,"entropy":0.46017953008413315},{"step":297,"reward":-0.859577476978302,"core":-0.24345850944519043,"cost":-0.0006098754820413888,"kl":0.05777100520208478,"entropy":0.5178085267543793},{"step":298,"reward":-2.727757215499878,"core":-0.8195399641990662,"cost":-0.18580299615859985,"kl":0.047323946841061115,"entropy":0.47632621228694916},{"step":299,"reward":-0.6734176278114319,"core":-0.15879982709884644,"cost":-0.006369141861796379,"kl":0.05403432436287403,"entropy":0.5334012359380722},{"step":300,"reward":-1.755103588104248,"core":-0.5037055611610413,"cost":-0.06165743246674538,"kl":0.03637733869254589,"entropy":0.5475235432386398},{"step":301,"reward":-1.925990104675293,"core":-0.5853980183601379,"cost":-0.04602288082242012,"kl":0.053641755133867264,"entropy":0.5382818132638931},{"step":302,"reward":-1.3070679903030396,"core":-0.37370941042900085,"cost":-0.013809953816235065,"kl":0.05038193892687559,"entropy":0.538753017783165},{"step":303,"reward":-2.5454041957855225,"core":-0.7710792422294617,"cost":-0.1678939312696457,"kl":0.04442026186734438,"entropy":0.5116926208138466},{"step":304,"reward":-2.3065035343170166,"core":-0.6785561442375183,"cost":-0.055524200201034546,"kl":0.05639322102069855,"entropy":0.4409317672252655},{"step":305,"reward":-0.25969985127449036,"core":0.07791989296674728,"cost":0.0,"kl":0.0695979529991746,"entropy":0.4661707505583763},{"step":306,"reward":-1.4260097742080688,"core":-0.4207886755466461,"cost":-0.001983163645491004,"kl":0.03631359105929732,"entropy":0.4370770677924156},{"step":307,"reward":-1.3047369718551636,"core":-0.29924193024635315,"cost":-0.024945423007011414,"kl":0.042885602451860905,"entropy":0.4394644945859909},{"step":308,"reward":-0.7614107728004456,"core":-0.09929446130990982,"cost":-0.06296428292989731,"kl":0.027781112119555473,"entropy":0.4763399362564087},{"step":309,"reward":-2.313364267349243,"core":-0.6769011616706848,"cost":-0.16814126074314117,"kl":0.06338134780526161,"entropy":0.517691545188427},{"step":310,"reward":-0.9433987736701965,"core":-0.21702013909816742,"cost":-0.03112632781267166,"kl":0.07322246301919222,"entropy":0.5000339224934578},{"step":311,"reward":-1.2704384326934814,"core":-0.3307702839374542,"cost":-0.02415355294942856,"kl":0.04117505718022585,"entropy":0.48235417902469635},{"step":312,"reward":-0.822368323802948,"core":-0.19747357070446014,"cost":-0.01724529080092907,"kl":0.044005285017192364,"entropy":0.4823683947324753},{"step":313,"reward":-2.586447238922119,"core":-0.8036448955535889,"cost":-0.09686976671218872,"kl":0.041840165853500366,"entropy":0.44594235718250275},{"step":314,"reward":-1.6555460691452026,"core":-0.4502575397491455,"cost":-0.1119149699807167,"kl":0.047879441641271114,"entropy":0.5166584551334381},{"step":315,"reward":-2.14229416847229,"core":-0.6473802924156189,"cost":-0.08605346828699112,"kl":0.04850201681256294,"entropy":0.46943284571170807},{"step":316,"reward":-2.120218276977539,"core":-0.6282370686531067,"cost":-0.08417675644159317,"kl":0.0420368192717433,"entropy":0.45420844852924347},{"step":317,"reward":-1.828641414642334,"core":-0.5401734709739685,"cost":-0.03277517482638359,"kl":0.08830992737784982,"entropy":0.44119662046432495},{"step":318,"reward":-2.5499045848846436,"core":-0.7087190747261047,"cost":-0.2294357270002365,"kl":0.04833944607526064,"entropy":0.4964792877435684},{"step":319,"reward":-1.2036703824996948,"core":-0.26833805441856384,"cost":-0.07026176899671555,"kl":0.05307674594223499,"entropy":0.4907682090997696},{"step":320,"reward":-2.2510383129119873,"core":-0.696343183517456,"cost":-0.042940422892570496,"kl":0.047345967032015324,"entropy":0.48535604774951935},{"step":321,"reward":-1.7052398920059204,"core":-0.505230188369751,"cost":-0.06385473161935806,"kl":0.04933631233870983,"entropy":0.5097259059548378},{"step":322,"reward":-2.0286505222320557,"core":-0.6295222043991089,"cost":-0.0473470501601696,"kl":0.07055539637804031,"entropy":0.5787259638309479},{"step":323,"reward":-1.3373256921768188,"core":-0.3523973524570465,"cost":-0.03658665344119072,"kl":0.051368855871260166,"entropy":0.5568531453609467},{"step":324,"reward":-0.8806589245796204,"core":-0.1940097063779831,"cost":-0.027423551306128502,"kl":0.03587117651477456,"entropy":0.4579874202609062},{"step":325,"reward":-1.408887505531311,"core":-0.38079333305358887,"cost":-0.015682222321629524,"kl":0.05106010474264622,"entropy":0.5379340201616287},{"step":326,"reward":-1.3012527227401733,"core":-0.3033181130886078,"cost":-0.06821099668741226,"kl":0.05907731503248215,"entropy":0.5538786724209785},{"step":327,"reward":-1.2547394037246704,"core":-0.37834668159484863,"cost":-0.042825788259506226,"kl":0.05204735789448023,"entropy":0.5155608206987381},{"step":328,"reward":-1.1930135488510132,"core":-0.3377479016780853,"cost":0.0,"kl":0.03877085819840431,"entropy":0.5179159492254257},{"step":329,"reward":-1.4785155057907104,"core":-0.3615554869174957,"cost":-0.10991144925355911,"kl":0.0695949625223875,"entropy":0.5608047917485237},{"step":330,"reward":-1.941301941871643,"core":-0.5021381974220276,"cost":-0.1270586997270584,"kl":0.03181187063455582,"entropy":0.4735181927680969},{"step":331,"reward":-1.1254161596298218,"core":-0.27110719680786133,"cost":-0.07616981863975525,"kl":0.07109145633876324,"entropy":0.5088380053639412},{"step":332,"reward":-2.3924291133880615,"core":-0.7454533576965332,"cost":-0.02727988362312317,"kl":0.04321394767612219,"entropy":0.49091506004333496},{"step":333,"reward":-1.3337997198104858,"core":-0.3560445308685303,"cost":-0.04428096115589142,"kl":0.06011708965525031,"entropy":0.4802754372358322},{"step":334,"reward":-1.8799108266830444,"core":-0.5479408502578735,"cost":-0.08505301922559738,"kl":0.048847944010049105,"entropy":0.5808033049106598},{"step":335,"reward":-0.5190951228141785,"core":-0.08318230509757996,"cost":-0.0008791661239229143,"kl":0.04575447924435139,"entropy":0.43380290269851685},{"step":336,"reward":-1.9864381551742554,"core":-0.6024838089942932,"cost":-0.06665459275245667,"kl":0.05857406370341778,"entropy":0.5243329703807831},{"step":337,"reward":-1.7795085906982422,"core":-0.5014883875846863,"cost":-0.07180272787809372,"kl":0.06061458494514227,"entropy":0.5307524055242538},{"step":338,"reward":-3.2468347549438477,"core":-0.9613788723945618,"cost":-0.24968492984771729,"kl":0.09435325115919113,"entropy":0.5448844060301781},{"step":339,"reward":-1.4897149801254272,"core":-0.4153849184513092,"cost":-0.03495781496167183,"kl":0.03745973855257034,"entropy":0.545478992164135},{"step":340,"reward":-1.7219074964523315,"core":-0.4591798782348633,"cost":-0.05220508202910423,"kl":0.07490995340049267,"entropy":0.5798707157373428},{"step":341,"reward":-0.8292410373687744,"core":-0.1453753262758255,"cost":-0.02671993523836136,"kl":0.050020113587379456,"entropy":0.41819918900728226},{"step":342,"reward":-1.2038882970809937,"core":-0.2520129978656769,"cost":-0.05539178475737572,"kl":0.05854228138923645,"entropy":0.522513821721077},{"step":343,"reward":-0.6439521908760071,"core":-0.12672485411167145,"cost":-0.014639387838542461,"kl":0.05183115229010582,"entropy":0.5216490030288696},{"step":344,"reward":-1.190705418586731,"core":-0.296334832906723,"cost":-0.026296937838196754,"kl":0.061749642714858055,"entropy":0.5292538776993752},{"step":345,"reward":-1.0544673204421997,"core":-0.2846517860889435,"cost":-0.03077598474919796,"kl":0.046945168636739254,"entropy":0.5211437046527863},{"step":346,"reward":-2.109067678451538,"core":-0.6484044194221497,"cost":-0.04325924441218376,"kl":0.04984928714111447,"entropy":0.46024785935878754},{"step":347,"reward":-0.9735413193702698,"core":-0.22537291049957275,"cost":-0.04912913218140602,"kl":0.06277884170413017,"entropy":0.520222544670105},{"step":348,"reward":-2.161128044128418,"core":-0.6097429394721985,"cost":-0.14292274415493011,"kl":0.055388469249010086,"entropy":0.5216960683465004},{"step":349,"reward":-0.3732466697692871,"core":0.038065988570451736,"cost":-0.025286631658673286,"kl":0.061271690763533115,"entropy":0.488936148583889},{"step":350,"reward":-2.3422019481658936,"core":-0.7158953547477722,"cost":-0.09616854041814804,"kl":0.05212805047631264,"entropy":0.5887040197849274},{"step":351,"reward":-0.11630505323410034,"core":0.043654490262269974,"cost":0.0,"kl":0.050198948942124844,"entropy":0.5779687762260437},{"step":352,"reward":-1.5995839834213257,"core":-0.4757899343967438,"cost":-0.03798438236117363,"kl":0.061182353645563126,"entropy":0.5853861272335052},{"step":353,"reward":-1.5154671669006348,"core":-0.4370799958705902,"cost":-0.01249813660979271,"kl":0.03807851625606418,"entropy":0.46632420271635056},{"step":354,"reward":-1.3482857942581177,"core":-0.37143436074256897,"cost":-0.04477165639400482,"kl":0.04697628226131201,"entropy":0.5192466080188751},{"step":355,"reward":-2.035660982131958,"core":-0.5833260416984558,"cost":-0.0912570133805275,"kl":0.046378239057958126,"entropy":0.4271078482270241},{"step":356,"reward":-0.925814151763916,"core":-0.2192186564207077,"cost":-0.010157736949622631,"kl":0.06852180138230324,"entropy":0.523883119225502},{"step":357,"reward":-1.3533892631530762,"core":-0.3984433710575104,"cost":-0.06834333389997482,"kl":0.07385847065597773,"entropy":0.5483797192573547},{"step":358,"reward":-1.9753111600875854,"core":-0.5615542531013489,"cost":-0.15001682937145233,"kl":0.06249390076845884,"entropy":0.5109242871403694},{"step":359,"reward":-0.9416065216064453,"core":-0.21688242256641388,"cost":-0.014406814239919186,"kl":0.03516856115311384,"entropy":0.4629799872636795},{"step":360,"reward":-1.7970705032348633,"core":-0.46591100096702576,"cost":-0.11262211948633194,"kl":0.08271803334355354,"entropy":0.5215542912483215},{"step":361,"reward":-1.8137292861938477,"core":-0.5053392052650452,"cost":-0.08852460980415344,"kl":0.048925235867500305,"entropy":0.4807850569486618},{"step":362,"reward":-1.0433436632156372,"core":-0.2862158715724945,"cost":-0.017821798101067543,"kl":0.07534261140972376,"entropy":0.603614330291748},{"step":363,"reward":-1.476679801940918,"core":-0.3957299292087555,"cost":-0.08256658166646957,"kl":0.059977068565785885,"entropy":0.5602252781391144},{"step":364,"reward":-2.7744834423065186,"core":-0.8402120471000671,"cost":-0.1330055445432663,"kl":0.056172480806708336,"entropy":0.48853443562984467},{"step":365,"reward":-1.118111252784729,"core":-0.27326667308807373,"cost":-0.06943506747484207,"kl":0.09823400899767876,"entropy":0.5403489321470261},{"step":366,"reward":-0.9598702788352966,"core":-0.25981274247169495,"cost":0.0,"kl":0.05176623724400997,"entropy":0.520323246717453},{"step":367,"reward":-1.328975796699524,"core":-0.35940852761268616,"cost":-0.060054078698158264,"kl":0.05195420142263174,"entropy":0.5257394015789032},{"step":368,"reward":-0.8064271807670593,"core":-0.13022863864898682,"cost":-0.020734494552016258,"kl":0.05611621029675007,"entropy":0.5054675936698914},{"step":369,"reward":-0.1640448123216629,"core":0.04682275280356407,"cost":-0.03406361863017082,"kl":0.06548788491636515,"entropy":0.5549253523349762},{"step":370,"reward":-1.0142167806625366,"core":-0.2546975910663605,"cost":-0.000724515353795141,"kl":0.042674449272453785,"entropy":0.47401515394449234},{"step":371,"reward":-0.7909810543060303,"core":-0.19729311764240265,"cost":-0.012123367749154568,"kl":0.05592873226851225,"entropy":0.5271269530057907},{"step":372,"reward":-1.4307283163070679,"core":-0.4145151674747467,"cost":-0.035105716437101364,"kl":0.07792830001562834,"entropy":0.5585203021764755},{"step":373,"reward":-3.858552932739258,"core":-1.1751171350479126,"cost":-0.26830029487609863,"kl":0.041089328937232494,"entropy":0.4573659524321556},{"step":374,"reward":-1.1197868585586548,"core":-0.24849790334701538,"cost":-0.03826213628053665,"kl":0.0830884650349617,"entropy":0.4579138085246086},{"step":375,"reward":-1.6278858184814453,"core":-0.47531846165657043,"cost":-0.09271644800901413,"kl":0.06345317233353853,"entropy":0.5364722460508347},{"step":376,"reward":-0.7912171483039856,"core":-0.1769501119852066,"cost":-0.011575856246054173,"kl":0.05819160211831331,"entropy":0.5373689532279968},{"step":377,"reward":-1.7209120988845825,"core":-0.4814186096191406,"cost":-0.04351156949996948,"kl":0.06354689318686724,"entropy":0.5146983042359352},{"step":378,"reward":-1.4491420984268188,"core":-0.39915692806243896,"cost":-0.007057027891278267,"kl":0.05521317198872566,"entropy":0.4953402355313301},{"step":379,"reward":-1.2896097898483276,"core":-0.3317820727825165,"cost":-0.0469142384827137,"kl":0.07856084126979113,"entropy":0.47106002271175385},{"step":380,"reward":-2.250613212585449,"core":-0.6691303849220276,"cost":-0.11790353059768677,"kl":0.0763164795935154,"entropy":0.5176080167293549},{"step":381,"reward":-0.9456809163093567,"core":-0.1910337209701538,"cost":-0.0646686777472496,"kl":0.06325387954711914,"entropy":0.5211920291185379},{"step":382,"reward":-0.4501989781856537,"core":-0.016010349616408348,"cost":-0.005202900618314743,"kl":0.04843438044190407,"entropy":0.5347133576869965},{"step":383,"reward":-0.9194930195808411,"core":-0.23487596213817596,"cost":-0.024315854534506798,"kl":0.09500377997756004,"entropy":0.59617580473423},{"step":384,"reward":-0.4051050841808319,"core":-0.06781057268381119,"cost":0.0,"kl":0.040076956152915955,"entropy":0.4163220375776291},{"step":385,"reward":-1.883318305015564,"core":-0.5650807619094849,"cost":-0.08128488808870316,"kl":0.06137709133327007,"entropy":0.5618705004453659},{"step":386,"reward":-0.6514089703559875,"core":-0.06600034981966019,"cost":-0.04092656075954437,"kl":0.05368589982390404,"entropy":0.4257769361138344},{"step":387,"reward":-1.6503812074661255,"core":-0.4739041328430176,"cost":-0.01968463324010372,"kl":0.055256642401218414,"entropy":0.48836206644773483},{"step":388,"reward":-1.8249168395996094,"core":-0.5351247787475586,"cost":-0.0754784420132637,"kl":0.06109444331377745,"entropy":0.48505760729312897},{"step":389,"reward":-0.5395930409431458,"core":-0.14697371423244476,"cost":0.0,"kl":0.07522257510572672,"entropy":0.5420990362763405},{"step":390,"reward":-1.0745428800582886,"core":-0.23841212689876556,"cost":-0.04081875830888748,"kl":0.06797538883984089,"entropy":0.48372848331928253},{"step":391,"reward":-1.48588228225708,"core":-0.4529874324798584,"cost":-0.012422796338796616,"kl":0.0510439844802022,"entropy":0.5588832646608353},{"step":392,"reward":-2.016678810119629,"core":-0.6186800599098206,"cost":-0.051730066537857056,"kl":0.04440223053097725,"entropy":0.5120708718895912},{"step":393,"reward":-0.8970704078674316,"core":-0.18350990116596222,"cost":-0.05749812349677086,"kl":0.07913656532764435,"entropy":0.5943732410669327},{"step":394,"reward":-0.6207411289215088,"core":-0.1263146549463272,"cost":-0.014626793563365936,"kl":0.06568094901740551,"entropy":0.5528693348169327},{"step":395,"reward":-1.202835202217102,"core":-0.28024184703826904,"cost":-0.05230193957686424,"kl":0.05795596353709698,"entropy":0.5322812795639038},{"step":396,"reward":-1.136177897453308,"core":-0.2423182874917984,"cost":-0.09397349506616592,"kl":0.06653558369725943,"entropy":0.45338716357946396},{"step":397,"reward":-1.9182806015014648,"core":-0.5450978875160217,"cost":-0.10907653719186783,"kl":0.059908594004809856,"entropy":0.5886021256446838},{"step":398,"reward":-1.382464051246643,"core":-0.4236118793487549,"cost":-0.0049363053403794765,"kl":0.06895884871482849,"entropy":0.5323305130004883},{"step":399,"reward":-0.6355205178260803,"core":-0.12135517597198486,"cost":-0.01758025959134102,"kl":0.06126905605196953,"entropy":0.4842964857816696},{"step":400,"reward":-1.0865988731384277,"core":-0.2669903635978699,"cost":-0.0009083342156372964,"kl":0.0684420894831419,"entropy":0.5281511768698692},{"step":401,"reward":-2.1709835529327393,"core":-0.6648597717285156,"cost":-0.06594744324684143,"kl":0.0823392691090703,"entropy":0.6055251657962799},{"step":402,"reward":-1.7406708002090454,"core":-0.40598997473716736,"cost":-0.17737166583538055,"kl":0.06848296523094177,"entropy":0.5314930453896523},{"step":403,"reward":-1.8081203699111938,"core":-0.5450085997581482,"cost":-0.03327574208378792,"kl":0.04370030574500561,"entropy":0.47475971281528473},{"step":404,"reward":-2.271801710128784,"core":-0.7086946964263916,"cost":-0.05715125799179077,"kl":0.07407889887690544,"entropy":0.5951429307460785},{"step":405,"reward":-0.4909510612487793,"core":-0.05001017823815346,"cost":0.0,"kl":0.06055806390941143,"entropy":0.5690110176801682},{"step":406,"reward":-0.558211624622345,"core":-0.1000501811504364,"cost":0.0,"kl":0.05446711555123329,"entropy":0.5178884342312813},{"step":407,"reward":-1.7504655122756958,"core":-0.453950434923172,"cost":-0.2048478126525879,"kl":0.0719705605879426,"entropy":0.5087678954005241},{"step":408,"reward":-0.9324626922607422,"core":-0.24771590530872345,"cost":-0.0530419796705246,"kl":0.09366154950112104,"entropy":0.5715434551239014},{"step":409,"reward":-2.0892608165740967,"core":-0.6415192484855652,"cost":-0.04521836340427399,"kl":0.06551078334450722,"entropy":0.5358591228723526},{"step":410,"reward":-1.0801740884780884,"core":-0.23302924633026123,"cost":-0.0395401231944561,"kl":0.0802454138174653,"entropy":0.4981102868914604},{"step":411,"reward":-1.4324911832809448,"core":-0.39392784237861633,"cost":-0.0907193049788475,"kl":0.07925975508987904,"entropy":0.45761170983314514},{"step":412,"reward":-1.6979179382324219,"core":-0.4861690104007721,"cost":-0.04536968469619751,"kl":0.06080138590186834,"entropy":0.4782107397913933},{"step":413,"reward":-1.385218620300293,"core":-0.42189154028892517,"cost":-0.003995025064796209,"kl":0.04781100619584322,"entropy":0.5036944970488548},{"step":414,"reward":-1.67850923538208,"core":-0.44890108704566956,"cost":-0.05725536867976189,"kl":0.0702689103782177,"entropy":0.5546831786632538},{"step":415,"reward":-3.2768592834472656,"core":-0.9893290400505066,"cost":-0.24553854763507843,"kl":0.0527567220851779,"entropy":0.5500118359923363},{"step":416,"reward":-1.3627837896347046,"core":-0.3507949411869049,"cost":-0.10316911339759827,"kl":0.08063607756048441,"entropy":0.595236748456955},{"step":417,"reward":-1.3282405138015747,"core":-0.34102150797843933,"cost":-0.07717708498239517,"kl":0.07284146454185247,"entropy":0.5253363698720932},{"step":418,"reward":-1.971013069152832,"core":-0.5841826796531677,"cost":-0.12759995460510254,"kl":0.05304818693548441,"entropy":0.5039297789335251},{"step":419,"reward":-2.124326467514038,"core":-0.6735880970954895,"cost":-0.035575851798057556,"kl":0.0693497471511364,"entropy":0.5271966978907585},{"step":420,"reward":-2.396294355392456,"core":-0.7267860770225525,"cost":-0.12000193446874619,"kl":0.06281623896211386,"entropy":0.5917836576700211},{"step":421,"reward":-1.8025511503219604,"core":-0.5263941884040833,"cost":-0.04448724910616875,"kl":0.057091549038887024,"entropy":0.4599297046661377},{"step":422,"reward":-1.2100005149841309,"core":-0.3358782231807709,"cost":-0.017618123441934586,"kl":0.06383033748716116,"entropy":0.474110372364521},{"step":423,"reward":-0.7575889229774475,"core":-0.1248617172241211,"cost":0.0,"kl":0.042999859899282455,"entropy":0.4579976946115494},{"step":424,"reward":-1.2819597721099854,"core":-0.3554946482181549,"cost":-0.07986148446798325,"kl":0.039167741779237986,"entropy":0.44179338961839676},{"step":425,"reward":-1.7031515836715698,"core":-0.40397414565086365,"cost":-0.17572635412216187,"kl":0.0661888662725687,"entropy":0.4670601412653923},{"step":426,"reward":-1.3490720987319946,"core":-0.3886904716491699,"cost":-0.04121141508221626,"kl":0.06128471251577139,"entropy":0.6098155230283737},{"step":427,"reward":-0.9869839549064636,"core":-0.26766061782836914,"cost":-0.027424484491348267,"kl":0.06035755015909672,"entropy":0.5940822958946228},{"step":428,"reward":-1.232955813407898,"core":-0.35821399092674255,"cost":-0.006946986075490713,"kl":0.08057178370654583,"entropy":0.38925427943468094},{"step":429,"reward":-0.49267879128456116,"core":-0.006483649369329214,"cost":0.0,"kl":0.05511805694550276,"entropy":0.4638728126883507},{"step":430,"reward":-1.2081853151321411,"core":-0.342416912317276,"cost":-0.021466506645083427,"kl":0.04620190104469657,"entropy":0.5746173709630966},{"step":431,"reward":-1.3910984992980957,"core":-0.3143046796321869,"cost":-0.07995138317346573,"kl":0.05837318394333124,"entropy":0.5249655395746231},{"step":432,"reward":-0.749045193195343,"core":-0.15880443155765533,"cost":-0.024567360058426857,"kl":0.07651486247777939,"entropy":0.5175809860229492},{"step":433,"reward":-1.0776287317276,"core":-0.28929319977760315,"cost":-0.03205498680472374,"kl":0.05648672953248024,"entropy":0.42971693724393845},{"step":434,"reward":-2.4603347778320312,"core":-0.7574028968811035,"cost":-0.11672791093587875,"kl":0.05408944468945265,"entropy":0.5234841257333755},{"step":435,"reward":-0.8469509482383728,"core":-0.16429896652698517,"cost":-0.040559083223342896,"kl":0.056290351785719395,"entropy":0.49030839651823044},{"step":436,"reward":-1.6237741708755493,"core":-0.4205078184604645,"cost":-0.06571275740861893,"kl":0.05331295542418957,"entropy":0.4403933063149452},{"step":437,"reward":-1.499240517616272,"core":-0.43046268820762634,"cost":-0.03855738043785095,"kl":0.06654167268425226,"entropy":0.6056731343269348},{"step":438,"reward":-2.079116106033325,"core":-0.6057283878326416,"cost":-0.16164888441562653,"kl":0.05188044440001249,"entropy":0.5382165163755417},{"step":439,"reward":-1.539581298828125,"core":-0.3232559263706207,"cost":-0.2248125523328781,"kl":0.07432548142969608,"entropy":0.5256334543228149},{"step":440,"reward":-0.7955451011657715,"core":-0.1513003557920456,"cost":-0.05158790946006775,"kl":0.07042399141937494,"entropy":0.48491375893354416},{"step":441,"reward":-1.4757682085037231,"core":-0.38903918862342834,"cost":-0.06603389233350754,"kl":0.07352710980921984,"entropy":0.5536650195717812},{"step":442,"reward":-1.764346957206726,"core":-0.5218544602394104,"cost":-0.06315373629331589,"kl":0.06478709261864424,"entropy":0.6014818698167801},{"step":443,"reward":-0.8855188488960266,"core":-0.24464482069015503,"cost":-0.003125546732917428,"kl":0.06072288192808628,"entropy":0.5270386338233948},{"step":444,"reward":-2.3709990978240967,"core":-0.6289194226264954,"cost":-0.2343279868364334,"kl":0.05867396015673876,"entropy":0.4982612356543541},{"step":445,"reward":-1.4911197423934937,"core":-0.4488125741481781,"cost":-0.02618550695478916,"kl":0.06262507382780313,"entropy":0.4795958176255226},{"step":446,"reward":-1.9605573415756226,"core":-0.5677118897438049,"cost":-0.11651834100484848,"kl":0.07514614053070545,"entropy":0.5800199955701828},{"step":447,"reward":-2.3805885314941406,"core":-0.7242726683616638,"cost":-0.129717156291008,"kl":0.036358434706926346,"entropy":0.4468895047903061},{"step":448,"reward":-0.8264656066894531,"core":-0.17800301313400269,"cost":0.0,"kl":0.0580058079212904,"entropy":0.5549484491348267},{"step":449,"reward":-1.9410165548324585,"core":-0.54973965883255,"cost":-0.08527139574289322,"kl":0.09103027917444706,"entropy":0.5207302421331406},{"step":450,"reward":-1.3205533027648926,"core":-0.3413375914096832,"cost":-0.09919425845146179,"kl":0.07027514185756445,"entropy":0.48027094453573227},{"step":451,"reward":-0.20393623411655426,"core":0.07229282706975937,"cost":-0.028070004656910896,"kl":0.06783366668969393,"entropy":0.4861317053437233},{"step":452,"reward":-1.125321388244629,"core":-0.29991886019706726,"cost":0.0,"kl":0.07698724046349525,"entropy":0.5292880088090897},{"step":453,"reward":-0.9332192540168762,"core":-0.1956343650817871,"cost":-0.021925752982497215,"kl":0.07586836256086826,"entropy":0.502516895532608},{"step":454,"reward":-1.1821991205215454,"core":-0.3003231883049011,"cost":-0.003010496264323592,"kl":0.04864273779094219,"entropy":0.677346408367157},{"step":455,"reward":-1.415209174156189,"core":-0.37612858414649963,"cost":-0.0388476587831974,"kl":0.040637457743287086,"entropy":0.5107972919940948},{"step":456,"reward":-3.401134490966797,"core":-0.9773754477500916,"cost":-0.32952937483787537,"kl":0.04384776297956705,"entropy":0.5122402235865593},{"step":457,"reward":-1.5191822052001953,"core":-0.43370211124420166,"cost":-0.0949316918849945,"kl":0.077944440767169,"entropy":0.5302828028798103},{"step":458,"reward":0.0867714062333107,"core":0.18721985816955566,"cost":0.0,"kl":0.05028013791888952,"entropy":0.5176952555775642},{"step":459,"reward":-1.6852785348892212,"core":-0.45078548789024353,"cost":-0.10546664148569107,"kl":0.06354681309312582,"entropy":0.5383344292640686},{"step":460,"reward":-1.5434446334838867,"core":-0.4297403395175934,"cost":-0.06519251316785812,"kl":0.0814659595489502,"entropy":0.5128542333841324},{"step":461,"reward":-1.6524864435195923,"core":-0.49357280135154724,"cost":-0.017228232696652412,"kl":0.041611301712691784,"entropy":0.525056965649128},{"step":462,"reward":-1.0073074102401733,"core":-0.27202144265174866,"cost":-0.05465563014149666,"kl":0.06494902446866035,"entropy":0.5552985519170761},{"step":463,"reward":-1.4087198972702026,"core":-0.3881549537181854,"cost":-0.014244445599615574,"kl":0.06519256811589003,"entropy":0.46375884115695953},{"step":464,"reward":-1.044839859008789,"core":-0.2616870105266571,"cost":0.0,"kl":0.053634168580174446,"entropy":0.5741505920886993},{"step":465,"reward":-0.4991719424724579,"core":-0.00012965004134457558,"cost":-0.08351249247789383,"kl":0.0923630204051733,"entropy":0.5029343590140343},{"step":466,"reward":-2.218304395675659,"core":-0.6356660723686218,"cost":-0.18261204659938812,"kl":0.07188329193741083,"entropy":0.5417152941226959},{"step":467,"reward":-1.857302188873291,"core":-0.5192709565162659,"cost":-0.10983534902334213,"kl":0.061676121316850185,"entropy":0.5238338634371758},{"step":468,"reward":-1.28793203830719,"core":-0.33490535616874695,"cost":-0.054397374391555786,"kl":0.05664192512631416,"entropy":0.5265325158834457},{"step":469,"reward":-1.0386343002319336,"core":-0.2665582299232483,"cost":-0.013256008736789227,"kl":0.09155881125479937,"entropy":0.5214168801903725},{"step":470,"reward":-1.3293007612228394,"core":-0.3449898660182953,"cost":-0.006956601981073618,"kl":0.05215548723936081,"entropy":0.5356603115797043},{"step":471,"reward":-1.9002984762191772,"core":-0.5322965979576111,"cost":-0.0935969352722168,"kl":0.050965815782547,"entropy":0.44997992366552353},{"step":472,"reward":-0.6161285638809204,"core":-0.05475878715515137,"cost":-0.017956476658582687,"kl":0.06560197286307812,"entropy":0.5443741232156754},{"step":473,"reward":-1.1261439323425293,"core":-0.2961995005607605,"cost":-0.0527191162109375,"kl":0.049521783366799355,"entropy":0.528128057718277},{"step":474,"reward":-0.8832563757896423,"core":-0.227876678109169,"cost":-0.010689039714634418,"kl":0.04932093154639006,"entropy":0.5185786485671997},{"step":475,"reward":-0.5376773476600647,"core":-0.05536078289151192,"cost":0.0,"kl":0.05836504232138395,"entropy":0.5069728195667267},{"step":476,"reward":-2.0498595237731934,"core":-0.5774286389350891,"cost":-0.18817831575870514,"kl":0.10350770689547062,"entropy":0.549611508846283},{"step":477,"reward":-1.4638115167617798,"core":-0.3075680136680603,"cost":-0.1193758174777031,"kl":0.0746813490986824,"entropy":0.5137175992131233},{"step":478,"reward":-2.371089220046997,"core":-0.7161008715629578,"cost":-0.09471649676561356,"kl":0.06809898186475039,"entropy":0.6053566783666611},{"step":479,"reward":-0.8893519043922424,"core":-0.21228869259357452,"cost":-0.02564372681081295,"kl":0.0545793529599905,"entropy":0.45114873349666595},{"step":480,"reward":-1.8647699356079102,"core":-0.48744288086891174,"cost":-0.1385287493467331,"kl":0.0666739596053958,"entropy":0.4891282096505165},{"step":481,"reward":-1.7977485656738281,"core":-0.41356775164604187,"cost":-0.142642542719841,"kl":0.12124844826757908,"entropy":0.5170938372612},{"step":482,"reward":-1.9342352151870728,"core":-0.5822322368621826,"cost":-0.06689482927322388,"kl":0.05441922880709171,"entropy":0.5553408712148666},{"step":483,"reward":-1.351258635520935,"core":-0.3760286867618561,"cost":-0.015206441283226013,"kl":0.0384504459798336,"entropy":0.4755394458770752},{"step":484,"reward":-0.8368625640869141,"core":-0.19783852994441986,"cost":-0.052454084157943726,"kl":0.09306548722088337,"entropy":0.5441380441188812},{"step":485,"reward":-1.5120105743408203,"core":-0.42798885703086853,"cost":-0.005451740697026253,"kl":0.05806487798690796,"entropy":0.6164328157901764},{"step":486,"reward":-0.7057309150695801,"core":-0.16384567320346832,"cost":-0.011144119314849377,"kl":0.046332656405866146,"entropy":0.514846958220005},{"step":487,"reward":-1.22752845287323,"core":-0.3003047704696655,"cost":-0.04830469563603401,"kl":0.07295506913214922,"entropy":0.49874458461999893},{"step":488,"reward":-0.7514756321907043,"core":-0.08606336265802383,"cost":-0.027895232662558556,"kl":0.07371632009744644,"entropy":0.5494369268417358},{"step":489,"reward":-2.347898244857788,"core":-0.7513942718505859,"cost":-0.029253477230668068,"kl":0.0595831423997879,"entropy":0.5773815512657166},{"step":490,"reward":-1.0506093502044678,"core":-0.2787189781665802,"cost":-0.05323567986488342,"kl":0.10141962580382824,"entropy":0.5892172902822495},{"step":491,"reward":-1.7145992517471313,"core":-0.45075857639312744,"cost":-0.09511556476354599,"kl":0.07338672131299973,"entropy":0.59454345703125},{"step":492,"reward":-2.105297327041626,"core":-0.6107717156410217,"cost":-0.132635235786438,"kl":0.07092523109167814,"entropy":0.5453565716743469},{"step":493,"reward":-1.5920346975326538,"core":-0.48041412234306335,"cost":-0.050026822835206985,"kl":0.06821392755955458,"entropy":0.5033550187945366},{"step":494,"reward":-1.0707470178604126,"core":-0.22908461093902588,"cost":-0.11871399730443954,"kl":0.06323884148150682,"entropy":0.5540373921394348},{"step":495,"reward":-1.2828494310379028,"core":-0.3937607705593109,"cost":-0.004769830033183098,"kl":0.059992242604494095,"entropy":0.5667305737733841},{"step":496,"reward":-1.4212560653686523,"core":-0.4133266508579254,"cost":-0.0676625669002533,"kl":0.05787104368209839,"entropy":0.46979067474603653},{"step":497,"reward":-1.579423427581787,"core":-0.4462836682796478,"cost":-0.03890213742852211,"kl":0.07177856098860502,"entropy":0.5380894392728806},{"step":498,"reward":-1.1787585020065308,"core":-0.3499757945537567,"cost":-0.03838685527443886,"kl":0.08842713758349419,"entropy":0.5065033733844757},{"step":499,"reward":-2.05825138092041,"core":-0.5887872576713562,"cost":-0.12843145430088043,"kl":0.07814545556902885,"entropy":0.5857897996902466},{"step":500,"reward":-2.1773617267608643,"core":-0.6150217056274414,"cost":-0.11982131004333496,"kl":0.0749356597661972,"entropy":0.6192649155855179},{"step":501,"reward":-0.7977581024169922,"core":-0.16752572357654572,"cost":-0.0010327176423743367,"kl":0.11377344559878111,"entropy":0.5962159037590027},{"step":502,"reward":-1.3383522033691406,"core":-0.3889615833759308,"cost":0.0,"kl":0.04435937898233533,"entropy":0.5144816637039185},{"step":503,"reward":-1.314841389656067,"core":-0.36727848649024963,"cost":-0.031141074374318123,"kl":0.07717419415712357,"entropy":0.5522457882761955},{"step":504,"reward":-1.476035714149475,"core":-0.4064745008945465,"cost":-0.06836790591478348,"kl":0.07631275057792664,"entropy":0.5510525405406952},{"step":505,"reward":-2.134880542755127,"core":-0.6694409251213074,"cost":-0.027009636163711548,"kl":0.053808944299817085,"entropy":0.5162441730499268},{"step":506,"reward":-2.3257246017456055,"core":-0.7079017758369446,"cost":-0.07542342692613602,"kl":0.07007624581456184,"entropy":0.6003907173871994},{"step":507,"reward":-1.9091029167175293,"core":-0.5523384213447571,"cost":-0.08364331722259521,"kl":0.06296275835484266,"entropy":0.5150727331638336},{"step":508,"reward":-1.6737899780273438,"core":-0.4837287366390228,"cost":-0.051740553230047226,"kl":0.0623608622699976,"entropy":0.5704474896192551},{"step":509,"reward":-0.8198985457420349,"core":-0.17491793632507324,"cost":-0.01406567171216011,"kl":0.07593054883182049,"entropy":0.5713040381669998},{"step":510,"reward":-0.5291103720664978,"core":-0.09417418390512466,"cost":-0.04018335044384003,"kl":0.07847699150443077,"entropy":0.49224551767110825},{"step":511,"reward":-1.553824782371521,"core":-0.3908118009567261,"cost":-0.07589830458164215,"kl":0.060168687254190445,"entropy":0.5883399248123169},{"step":512,"reward":-1.182601809501648,"core":-0.2953806221485138,"cost":-0.07426729798316956,"kl":0.05455926712602377,"entropy":0.5635355040431023},{"step":513,"reward":0.3611104190349579,"core":0.2877660095691681,"cost":-0.0002462432312313467,"kl":0.07505921833217144,"entropy":0.5300666093826294},{"step":514,"reward":-1.7015529870986938,"core":-0.40489649772644043,"cost":-0.15752306580543518,"kl":0.09260242618620396,"entropy":0.5764065384864807},{"step":515,"reward":-0.12089890241622925,"core":0.07748384028673172,"cost":-0.04510532692074776,"kl":0.08291635848581791,"entropy":0.5414819568395615},{"step":516,"reward":-1.9803166389465332,"core":-0.5203443169593811,"cost":-0.20911483466625214,"kl":0.13279256131500006,"entropy":0.5716565102338791},{"step":517,"reward":-0.38521361351013184,"core":-0.06844504922628403,"cost":-0.001221813727170229,"kl":0.054555486887693405,"entropy":0.4954785257577896},{"step":518,"reward":-2.2871015071868896,"core":-0.703636646270752,"cost":-0.078055240213871,"kl":0.07629959844052792,"entropy":0.5817097425460815},{"step":519,"reward":-1.1238057613372803,"core":-0.31147047877311707,"cost":-0.008341271430253983,"kl":0.04996159952133894,"entropy":0.5693666934967041},{"step":520,"reward":-1.067193865776062,"core":-0.29058536887168884,"cost":-0.02090134285390377,"kl":0.045645792968571186,"entropy":0.5172325223684311},{"step":521,"reward":-1.5612729787826538,"core":-0.4424927234649658,"cost":-0.07714535295963287,"kl":0.050588115118443966,"entropy":0.4891698658466339},{"step":522,"reward":-2.374103307723999,"core":-0.6683771014213562,"cost":-0.23477201163768768,"kl":0.06850665342062712,"entropy":0.4818238243460655},{"step":523,"reward":-0.5847191214561462,"core":-0.10558503866195679,"cost":0.0,"kl":0.06343848910182714,"entropy":0.5662195682525635},{"step":524,"reward":-1.3844975233078003,"core":-0.3944762945175171,"cost":-0.05660941079258919,"kl":0.07725848443806171,"entropy":0.5813441723585129},{"step":525,"reward":-1.0051302909851074,"core":-0.2290560007095337,"cost":-0.012333382852375507,"kl":0.09669926390051842,"entropy":0.535492017865181},{"step":526,"reward":-1.1492902040481567,"core":-0.24250781536102295,"cost":-0.036423202604055405,"kl":0.08450715616345406,"entropy":0.5175989642739296},{"step":527,"reward":-1.5943228006362915,"core":-0.4160579741001129,"cost":-0.06855632364749908,"kl":0.0616217190399766,"entropy":0.4998382478952408},{"step":528,"reward":-1.4187852144241333,"core":-0.3959914743900299,"cost":-0.07969001680612564,"kl":0.06136290542781353,"entropy":0.5552867650985718},{"step":529,"reward":-1.9814056158065796,"core":-0.6222811937332153,"cost":-0.02752145566046238,"kl":0.04527301201596856,"entropy":0.39426613599061966},{"step":530,"reward":-1.8100990056991577,"core":-0.5394308567047119,"cost":-0.06050599738955498,"kl":0.0590084558352828,"entropy":0.5099540874361992},{"step":531,"reward":-0.7175487875938416,"core":-0.17173564434051514,"cost":-0.04976898804306984,"kl":0.08565891347825527,"entropy":0.5600198656320572},{"step":532,"reward":-1.4681228399276733,"core":-0.3834908902645111,"cost":-0.09334802627563477,"kl":0.0571509413421154,"entropy":0.4816358759999275},{"step":533,"reward":-0.8172211050987244,"core":-0.21227045357227325,"cost":-0.009090698324143887,"kl":0.07738757226616144,"entropy":0.6031533181667328},{"step":534,"reward":-1.6368178129196167,"core":-0.4725923538208008,"cost":-0.018391193822026253,"kl":0.06037628743797541,"entropy":0.5610641315579414},{"step":535,"reward":-2.182732343673706,"core":-0.5492391586303711,"cost":-0.261075496673584,"kl":0.0719109233468771,"entropy":0.511830747127533},{"step":536,"reward":-1.9378126859664917,"core":-0.5810297131538391,"cost":-0.050228577107191086,"kl":0.05497821606695652,"entropy":0.5261687785387039},{"step":537,"reward":-0.6632959246635437,"core":-0.09408833831548691,"cost":-0.03263477981090546,"kl":0.0702925156801939,"entropy":0.4849366918206215},{"step":538,"reward":0.2845194637775421,"core":0.3307335376739502,"cost":0.0,"kl":0.04270081780850887,"entropy":0.5289121866226196},{"step":539,"reward":-2.3524329662323,"core":-0.7381821274757385,"cost":-0.042526718229055405,"kl":0.05971100274473429,"entropy":0.595193475484848},{"step":540,"reward":-1.0774502754211426,"core":-0.22270379960536957,"cost":-0.029935158789157867,"kl":0.0671392111107707,"entropy":0.5574289262294769},{"step":541,"reward":-0.9371077418327332,"core":-0.196405827999115,"cost":-0.0845596194267273,"kl":0.06369152199476957,"entropy":0.5730115920305252},{"step":542,"reward":-1.5315109491348267,"core":-0.3825816214084625,"cost":-0.09246829897165298,"kl":0.09925605542957783,"entropy":0.587252750992775},{"step":543,"reward":-0.6960925459861755,"core":-0.09653642028570175,"cost":-0.05389336124062538,"kl":0.08625430427491665,"entropy":0.5307492911815643},{"step":544,"reward":-1.8124961853027344,"core":-0.5289396643638611,"cost":-0.0029676342383027077,"kl":0.053169798105955124,"entropy":0.5621999204158783},{"step":545,"reward":-1.2324532270431519,"core":-0.326753705739975,"cost":-0.0911782756447792,"kl":0.07306610606610775,"entropy":0.6090302020311356},{"step":546,"reward":-1.2112635374069214,"core":-0.22087876498699188,"cost":-0.0866156816482544,"kl":0.06706227641552687,"entropy":0.4699060171842575},{"step":547,"reward":-0.36307427287101746,"core":-0.01336052268743515,"cost":-0.003960145637392998,"kl":0.05752951744943857,"entropy":0.5943526327610016},{"step":548,"reward":-1.1922732591629028,"core":-0.313241571187973,"cost":-0.02188473753631115,"kl":0.03840153384953737,"entropy":0.5392139554023743},{"step":549,"reward":-1.492170810699463,"core":-0.4415677487850189,"cost":-0.0035112423356622458,"kl":0.05372729618102312,"entropy":0.5028066709637642},{"step":550,"reward":-0.8267137408256531,"core":-0.1497468501329422,"cost":-0.03714678809046745,"kl":0.06611064169555902,"entropy":0.5038725435733795},{"step":551,"reward":-0.8485615253448486,"core":-0.18197117745876312,"cost":-0.03664710000157356,"kl":0.09243921749293804,"entropy":0.5205194354057312},{"step":552,"reward":-1.7830371856689453,"core":-0.5055382251739502,"cost":-0.1489100605249405,"kl":0.07768115028738976,"entropy":0.5892691165208817},{"step":553,"reward":-1.0739411115646362,"core":-0.29945480823516846,"cost":-0.01522285770624876,"kl":0.07310137152671814,"entropy":0.5842292308807373},{"step":554,"reward":-1.366155982017517,"core":-0.32168930768966675,"cost":-0.07977668195962906,"kl":0.07037728000432253,"entropy":0.5584818571805954},{"step":555,"reward":-1.9754328727722168,"core":-0.5917617082595825,"cost":-0.08973250538110733,"kl":0.08136658929288387,"entropy":0.48505108058452606},{"step":556,"reward":-0.694747269153595,"core":-0.14282654225826263,"cost":-0.00855148397386074,"kl":0.06550083681941032,"entropy":0.6421801745891571},{"step":557,"reward":-2.414503335952759,"core":-0.7223685383796692,"cost":-0.1410214751958847,"kl":0.08727098256349564,"entropy":0.5594897866249084},{"step":558,"reward":-0.34637370705604553,"core":0.002917726757004857,"cost":-0.020531142130494118,"kl":0.1019311361014843,"entropy":0.5617877244949341},{"step":559,"reward":-2.655499219894409,"core":-0.8067817687988281,"cost":-0.14654429256916046,"kl":0.07464047241955996,"entropy":0.4613126888871193},{"step":560,"reward":-2.14577054977417,"core":-0.6457483172416687,"cost":-0.061558857560157776,"kl":0.03914445452392101,"entropy":0.47969721257686615},{"step":561,"reward":-0.7182779312133789,"core":-0.14967449009418488,"cost":-0.032308850437402725,"kl":0.04942131042480469,"entropy":0.4578109607100487},{"step":562,"reward":-1.0173197984695435,"core":-0.20666038990020752,"cost":-0.06062346324324608,"kl":0.07010340690612793,"entropy":0.5630400627851486},{"step":563,"reward":-1.8501323461532593,"core":-0.5824833512306213,"cost":-0.03337119147181511,"kl":0.08635461609810591,"entropy":0.6405730992555618},{"step":564,"reward":-1.4065332412719727,"core":-0.41874346137046814,"cost":-0.04293549433350563,"kl":0.06735910754650831,"entropy":0.4863736853003502},{"step":565,"reward":-1.4336484670639038,"core":-0.35662174224853516,"cost":-0.13872818648815155,"kl":0.049830131232738495,"entropy":0.5344499945640564},{"step":566,"reward":-0.17069004476070404,"core":0.01576279290020466,"cost":0.0,"kl":0.055286227725446224,"entropy":0.5756274908781052},{"step":567,"reward":-0.8283246159553528,"core":-0.10971945524215698,"cost":-0.017482351511716843,"kl":0.07018907554447651,"entropy":0.4968886077404022},{"step":568,"reward":-1.6657558679580688,"core":-0.5028679966926575,"cost":-0.07069788128137589,"kl":0.05721851717680693,"entropy":0.5842548757791519},{"step":569,"reward":-2.0508222579956055,"core":-0.6333041787147522,"cost":-0.06598740071058273,"kl":0.0561085008084774,"entropy":0.5159300714731216},{"step":570,"reward":-2.21724009513855,"core":-0.6327844262123108,"cost":-0.17342430353164673,"kl":0.04979674983769655,"entropy":0.508722297847271},{"step":571,"reward":-0.9491727948188782,"core":-0.21687035262584686,"cost":-0.003695273771882057,"kl":0.055503775365650654,"entropy":0.5224109068512917},{"step":572,"reward":-0.6920130848884583,"core":-0.1423572301864624,"cost":-0.0648396834731102,"kl":0.046153608709573746,"entropy":0.495405949652195},{"step":573,"reward":-1.6280517578125,"core":-0.4742911756038666,"cost":-0.07981985062360764,"kl":0.08405090775340796,"entropy":0.5047991648316383},{"step":574,"reward":-1.1408919095993042,"core":-0.3228408098220825,"cost":-0.010855890810489655,"kl":0.06941335089504719,"entropy":0.45145127177238464},{"step":575,"reward":-2.320981740951538,"core":-0.7012456059455872,"cost":-0.1237289309501648,"kl":0.05698008835315704,"entropy":0.39261574298143387},{"step":576,"reward":-1.597038745880127,"core":-0.48103880882263184,"cost":-0.04235773906111717,"kl":0.06588610075414181,"entropy":0.5302861034870148},{"step":577,"reward":-0.7961180210113525,"core":-0.18065635859966278,"cost":-0.005212857853621244,"kl":0.06943588703870773,"entropy":0.565038725733757},{"step":578,"reward":-0.19886714220046997,"core":0.07540532201528549,"cost":0.0,"kl":0.05262343306094408,"entropy":0.493418350815773},{"step":579,"reward":-1.7525955438613892,"core":-0.4854927062988281,"cost":-0.023110242560505867,"kl":0.07001511473208666,"entropy":0.5118418708443642},{"step":580,"reward":-0.744929313659668,"core":-0.08777353912591934,"cost":-0.030509330332279205,"kl":0.07262372598052025,"entropy":0.4928305894136429},{"step":581,"reward":-2.619493246078491,"core":-0.7917231917381287,"cost":-0.14739646017551422,"kl":0.0407811664044857,"entropy":0.4524817690253258},{"step":582,"reward":-1.9485896825790405,"core":-0.5772273540496826,"cost":-0.05485020950436592,"kl":0.057447257451713085,"entropy":0.5113550275564194},{"step":583,"reward":-1.7818936109542847,"core":-0.5322331786155701,"cost":-0.08773868530988693,"kl":0.08180179260671139,"entropy":0.5786852985620499},{"step":584,"reward":-2.3359572887420654,"core":-0.6971709132194519,"cost":-0.11154083162546158,"kl":0.05887515842914581,"entropy":0.5060949549078941},{"step":585,"reward":-1.104374647140503,"core":-0.31383103132247925,"cost":0.0,"kl":0.06308919470757246,"entropy":0.49104297161102295},{"step":586,"reward":-1.460024356842041,"core":-0.3662637174129486,"cost":-0.026289986446499825,"kl":0.060738968662917614,"entropy":0.5304004773497581},{"step":587,"reward":-1.0566890239715576,"core":-0.2256920337677002,"cost":-0.043037962168455124,"kl":0.07434867788106203,"entropy":0.4932412952184677},{"step":588,"reward":-1.26986563205719,"core":-0.23592953383922577,"cost":-0.10005825012922287,"kl":0.06896594725549221,"entropy":0.5990379154682159},{"step":589,"reward":-0.8711426854133606,"core":-0.20984923839569092,"cost":-0.039425697177648544,"kl":0.07114134728908539,"entropy":0.4844737946987152},{"step":590,"reward":-1.9672836065292358,"core":-0.6101886034011841,"cost":-0.020865874364972115,"kl":0.04510993044823408,"entropy":0.5612106025218964},{"step":591,"reward":-0.6560478210449219,"core":-0.1199108138680458,"cost":-0.0011888362932950258,"kl":0.05519360862672329,"entropy":0.44990255683660507},{"step":592,"reward":-2.2338736057281494,"core":-0.6346687078475952,"cost":-0.09571382403373718,"kl":0.07405733224004507,"entropy":0.514166846871376},{"step":593,"reward":-1.0842337608337402,"core":-0.26085150241851807,"cost":-0.01796862669289112,"kl":0.057136871851980686,"entropy":0.5780708640813828},{"step":594,"reward":-3.3378188610076904,"core":-1.0032812356948853,"cost":-0.2479465752840042,"kl":0.036940130405128,"entropy":0.43416059762239456},{"step":595,"reward":-2.214195966720581,"core":-0.6445523500442505,"cost":-0.18235468864440918,"kl":0.05426346883177757,"entropy":0.49051041156053543},{"step":596,"reward":-1.6774355173110962,"core":-0.4285658895969391,"cost":-0.10196227580308914,"kl":0.07391250506043434,"entropy":0.6442728042602539},{"step":597,"reward":-1.341251015663147,"core":-0.37194523215293884,"cost":-0.07098417729139328,"kl":0.05846324283629656,"entropy":0.49214065819978714},{"step":598,"reward":-0.8544302582740784,"core":-0.22161294519901276,"cost":-0.03431786596775055,"kl":0.059255472384393215,"entropy":0.5253633484244347},{"step":599,"reward":-1.8454703092575073,"core":-0.5241983532905579,"cost":-0.12266495823860168,"kl":0.055890501476824284,"entropy":0.4966697171330452},{"step":600,"reward":-1.1517122983932495,"core":-0.3049386441707611,"cost":-0.05195854604244232,"kl":0.0918249674141407,"entropy":0.5841208845376968}],"nonzero_advantage":[{"step":1,"nonzero":0.4166666666666667},{"step":3,"nonzero":0.1875},{"step":5,"nonzero":0.5416666666666666},{"step":7,"nonzero":0.3958333333333333},{"step":9,"nonzero":0.10416666666666667},{"step":11,"nonzero":0.5416666666666666},{"step":13,"nonzero":0.10416666666666667},{"step":15,"nonzero":0.4166666666666667},{"step":17,"nonzero":0.3125},{"step":19,"nonzero":0.20833333333333334},{"step":21,"nonzero":0.3541666666666667},{"step":23,"nonzero":0.3541666666666667},{"step":25,"nonzero":0.4166666666666667},{"step":27,"nonzero":0.5416666666666666},{"step":29,"nonzero":0.4166666666666667},{"step":31,"nonzero":0.4375},{"step":33,"nonzero":0.16666666666666666},{"step":35,"nonzero":0.1875},{"step":37,"nonzero":0.4166666666666667},{"step":39,"nonzero":0.4791666666666667},{"step":41,"nonzero":0.3541666666666667},{"step":43,"nonzero":0.4166666666666667},{"step":45,"nonzero":0.4583333333333333},{"step":47,"nonzero":0.4375},{"step":49,"nonzero":0.16666666666666666},{"step":151,"nonzero":0.3125},{"step":153,"nonzero":0.4583333333333333},{"step":155,"nonzero":0.4791666666666667},{"step":157,"nonzero":0.6041666666666666},{"step":159,"nonzero":0.5},{"step":161,"nonzero":0.4791666666666667},{"step":163,"nonzero":0.2916666666666667},{"step":165,"nonzero":0.6458333333333334},{"step":167,"nonzero":0.7916666666666666},{"step":169,"nonzero":0.5208333333333334},{"step":171,"nonzero":0.3958333333333333},{"step":173,"nonzero":0.7083333333333334},{"step":175,"nonzero":0.3541666666666667},{"step":177,"nonzero":0.4375},{"step":179,"nonzero":0.8333333333333334},{"step":181,"nonzero":0.5},{"step":183,"nonzero":0.6458333333333334},{"step":185,"nonzero":0.5416666666666666},{"step":187,"nonzero":0.4166666666666667},{"step":189,"nonzero":0.5208333333333334},{"step":191,"nonzero":0.375},{"step":193,"nonzero":0.8125},{"step":195,"nonzero":0.5833333333333334},{"step":197,"nonzero":0.4791666666666667},{"step":199,"nonzero":0.4583333333333333},{"step":201,"nonzero":0.3125},{"step":203,"nonzero":0.3333333333333333},{"step":205,"nonzero":0.5625},{"step":207,"nonzero":0.6041666666666666},{"step":209,"nonzero":0.4583333333333333},{"step":211,"nonzero":0.4166666666666667},{"step":213,"nonzero":0.3541666666666667},{"step":215,"nonzero":0.3541666666666667},{"step":217,"nonzero":0.5625},{"step":219,"nonzero":0.75},{"step":221,"nonzero":0.4375},{"step":223,"nonzero":0.5},{"step":225,"nonzero":0.375},{"step":227,"nonzero":0.375},{"step":229,"nonzero":0.5625},{"step":231,"nonzero":0.4375},{"step":233,"nonzero":0.6666666666666666},{"step":235,"nonzero":0.3125},{"step":237,"nonzero":0.5},{"step":239,"nonzero":0.6458333333333334},{"step":241,"nonzero":0.6666666666666666},{"step":243,"nonzero":0.25},{"step":245,"nonzero":0.5833333333333334},{"step":247,"nonzero":0.3958333333333333},{"step":249,"nonzero":0.7916666666666666},{"step":251,"nonzero":0.8958333333333334},{"step":253,"nonzero":0.3333333333333333},{"step":255,"nonzero":0.75},{"step":257,"nonzero":0.6875},{"step":259,"nonzero":0.5833333333333334},{"step":261,"nonzero":0.3333333333333333},{"step":263,"nonzero":0.5416666666666666},{"step":265,"nonzero":0.1875},{"step":267,"nonzero":0.3958333333333333},{"step":269,"nonzero":0.75},{"step":271,"nonzero":0.4791666666666667},{"step":273,"nonzero":0.2916666666666667},{"step":275,"nonzero":0.3125},{"step":277,"nonzero":0.5208333333333334},{"step":279,"nonzero":0.5833333333333334},{"step":281,"nonzero":0.8958333333333334},{"step":283,"nonzero":0.4583333333333333},{"step":285,"nonzero":0.4166666666666667},{"step":287,"nonzero":0.6458333333333334},{"step":289,"nonzero":0.5833333333333334},{"step":291,"nonzero":0.4166666666666667},{"step":293,"nonzero":0.625},{"step":295,"nonzero":0.5416666666666666},{"step":297,"nonzero":0.4583333333333333},{"step":299,"nonzero":0.6875},{"step":301,"nonzero":0.5208333333333334},{"step":303,"nonzero":0.5625},{"step":305,"nonzero":0.8958333333333334},{"step":307,"nonzero":0.4583333333333333},{"step":309,"nonzero":0.9583333333333334},{"step":311,"nonzero":0.1875},{"step":313,"nonzero":0.4583333333333333},{"step":315,"nonzero":0.625},{"step":317,"nonzero":0.375},{"step":319,"nonzero":0.3958333333333333},{"step":321,"nonzero":0.6458333333333334},{"step":323,"nonzero":0.7083333333333334},{"step":325,"nonzero":0.4375},{"step":327,"nonzero":0.75},{"step":329,"nonzero":0.7708333333333334},{"step":331,"nonzero":0.6458333333333334},{"step":333,"nonzero":0.4375},{"step":335,"nonzero":0.7708333333333334},{"step":337,"nonzero":0.625},{"step":339,"nonzero":0.4791666666666667},{"step":341,"nonzero":0.5833333333333334},{"step":343,"nonzero":0.6458333333333334},{"step":345,"nonzero":0.7083333333333334},{"step":347,"nonzero":0.5208333333333334},{"step":349,"nonzero":0.5416666666666666},{"step":351,"nonzero":0.5208333333333334},{"step":353,"nonzero":0.7291666666666666},{"step":355,"nonzero":0.6041666666666666},{"step":357,"nonzero":0.5833333333333334},{"step":359,"nonzero":0.5833333333333334},{"step":361,"nonzero":0.4375},{"step":363,"nonzero":0.4375},{"step":365,"nonzero":0.5208333333333334},{"step":367,"nonzero":0.8125},{"step":369,"nonzero":0.3541666666666667},{"step":371,"nonzero":0.3958333333333333},{"step":373,"nonzero":0.7708333333333334},{"step":375,"nonzero":0.5833333333333334},{"step":377,"nonzero":0.4583333333333333},{"step":379,"nonzero":0.5416666666666666},{"step":381,"nonzero":0.4375},{"step":383,"nonzero":0.7708333333333334},{"step":385,"nonzero":0.4375},{"step":387,"nonzero":0.6458333333333334},{"step":389,"nonzero":0.2916666666666667},{"step":391,"nonzero":0.22916666666666666},{"step":393,"nonzero":0.5416666666666666},{"step":395,"nonzero":0.20833333333333334},{"step":397,"nonzero":0.5833333333333334},{"step":399,"nonzero":0.3125},{"step":401,"nonzero":0.5},{"step":403,"nonzero":0.6666666666666666},{"step":405,"nonzero":0.7291666666666666},{"step":407,"nonzero":0.6458333333333334},{"step":409,"nonzero":0.3958333333333333},{"step":411,"nonzero":0.5},{"step":413,"nonzero":0.5208333333333334},{"step":415,"nonzero":0.625},{"step":417,"nonzero":0.6041666666666666},{"step":419,"nonzero":0.6041666666666666},{"step":421,"nonzero":0.4583333333333333},{"step":423,"nonzero":0.8333333333333334},{"step":425,"nonzero":0.9166666666666666},{"step":427,"nonzero":0.6458333333333334},{"step":429,"nonzero":0.3958333333333333},{"step":431,"nonzero":0.25},{"step":433,"nonzero":0.9166666666666666},{"step":435,"nonzero":0.9375},{"step":437,"nonzero":0.375},{"step":439,"nonzero":0.7291666666666666},{"step":441,"nonzero":0.375},{"step":443,"nonzero":0.8125},{"step":445,"nonzero":0.5625},{"step":447,"nonzero":0.75},{"step":449,"nonzero":0.4166666666666667},{"step":451,"nonzero":0.6458333333333334},{"step":453,"nonzero":0.4583333333333333},{"step":455,"nonzero":0.5},{"step":457,"nonzero":0.3958333333333333},{"step":459,"nonzero":0.6041666666666666},{"step":461,"nonzero":0.7291666666666666},{"step":463,"nonzero":0.6458333333333334},{"step":465,"nonzero":0.5208333333333334},{"step":467,"nonzero":0.4375},{"step":469,"nonzero":0.4791666666666667},{"step":471,"nonzero":0.375},{"step":473,"nonzero":0.6458333333333334},{"step":475,"nonzero":0.625},{"step":477,"nonzero":0.9166666666666666},{"step":479,"nonzero":0.6875},{"step":481,"nonzero":0.5625},{"step":483,"nonzero":0.5416666666666666},{"step":485,"nonzero":0.5625},{"step":487,"nonzero":0.625},{"step":489,"nonzero":0.4583333333333333},{"step":491,"nonzero":0.3333333333333333},{"step":493,"nonzero":0.5625},{"step":495,"nonzero":0.3958333333333333},{"step":497,"nonzero":0.4583333333333333},{"step":499,"nonzero":0.7291666666666666},{"step":501,"nonzero":0.3958333333333333},{"step":503,"nonzero":0.6458333333333334},{"step":505,"nonzero":0.5625},{"step":507,"nonzero":0.6875},{"step":509,"nonzero":0.7916666666666666},{"step":511,"nonzero":0.8958333333333334},{"step":513,"nonzero":0.4583333333333333},{"step":515,"nonzero":0.4791666666666667},{"step":517,"nonzero":0.0},{"step":519,"nonzero":0.4375},{"step":521,"nonzero":0.3541666666666667},{"step":523,"nonzero":0.7916666666666666},{"step":525,"nonzero":0.9375},{"step":527,"nonzero":0.3125},{"step":529,"nonzero":0.4375},{"step":531,"nonzero":0.8333333333333334},{"step":533,"nonzero":0.7916666666666666},{"step":535,"nonzero":0.25},{"step":537,"nonzero":0.0},{"step":539,"nonzero":0.7083333333333334},{"step":541,"nonzero":0.4583333333333333},{"step":543,"nonzero":0.5833333333333334},{"step":545,"nonzero":0.5416666666666666},{"step":547,"nonzero":0.5625},{"step":549,"nonzero":0.8125},{"step":551,"nonzero":0.5208333333333334},{"step":553,"nonzero":0.7708333333333334},{"step":555,"nonzero":0.3333333333333333},{"step":557,"nonzero":0.875},{"step":559,"nonzero":0.4166666666666667},{"step":561,"nonzero":0.4166666666666667},{"step":563,"nonzero":0.5833333333333334},{"step":565,"nonzero":0.6875},{"step":567,"nonzero":0.4583333333333333},{"step":569,"nonzero":0.4583333333333333},{"step":571,"nonzero":0.25},{"step":573,"nonzero":0.3958333333333333},{"step":575,"nonzero":0.8125},{"step":577,"nonzero":0.5833333333333334},{"step":579,"nonzero":0.4166666666666667},{"step":581,"nonzero":0.3125},{"step":583,"nonzero":0.5416666666666666},{"step":585,"nonzero":0.5833333333333334},{"step":587,"nonzero":0.6875},{"step":589,"nonzero":0.5},{"step":591,"nonzero":0.6458333333333334},{"step":593,"nonzero":0.6041666666666666},{"step":595,"nonzero":0.22916666666666666},{"step":597,"nonzero":0.8958333333333334},{"step":599,"nonzero":0.3333333333333333},{"step":601,"nonzero":0.5208333333333334},{"step":603,"nonzero":0.25},{"step":605,"nonzero":0.6041666666666666},{"step":607,"nonzero":0.7708333333333334},{"step":609,"nonzero":0.4791666666666667},{"step":611,"nonzero":0.4583333333333333},{"step":613,"nonzero":0.5208333333333334},{"step":615,"nonzero":0.3125},{"step":617,"nonzero":0.4791666666666667},{"step":619,"nonzero":0.6666666666666666},{"step":621,"nonzero":0.75},{"step":623,"nonzero":0.6875},{"step":625,"nonzero":0.2708333333333333},{"step":627,"nonzero":0.5},{"step":629,"nonzero":0.4375},{"step":631,"nonzero":0.8958333333333334},{"step":633,"nonzero":0.6875},{"step":635,"nonzero":0.4375},{"step":637,"nonzero":0.6041666666666666},{"step":639,"nonzero":0.8333333333333334},{"step":641,"nonzero":0.4791666666666667},{"step":643,"nonzero":0.625},{"step":645,"nonzero":0.125},{"step":647,"nonzero":0.8333333333333334},{"step":649,"nonzero":0.75},{"step":651,"nonzero":0.4375},{"step":653,"nonzero":0.8125},{"step":655,"nonzero":0.375},{"step":657,"nonzero":0.375},{"step":659,"nonzero":0.9166666666666666},{"step":661,"nonzero":0.6041666666666666},{"step":663,"nonzero":0.6041666666666666},{"step":665,"nonzero":0.6875},{"step":667,"nonzero":0.6458333333333334},{"step":669,"nonzero":0.75},{"step":671,"nonzero":0.875},{"step":673,"nonzero":0.8541666666666666},{"step":675,"nonzero":0.5208333333333334},{"step":677,"nonzero":0.6458333333333334},{"step":679,"nonzero":0.625},{"step":681,"nonzero":0.4583333333333333},{"step":683,"nonzero":0.5},{"step":685,"nonzero":0.6875},{"step":687,"nonzero":0.7291666666666666},{"step":689,"nonzero":0.3333333333333333},{"step":691,"nonzero":0.4375},{"step":693,"nonzero":0.3958333333333333},{"step":695,"nonzero":0.4791666666666667},{"step":697,"nonzero":0.8958333333333334},{"step":699,"nonzero":0.5416666666666666},{"step":701,"nonzero":0.6875},{"step":703,"nonzero":0.6666666666666666},{"step":705,"nonzero":0.5208333333333334},{"step":707,"nonzero":0.7708333333333334},{"step":709,"nonzero":0.5},{"step":711,"nonzero":0.8125},{"step":713,"nonzero":0.3958333333333333},{"step":715,"nonzero":0.75},{"step":717,"nonzero":0.4583333333333333},{"step":719,"nonzero":0.75},{"step":721,"nonzero":0.3333333333333333},{"step":723,"nonzero":0.5416666666666666},{"step":725,"nonzero":0.4791666666666667},{"step":727,"nonzero":0.4583333333333333},{"step":729,"nonzero":0.5625},{"step":731,"nonzero":0.625},{"step":733,"nonzero":0.75},{"step":735,"nonzero":0.75},{"step":737,"nonzero":0.4791666666666667},{"step":739,"nonzero":0.625},{"step":741,"nonzero":0.4583333333333333},{"step":743,"nonzero":0.8333333333333334},{"step":745,"nonzero":0.4791666666666667},{"step":747,"nonzero":0.8541666666666666},{"step":749,"nonzero":0.4375},{"step":751,"nonzero":0.9166666666666666},{"step":753,"nonzero":0.7916666666666666},{"step":755,"nonzero":0.5416666666666666},{"step":757,"nonzero":0.5},{"step":759,"nonzero":0.625},{"step":761,"nonzero":0.7291666666666666},{"step":763,"nonzero":0.7708333333333334},{"step":765,"nonzero":0.5},{"step":767,"nonzero":0.5},{"step":769,"nonzero":0.6875},{"step":771,"nonzero":0.3958333333333333},{"step":773,"nonzero":0.5625},{"step":775,"nonzero":0.7083333333333334},{"step":777,"nonzero":0.4791666666666667},{"step":779,"nonzero":0.7291666666666666},{"step":781,"nonzero":0.6041666666666666},{"step":783,"nonzero":0.6458333333333334},{"step":785,"nonzero":0.75},{"step":787,"nonzero":0.9166666666666666},{"step":789,"nonzero":0.7291666666666666},{"step":791,"nonzero":0.8125},{"step":793,"nonzero":0.75},{"step":795,"nonzero":0.1875},{"step":797,"nonzero":0.6666666666666666},{"step":799,"nonzero":0.5},{"step":801,"nonzero":0.6666666666666666},{"step":803,"nonzero":0.4791666666666667},{"step":805,"nonzero":0.625},{"step":807,"nonzero":0.8125},{"step":809,"nonzero":0.4166666666666667},{"step":811,"nonzero":0.875},{"step":813,"nonzero":0.9375},{"step":815,"nonzero":0.7708333333333334},{"step":817,"nonzero":0.8541666666666666},{"step":819,"nonzero":0.3541666666666667},{"step":821,"nonzero":0.6666666666666666},{"step":823,"nonzero":0.6666666666666666},{"step":825,"nonzero":0.3125},{"step":827,"nonzero":0.6875},{"step":829,"nonzero":0.6875},{"step":831,"nonzero":0.5625},{"step":833,"nonzero":0.5208333333333334},{"step":835,"nonzero":0.875},{"step":837,"nonzero":0.6666666666666666},{"step":839,"nonzero":0.6458333333333334},{"step":841,"nonzero":0.7083333333333334},{"step":843,"nonzero":0.6041666666666666},{"step":845,"nonzero":0.6875},{"step":847,"nonzero":0.7291666666666666},{"step":849,"nonzero":0.7083333333333334},{"step":851,"nonzero":0.375},{"step":853,"nonzero":0.5833333333333334},{"step":855,"nonzero":0.5625},{"step":857,"nonzero":0.4166666666666667},{"step":859,"nonzero":0.8958333333333334},{"step":861,"nonzero":0.8333333333333334},{"step":863,"nonzero":0.6666666666666666},{"step":865,"nonzero":0.6458333333333334},{"step":867,"nonzero":0.22916666666666666},{"step":869,"nonzero":0.6666666666666666},{"step":871,"nonzero":0.5625},{"step":873,"nonzero":0.6666666666666666},{"step":875,"nonzero":0.7083333333333334},{"step":877,"nonzero":0.6875},{"step":879,"nonzero":0.3125},{"step":881,"nonzero":0.8125},{"step":883,"nonzero":0.625},{"step":885,"nonzero":0.4375},{"step":887,"nonzero":0.4791666666666667},{"step":889,"nonzero":0.7083333333333334},{"step":891,"nonzero":0.4375},{"step":893,"nonzero":0.5416666666666666},{"step":895,"nonzero":0.5833333333333334},{"step":897,"nonzero":0.6875},{"step":899,"nonzero":0.6666666666666666},{"step":901,"nonzero":0.625},{"step":903,"nonzero":0.6875},{"step":905,"nonzero":0.5416666666666666},{"step":907,"nonzero":0.9166666666666666},{"step":909,"nonzero":0.625},{"step":911,"nonzero":0.4791666666666667},{"step":913,"nonzero":0.6041666666666666},{"step":915,"nonzero":0.625},{"step":917,"nonzero":0.2916666666666667},{"step":919,"nonzero":0.7916666666666666},{"step":921,"nonzero":0.6041666666666666},{"step":923,"nonzero":0.75},{"step":925,"nonzero":0.5416666666666666},{"step":927,"nonzero":0.6458333333333334},{"step":929,"nonzero":0.7708333333333334},{"step":931,"nonzero":0.8333333333333334},{"step":933,"nonzero":0.5416666666666666},{"step":935,"nonzero":0.6666666666666666},{"step":937,"nonzero":0.6041666666666666},{"step":939,"nonzero":0.75},{"step":941,"nonzero":0.6041666666666666},{"step":943,"nonzero":0.2708333333333333},{"step":945,"nonzero":0.875},{"step":947,"nonzero":0.3958333333333333},{"step":949,"nonzero":0.75},{"step":951,"nonzero":0.9791666666666666},{"step":953,"nonzero":0.7708333333333334},{"step":955,"nonzero":0.8541666666666666},{"step":957,"nonzero":0.7083333333333334},{"step":959,"nonzero":0.5416666666666666},{"step":961,"nonzero":0.5208333333333334},{"step":963,"nonzero":0.7083333333333334},{"step":965,"nonzero":0.9583333333333334},{"step":967,"nonzero":0.5416666666666666},{"step":969,"nonzero":0.75},{"step":971,"nonzero":0.5},{"step":973,"nonzero":0.6666666666666666},{"step":975,"nonzero":0.8125},{"step":977,"nonzero":0.6041666666666666},{"step":979,"nonzero":0.8541666666666666},{"step":981,"nonzero":0.4791666666666667},{"step":983,"nonzero":0.5833333333333334},{"step":985,"nonzero":0.6875},{"step":987,"nonzero":0.8125},{"step":989,"nonzero":0.6875},{"step":991,"nonzero":0.5416666666666666},{"step":993,"nonzero":0.8333333333333334},{"step":995,"nonzero":0.7291666666666666},{"step":997,"nonzero":0.7916666666666666},{"step":999,"nonzero":0.875},{"step":1001,"nonzero":0.625},{"step":1003,"nonzero":0.5},{"step":1005,"nonzero":0.4166666666666667},{"step":1007,"nonzero":0.6666666666666666},{"step":1009,"nonzero":0.8541666666666666},{"step":1011,"nonzero":0.4791666666666667},{"step":1013,"nonzero":0.5416666666666666},{"step":1015,"nonzero":0.7916666666666666},{"step":1017,"nonzero":0.5625},{"step":1019,"nonzero":0.4583333333333333},{"step":1021,"nonzero":0.75},{"step":1023,"nonzero":0.7291666666666666},{"step":1025,"nonzero":0.75},{"step":1027,"nonzero":0.3333333333333333},{"step":1029,"nonzero":0.7708333333333334},{"step":1031,"nonzero":0.5833333333333334},{"step":1033,"nonzero":0.6875},{"step":1035,"nonzero":0.7916666666666666},{"step":1037,"nonzero":0.7291666666666666},{"step":1039,"nonzero":0.625},{"step":1041,"nonzero":0.5833333333333334},{"step":1043,"nonzero":0.7291666666666666},{"step":1045,"nonzero":0.6041666666666666},{"step":1047,"nonzero":0.5625},{"step":1049,"nonzero":0.3541666666666667},{"step":1051,"nonzero":0.7291666666666666},{"step":1053,"nonzero":0.625},{"step":1055,"nonzero":0.5833333333333334},{"step":1057,"nonzero":0.6458333333333334},{"step":1059,"nonzero":0.3125},{"step":1061,"nonzero":0.4791666666666667},{"step":1063,"nonzero":0.5208333333333334},{"step":1065,"nonzero":0.7083333333333334},{"step":1067,"nonzero":0.6666666666666666},{"step":1069,"nonzero":0.7708333333333334},{"step":1071,"nonzero":0.7083333333333334},{"step":1073,"nonzero":0.75},{"step":1075,"nonzero":0.9166666666666666},{"step":1077,"nonzero":0.5},{"step":1079,"nonzero":0.875},{"step":1081,"nonzero":0.625},{"step":1083,"nonzero":0.8958333333333334},{"step":1085,"nonzero":0.5208333333333334},{"step":1087,"nonzero":0.6041666666666666},{"step":1089,"nonzero":0.5},{"step":1091,"nonzero":0.4791666666666667},{"step":1093,"nonzero":0.6041666666666666},{"step":1095,"nonzero":0.875},{"step":1097,"nonzero":0.6666666666666666},{"step":1099,"nonzero":0.8333333333333334},{"step":1101,"nonzero":0.625},{"step":1103,"nonzero":0.5416666666666666},{"step":1105,"nonzero":0.5},{"step":1107,"nonzero":0.25},{"step":1109,"nonzero":0.375},{"step":1111,"nonzero":0.8333333333333334},{"step":1113,"nonzero":0.875},{"step":1115,"nonzero":0.8958333333333334},{"step":1117,"nonzero":0.3958333333333333},{"step":1119,"nonzero":0.6666666666666666},{"step":1121,"nonzero":0.4791666666666667},{"step":1123,"nonzero":0.6875},{"step":1125,"nonzero":0.9791666666666666},{"step":1127,"nonzero":0.4791666666666667},{"step":1129,"nonzero":0.8333333333333334},{"step":1131,"nonzero":0.75},{"step":1133,"nonzero":0.8125},{"step":1135,"nonzero":0.8958333333333334},{"step":1137,"nonzero":0.6875},{"step":1139,"nonzero":0.5833333333333334},{"step":1141,"nonzero":0.7083333333333334},{"step":1143,"nonzero":0.9583333333333334},{"step":1145,"nonzero":0.625},{"step":1147,"nonzero":0.5625},{"step":1149,"nonzero":0.8333333333333334},{"step":1151,"nonzero":0.8333333333333334},{"step":1153,"nonzero":0.7291666666666666},{"step":1155,"nonzero":0.8125},{"step":1157,"nonzero":0.6875},{"step":1159,"nonzero":0.6666666666666666},{"step":1161,"nonzero":0.8333333333333334},{"step":1163,"nonzero":0.4166666666666667},{"step":1165,"nonzero":0.9166666666666666},{"step":1167,"nonzero":0.1875},{"step":1169,"nonzero":0.375},{"step":1171,"nonzero":0.8125},{"step":1173,"nonzero":0.8541666666666666},{"step":1175,"nonzero":0.7083333333333334},{"step":1177,"nonzero":0.6041666666666666},{"step":1179,"nonzero":0.6875},{"step":1181,"nonzero":0.9583333333333334},{"step":1183,"nonzero":0.875},{"step":1185,"nonzero":0.6458333333333334},{"step":1187,"nonzero":0.3541666666666667},{"step":1189,"nonzero":0.5},{"step":1191,"nonzero":0.6666666666666666},{"step":1193,"nonzero":0.7916666666666666},{"step":1195,"nonzero":0.5625},{"step":1197,"nonzero":0.6041666666666666},{"step":1199,"nonzero":0.3333333333333333},{"step":1201,"nonzero":0.6458333333333334},{"step":1203,"nonzero":0.625},{"step":1205,"nonzero":0.9166666666666666},{"step":1207,"nonzero":0.375},{"step":1209,"nonzero":0.7083333333333334},{"step":1211,"nonzero":0.3125},{"step":1213,"nonzero":0.4375},{"step":1215,"nonzero":0.6041666666666666},{"step":1217,"nonzero":0.375},{"step":1219,"nonzero":0.5416666666666666},{"step":1221,"nonzero":0.5416666666666666},{"step":1223,"nonzero":0.625},{"step":1225,"nonzero":0.625},{"step":1227,"nonzero":0.8125},{"step":1229,"nonzero":0.5833333333333334},{"step":1231,"nonzero":0.7291666666666666},{"step":1233,"nonzero":0.5833333333333334},{"step":1235,"nonzero":0.7083333333333334},{"step":1237,"nonzero":0.5208333333333334},{"step":1239,"nonzero":0.5625},{"step":1241,"nonzero":0.6666666666666666},{"step":1243,"nonzero":0.8125},{"step":1245,"nonzero":0.5833333333333334},{"step":1247,"nonzero":0.5833333333333334},{"step":1249,"nonzero":0.5416666666666666}]},{"id":"dapo-dora-500","label":"DAPO \u00b7 DoRA \u00b7 500 prompts","run_ids":["1790888478"],"loss":"DAPO","adapter":"DoRA","prompts":500,"repo_r":false,"holdout_by_checkpoint":[{"step":0,"holdout":-1.3664,"train_core":-0.9836,"valid_rate":0.9},{"step":100,"holdout":-0.5921,"train_core":-0.3014,"valid_rate":0.94},{"step":200,"holdout":-0.2871,"train_core":-0.2174,"valid_rate":0.9},{"step":300,"holdout":-0.1811,"train_core":-0.2697,"valid_rate":0.96},{"step":400,"holdout":-0.1349,"train_core":-0.2306,"valid_rate":0.96},{"step":500,"holdout":-0.3163,"train_core":-0.3094,"valid_rate":0.96},{"step":600,"holdout":-0.2793,"train_core":-0.2758,"valid_rate":0.96}],"selected_step":200,"selected_holdout":-0.2871,"train_steps":[{"step":1,"reward":-2.2583541870117188,"core":-0.7187981605529785,"cost":-0.08142795413732529,"kl":0.0,"entropy":0.5175154954195023},{"step":2,"reward":-2.8222386837005615,"core":-0.8446833491325378,"cost":-0.1883409470319748,"kl":0.0003414341790630715,"entropy":0.46462152898311615},{"step":3,"reward":-3.0783119201660156,"core":-0.9070556163787842,"cost":-0.27601465582847595,"kl":0.0003648601195891388,"entropy":0.525731697678566},{"step":4,"reward":-3.2396552562713623,"core":-0.9910776615142822,"cost":-0.22459912300109863,"kl":0.00042319940257584676,"entropy":0.5602856278419495},{"step":5,"reward":-4.079254150390625,"core":-1.2026258707046509,"cost":-0.4129016399383545,"kl":0.0007077228437992744,"entropy":0.5397979915142059},{"step":6,"reward":-2.7479960918426514,"core":-0.8262714743614197,"cost":-0.18225421011447906,"kl":0.0012136566365370527,"entropy":0.47236866503953934},{"step":7,"reward":-4.372781753540039,"core":-1.296641230583191,"cost":-0.4673194885253906,"kl":0.0016940231435000896,"entropy":0.5161081850528717},{"step":8,"reward":-4.14979887008667,"core":-1.2115402221679688,"cost":-0.3921591341495514,"kl":0.002117889904184267,"entropy":0.5413282513618469},{"step":9,"reward":-1.6470260620117188,"core":-0.4786129891872406,"cost":-0.07390362024307251,"kl":0.0032924010884016752,"entropy":0.5062985196709633},{"step":10,"reward":-2.3623363971710205,"core":-0.7036617398262024,"cost":-0.14993514120578766,"kl":0.004643583029974252,"entropy":0.5326384902000427},{"step":11,"reward":-2.4492290019989014,"core":-0.7705448269844055,"cost":-0.08357500284910202,"kl":0.0046980761107988656,"entropy":0.5452654212713242},{"step":12,"reward":-2.592170000076294,"core":-0.7366448044776917,"cost":-0.2468019723892212,"kl":0.005522349732927978,"entropy":0.46091198176145554},{"step":13,"reward":-1.9815053939819336,"core":-0.6026206612586975,"cost":-0.05854995921254158,"kl":0.006594213307835162,"entropy":0.4190223142504692},{"step":14,"reward":-2.154705047607422,"core":-0.5868971347808838,"cost":-0.20243513584136963,"kl":0.005879727657884359,"entropy":0.5007992088794708},{"step":15,"reward":-3.38305926322937,"core":-0.9970297813415527,"cost":-0.2985580861568451,"kl":0.008329997188411653,"entropy":0.603497177362442},{"step":16,"reward":-1.9868131875991821,"core":-0.5928412079811096,"cost":-0.07513251900672913,"kl":0.013686114456504583,"entropy":0.5518319010734558},{"step":17,"reward":-4.495201587677002,"core":-1.353699803352356,"cost":-0.4010500907897949,"kl":0.011663946090266109,"entropy":0.48530174791812897},{"step":18,"reward":-3.5310420989990234,"core":-1.0752028226852417,"cost":-0.27868059277534485,"kl":0.0210705129429698,"entropy":0.6163071691989899},{"step":19,"reward":-2.9645843505859375,"core":-0.8911211490631104,"cost":-0.21344445645809174,"kl":0.018595502246171236,"entropy":0.5424124896526337},{"step":20,"reward":-1.9027677774429321,"core":-0.5238895416259766,"cost":-0.0915633961558342,"kl":0.017686411971226335,"entropy":0.5093543380498886},{"step":21,"reward":-2.3971798419952393,"core":-0.7219164371490479,"cost":-0.15371887385845184,"kl":0.018117581494152546,"entropy":0.5575638562440872},{"step":22,"reward":-2.6257119178771973,"core":-0.7469937205314636,"cost":-0.23276710510253906,"kl":0.020029794424772263,"entropy":0.5521185100078583},{"step":23,"reward":-2.5266544818878174,"core":-0.7490949034690857,"cost":-0.16184964776039124,"kl":0.016204269602894783,"entropy":0.5301552712917328},{"step":24,"reward":-1.6437348127365112,"core":-0.4907893240451813,"cost":-0.033517707139253616,"kl":0.019893803633749485,"entropy":0.4946097880601883},{"step":25,"reward":-3.564955949783325,"core":-1.0510613918304443,"cost":-0.345501184463501,"kl":0.030477065593004227,"entropy":0.48024751245975494},{"step":26,"reward":-1.4705771207809448,"core":-0.3810986578464508,"cost":-0.09301191568374634,"kl":0.022909884341061115,"entropy":0.5050311833620071},{"step":27,"reward":-1.8076300621032715,"core":-0.5036688446998596,"cost":-0.12063080072402954,"kl":0.02654599118977785,"entropy":0.4766043722629547},{"step":28,"reward":-3.3686485290527344,"core":-0.9462370276451111,"cost":-0.39950668811798096,"kl":0.028403771575540304,"entropy":0.48003337532281876},{"step":29,"reward":-0.7442280650138855,"core":-0.20105434954166412,"cost":-0.0037409404758363962,"kl":0.03241538489237428,"entropy":0.5861381739377975},{"step":30,"reward":-2.150768518447876,"core":-0.5830921530723572,"cost":-0.13031792640686035,"kl":0.027019452303647995,"entropy":0.4318394511938095},{"step":31,"reward":-2.798632860183716,"core":-0.8220300674438477,"cost":-0.19317668676376343,"kl":0.04871866665780544,"entropy":0.4760742634534836},{"step":32,"reward":-1.4486538171768188,"core":-0.44297853112220764,"cost":-0.024645889177918434,"kl":0.03409888409078121,"entropy":0.5341331511735916},{"step":33,"reward":-2.1901447772979736,"core":-0.6644293665885925,"cost":-0.08933360129594803,"kl":0.051316557452082634,"entropy":0.6125934720039368},{"step":34,"reward":-1.62333345413208,"core":-0.476178377866745,"cost":-0.03446215018630028,"kl":0.037453255616128445,"entropy":0.5190038159489632},{"step":35,"reward":-2.5849969387054443,"core":-0.7654110789299011,"cost":-0.1317792385816574,"kl":0.04998639598488808,"entropy":0.5914313346147537},{"step":36,"reward":-3.0871522426605225,"core":-0.9132447242736816,"cost":-0.27900490164756775,"kl":0.021493727806955576,"entropy":0.4078662842512131},{"step":37,"reward":-2.8518083095550537,"core":-0.8164014220237732,"cost":-0.23423223197460175,"kl":0.034611737821251154,"entropy":0.5225715488195419},{"step":38,"reward":-1.357528567314148,"core":-0.4000699818134308,"cost":-0.029816126450896263,"kl":0.03275314159691334,"entropy":0.5092587992548943},{"step":39,"reward":-1.1294904947280884,"core":-0.2510816156864166,"cost":-0.05714567005634308,"kl":0.04660669947043061,"entropy":0.4443307891488075},{"step":40,"reward":-1.3249813318252563,"core":-0.377162903547287,"cost":-0.014988139271736145,"kl":0.06307777110487223,"entropy":0.46834737062454224},{"step":41,"reward":-2.5335960388183594,"core":-0.7421789169311523,"cost":-0.14712058007717133,"kl":0.04427155340090394,"entropy":0.43947839736938477},{"step":42,"reward":-1.3910945653915405,"core":-0.4118584096431732,"cost":-0.07228801399469376,"kl":0.0296913618221879,"entropy":0.47731897979974747},{"step":43,"reward":-2.106637716293335,"core":-0.6085387468338013,"cost":-0.1437252312898636,"kl":0.02749614743515849,"entropy":0.4830034524202347},{"step":44,"reward":-2.1712100505828857,"core":-0.6895913481712341,"cost":-0.022903887555003166,"kl":0.026979173067957163,"entropy":0.5258966907858849},{"step":45,"reward":-1.828173041343689,"core":-0.5475524067878723,"cost":-0.05532899498939514,"kl":0.03761504264548421,"entropy":0.5310668721795082},{"step":46,"reward":-2.364797353744507,"core":-0.7118757367134094,"cost":-0.07704820483922958,"kl":0.04340431932359934,"entropy":0.6118668615818024},{"step":47,"reward":-2.0722875595092773,"core":-0.6281713843345642,"cost":-0.0438256710767746,"kl":0.029606439173221588,"entropy":0.5650274902582169},{"step":48,"reward":-1.8660558462142944,"core":-0.5237553715705872,"cost":-0.13010650873184204,"kl":0.027632316574454308,"entropy":0.5199741050601006},{"step":49,"reward":-2.1725738048553467,"core":-0.62917160987854,"cost":-0.09703946858644485,"kl":0.030300951562821865,"entropy":0.5300090909004211},{"step":50,"reward":-1.8848165273666382,"core":-0.5383018851280212,"cost":-0.15160347521305084,"kl":0.02689758874475956,"entropy":0.5104280263185501},{"step":51,"reward":-2.502105951309204,"core":-0.759929358959198,"cost":-0.09830498695373535,"kl":0.0326392543502152,"entropy":0.5142777264118195},{"step":52,"reward":-2.7332212924957275,"core":-0.8200030326843262,"cost":-0.18530379235744476,"kl":0.039379815105348825,"entropy":0.5344063341617584},{"step":53,"reward":-1.946760654449463,"core":-0.5853051543235779,"cost":-0.06786800175905228,"kl":0.03130850335583091,"entropy":0.5276036709547043},{"step":54,"reward":-2.557898759841919,"core":-0.7492439150810242,"cost":-0.22296027839183807,"kl":0.02927253767848015,"entropy":0.5857551842927933},{"step":55,"reward":-2.5176916122436523,"core":-0.7456750869750977,"cost":-0.16145391762256622,"kl":0.04145596455782652,"entropy":0.6178131699562073},{"step":56,"reward":-1.7590694427490234,"core":-0.5133083462715149,"cost":-0.14465978741645813,"kl":0.036444948986172676,"entropy":0.5177262276411057},{"step":57,"reward":-2.51888108253479,"core":-0.7908404469490051,"cost":-0.1031838059425354,"kl":0.03290024632588029,"entropy":0.532370537519455},{"step":58,"reward":-1.9363027811050415,"core":-0.5949809551239014,"cost":-0.07153964042663574,"kl":0.025100387632846832,"entropy":0.507893018424511},{"step":59,"reward":-1.3449989557266235,"core":-0.3713056147098541,"cost":-0.053951140493154526,"kl":0.04196404945105314,"entropy":0.5301215648651123},{"step":60,"reward":-2.255572557449341,"core":-0.6616412997245789,"cost":-0.1601010411977768,"kl":0.03791634785011411,"entropy":0.5169960036873817},{"step":61,"reward":-1.9476432800292969,"core":-0.5995694994926453,"cost":-0.08991929143667221,"kl":0.022725319024175406,"entropy":0.4246878996491432},{"step":62,"reward":-2.5129587650299072,"core":-0.7557328343391418,"cost":-0.15850716829299927,"kl":0.043456084094941616,"entropy":0.5279121026396751},{"step":63,"reward":-3.5527961254119873,"core":-1.0748368501663208,"cost":-0.2808397114276886,"kl":0.032251797150820494,"entropy":0.5008651539683342},{"step":64,"reward":-1.6249265670776367,"core":-0.48082423210144043,"cost":-0.018108127638697624,"kl":0.029343940317630768,"entropy":0.462602399289608},{"step":65,"reward":-1.432915210723877,"core":-0.41015371680259705,"cost":-0.05785098671913147,"kl":0.033832135144621134,"entropy":0.4729837328195572},{"step":66,"reward":-1.7170195579528809,"core":-0.5119597315788269,"cost":-0.025595514103770256,"kl":0.02844525407999754,"entropy":0.4983122944831848},{"step":67,"reward":-2.290572166442871,"core":-0.7069031596183777,"cost":-0.11942615360021591,"kl":0.027568696066737175,"entropy":0.5554464161396027},{"step":68,"reward":-2.212280511856079,"core":-0.6457431316375732,"cost":-0.0994017943739891,"kl":0.029378199484199286,"entropy":0.5325375199317932},{"step":69,"reward":-1.824541449546814,"core":-0.5339727997779846,"cost":-0.06588203459978104,"kl":0.02620895765721798,"entropy":0.4899812266230583},{"step":70,"reward":-2.3865201473236084,"core":-0.7245516180992126,"cost":-0.10365074127912521,"kl":0.029014612082391977,"entropy":0.48940690606832504},{"step":71,"reward":-3.1606369018554688,"core":-0.9338743090629578,"cost":-0.2696571350097656,"kl":0.0304777342826128,"entropy":0.4840560704469681},{"step":72,"reward":-0.6216464042663574,"core":-0.15396271646022797,"cost":0.0,"kl":0.030215533450245857,"entropy":0.461787112057209},{"step":73,"reward":-1.6633821725845337,"core":-0.4743526875972748,"cost":-0.06699735671281815,"kl":0.021076489239931107,"entropy":0.3998277261853218},{"step":74,"reward":-2.1773746013641357,"core":-0.6407776474952698,"cost":-0.09755244106054306,"kl":0.03812991175800562,"entropy":0.4867136627435684},{"step":75,"reward":-3.311882257461548,"core":-0.9731907844543457,"cost":-0.23880775272846222,"kl":0.03614260023459792,"entropy":0.3855713829398155},{"step":76,"reward":-2.012470245361328,"core":-0.5462887287139893,"cost":-0.12316068261861801,"kl":0.0337161966599524,"entropy":0.4631222262978554},{"step":77,"reward":-2.091233730316162,"core":-0.630896270275116,"cost":-0.12365353852510452,"kl":0.027528538834303617,"entropy":0.46210697293281555},{"step":78,"reward":-2.091110944747925,"core":-0.6088910698890686,"cost":-0.12278863787651062,"kl":0.03393101692199707,"entropy":0.4979391470551491},{"step":79,"reward":-2.6561028957366943,"core":-0.7765931487083435,"cost":-0.1792311668395996,"kl":0.03411144996061921,"entropy":0.5099152475595474},{"step":80,"reward":-2.768949270248413,"core":-0.7284974455833435,"cost":-0.38991978764533997,"kl":0.03280480718240142,"entropy":0.4228607192635536},{"step":81,"reward":-2.5505127906799316,"core":-0.724075973033905,"cost":-0.1916048526763916,"kl":0.041271690744906664,"entropy":0.432807981967926},{"step":82,"reward":-2.58038067817688,"core":-0.7840139269828796,"cost":-0.13373248279094696,"kl":0.04298915434628725,"entropy":0.4699771776795387},{"step":83,"reward":-3.816054344177246,"core":-1.121380090713501,"cost":-0.32663068175315857,"kl":0.04792900197207928,"entropy":0.43402470648288727},{"step":84,"reward":-1.7678827047348022,"core":-0.5075635313987732,"cost":-0.08510518074035645,"kl":0.03541494160890579,"entropy":0.48696595430374146},{"step":85,"reward":-1.2124921083450317,"core":-0.2664916217327118,"cost":-0.0833478644490242,"kl":0.045296838507056236,"entropy":0.5211653783917427},{"step":86,"reward":-3.4442903995513916,"core":-1.0491682291030884,"cost":-0.23249578475952148,"kl":0.05181323178112507,"entropy":0.5423404723405838},{"step":87,"reward":-1.6897567510604858,"core":-0.5004059672355652,"cost":-0.05710388347506523,"kl":0.04151967214420438,"entropy":0.5096144750714302},{"step":88,"reward":-2.5592079162597656,"core":-0.7309944033622742,"cost":-0.2654210031032562,"kl":0.04466244671493769,"entropy":0.47746598720550537},{"step":89,"reward":-1.7266311645507812,"core":-0.5207023620605469,"cost":0.0,"kl":0.04978260584175587,"entropy":0.4484492912888527},{"step":90,"reward":-1.8229247331619263,"core":-0.5529453754425049,"cost":-0.038413312286138535,"kl":0.05047748237848282,"entropy":0.5827973335981369},{"step":91,"reward":-1.4080638885498047,"core":-0.34426388144493103,"cost":-0.09113077074289322,"kl":0.049232049845159054,"entropy":0.4117197245359421},{"step":92,"reward":-1.7551628351211548,"core":-0.45359620451927185,"cost":-0.06806603819131851,"kl":0.04993007984012365,"entropy":0.4303789660334587},{"step":93,"reward":-2.6721136569976807,"core":-0.7749433517456055,"cost":-0.2533738911151886,"kl":0.024836143013089895,"entropy":0.4117610827088356},{"step":94,"reward":-2.077385187149048,"core":-0.5906595587730408,"cost":-0.09083869308233261,"kl":0.030222194734960794,"entropy":0.4435240775346756},{"step":95,"reward":-1.7035261392593384,"core":-0.4590916633605957,"cost":-0.14048121869564056,"kl":0.04518442042171955,"entropy":0.4942728951573372},{"step":96,"reward":-1.7640098333358765,"core":-0.5073683857917786,"cost":-0.07157260179519653,"kl":0.04312931094318628,"entropy":0.49415842443704605},{"step":97,"reward":-3.979565382003784,"core":-1.1599020957946777,"cost":-0.3666039705276489,"kl":0.03745116526260972,"entropy":0.5326782912015915},{"step":98,"reward":-1.6957165002822876,"core":-0.4183351993560791,"cost":-0.09816071391105652,"kl":0.03027441492304206,"entropy":0.4041472002863884},{"step":99,"reward":-2.5398595333099365,"core":-0.7224586009979248,"cost":-0.21813519299030304,"kl":0.03895319811999798,"entropy":0.367630772292614},{"step":100,"reward":-1.1186811923980713,"core":-0.2595687806606293,"cost":-0.07943575829267502,"kl":0.03769024182111025,"entropy":0.5187031626701355},{"step":101,"reward":-1.3209997415542603,"core":-0.3430666923522949,"cost":-0.05925421789288521,"kl":0.05336556676775217,"entropy":0.5642230957746506},{"step":102,"reward":-4.077831745147705,"core":-1.2334696054458618,"cost":-0.23811078071594238,"kl":0.023001310182735324,"entropy":0.41316618770360947},{"step":103,"reward":-1.2808353900909424,"core":-0.3806631565093994,"cost":-0.047156840562820435,"kl":0.04824428539723158,"entropy":0.5396380871534348},{"step":104,"reward":-1.2167550325393677,"core":-0.3544147312641144,"cost":0.0,"kl":0.04036737885326147,"entropy":0.5788947343826294},{"step":105,"reward":-1.4340168237686157,"core":-0.4368489980697632,"cost":-0.0030707214027643204,"kl":0.031128973234444857,"entropy":0.46605631709098816},{"step":106,"reward":-2.287844657897949,"core":-0.6482540965080261,"cost":-0.164870023727417,"kl":0.08151441812515259,"entropy":0.5148320719599724},{"step":107,"reward":-1.653283953666687,"core":-0.473600298166275,"cost":-0.06366004794836044,"kl":0.025611489079892635,"entropy":0.4415111690759659},{"step":108,"reward":-2.484212636947632,"core":-0.7439934611320496,"cost":-0.12332481145858765,"kl":0.028829993214458227,"entropy":0.47094661742448807},{"step":109,"reward":-2.666764497756958,"core":-0.7820723056793213,"cost":-0.19875602424144745,"kl":0.044819862581789494,"entropy":0.5206407606601715},{"step":110,"reward":-0.7467561364173889,"core":-0.08824241906404495,"cost":-0.09657556563615799,"kl":0.14276649709790945,"entropy":0.5547252893447876},{"step":111,"reward":-1.7656563520431519,"core":-0.48803138732910156,"cost":-0.09370791912078857,"kl":0.05021661054342985,"entropy":0.504486657679081},{"step":112,"reward":-1.6140289306640625,"core":-0.4781523644924164,"cost":-0.014183745719492435,"kl":0.04165044566616416,"entropy":0.5428533405065536},{"step":113,"reward":-1.8938220739364624,"core":-0.5747334361076355,"cost":-0.03985725715756416,"kl":0.04295138921588659,"entropy":0.5486041754484177},{"step":114,"reward":-2.8314197063446045,"core":-0.8392234444618225,"cost":-0.2202368527650833,"kl":0.029161239974200726,"entropy":0.41100867092609406},{"step":115,"reward":-1.3638941049575806,"core":-0.391702800989151,"cost":-0.0006192874279804528,"kl":0.03477204591035843,"entropy":0.4728422984480858},{"step":116,"reward":-0.4994257986545563,"core":-0.033590007573366165,"cost":-0.03516728803515434,"kl":0.04695186857134104,"entropy":0.5097825229167938},{"step":117,"reward":-2.3779799938201904,"core":-0.7103433609008789,"cost":-0.12228352576494217,"kl":0.03964553587138653,"entropy":0.508890874683857},{"step":118,"reward":-2.634070634841919,"core":-0.7934896349906921,"cost":-0.16454577445983887,"kl":0.04168548434972763,"entropy":0.5001868009567261},{"step":119,"reward":-1.2772618532180786,"core":-0.3662048876285553,"cost":-0.019303523004055023,"kl":0.03010126482695341,"entropy":0.49292605370283127},{"step":120,"reward":-2.6118836402893066,"core":-0.7468083500862122,"cost":-0.21225090324878693,"kl":0.035726485308259726,"entropy":0.501760870218277},{"step":121,"reward":-3.0275180339813232,"core":-0.8993127942085266,"cost":-0.18457698822021484,"kl":0.023306324612349272,"entropy":0.46750691533088684},{"step":122,"reward":-1.2352417707443237,"core":-0.27435103058815,"cost":-0.02349540777504444,"kl":0.030109284445643425,"entropy":0.5490787476301193},{"step":123,"reward":-1.8252366781234741,"core":-0.5588381886482239,"cost":-0.04220656678080559,"kl":0.040081558749079704,"entropy":0.526354469358921},{"step":124,"reward":-2.013472318649292,"core":-0.6024989485740662,"cost":-0.06725873798131943,"kl":0.02563102263957262,"entropy":0.46757063269615173},{"step":125,"reward":-2.285832166671753,"core":-0.5087155699729919,"cost":-0.3593169152736664,"kl":0.04814954847097397,"entropy":0.502959132194519},{"step":126,"reward":-1.663586139678955,"core":-0.46001169085502625,"cost":-0.12985561788082123,"kl":0.03486048802733421,"entropy":0.5692003071308136},{"step":127,"reward":-1.7155803442001343,"core":-0.49928057193756104,"cost":-0.05696810409426689,"kl":0.03044707700610161,"entropy":0.4385080859065056},{"step":128,"reward":-1.2681964635849,"core":-0.3592963218688965,"cost":0.0,"kl":0.0350230960175395,"entropy":0.5475611388683319},{"step":129,"reward":-2.7562711238861084,"core":-0.8163196444511414,"cost":-0.1610969454050064,"kl":0.034524408634752035,"entropy":0.5553793758153915},{"step":130,"reward":-1.0149344205856323,"core":-0.2964383065700531,"cost":-0.012002676725387573,"kl":0.028376285918056965,"entropy":0.532302588224411},{"step":131,"reward":-4.0955491065979,"core":-1.1916794776916504,"cost":-0.45712241530418396,"kl":0.025717863347381353,"entropy":0.5063357800245285},{"step":132,"reward":-2.914886236190796,"core":-0.8356122970581055,"cost":-0.22432611882686615,"kl":0.0212113237939775,"entropy":0.5490123555064201},{"step":133,"reward":-1.4710965156555176,"core":-0.4466257393360138,"cost":-0.016432899981737137,"kl":0.0434797341004014,"entropy":0.5151384770870209},{"step":134,"reward":-2.4560203552246094,"core":-0.7551843523979187,"cost":-0.06031244620680809,"kl":0.029577048495411873,"entropy":0.428433820605278},{"step":135,"reward":-1.969560980796814,"core":-0.5619977712631226,"cost":-0.08142593502998352,"kl":0.03064304357394576,"entropy":0.5617777109146118},{"step":136,"reward":-3.1287155151367188,"core":-0.847884476184845,"cost":-0.2946149408817291,"kl":0.039413224905729294,"entropy":0.5142714902758598},{"step":137,"reward":-1.2574447393417358,"core":-0.3415938913822174,"cost":-0.08007626235485077,"kl":0.04431325988844037,"entropy":0.5446592047810555},{"step":138,"reward":-2.5315189361572266,"core":-0.7526838183403015,"cost":-0.17230994999408722,"kl":0.028064455837011337,"entropy":0.46199432015419006},{"step":139,"reward":-2.5649354457855225,"core":-0.7640125155448914,"cost":-0.1463962197303772,"kl":0.05958856828510761,"entropy":0.5865035802125931},{"step":140,"reward":-1.5007318258285522,"core":-0.451430082321167,"cost":-0.04800720140337944,"kl":0.040525340009480715,"entropy":0.5496735125780106},{"step":141,"reward":-2.0981743335723877,"core":-0.6521201133728027,"cost":-0.07376662641763687,"kl":0.029093934688717127,"entropy":0.5630480870604515},{"step":142,"reward":-1.819861888885498,"core":-0.49169158935546875,"cost":-0.13362757861614227,"kl":0.046658055391162634,"entropy":0.5256420224905014},{"step":143,"reward":-1.4757436513900757,"core":-0.42477384209632874,"cost":-0.034444861114025116,"kl":0.033122763969004154,"entropy":0.5107346251606941},{"step":144,"reward":-1.7449983358383179,"core":-0.4254261255264282,"cost":-0.10845080763101578,"kl":0.053417584858834743,"entropy":0.5291166752576828},{"step":145,"reward":-2.6999270915985107,"core":-0.7964989542961121,"cost":-0.1857130527496338,"kl":0.030895366799086332,"entropy":0.4801866039633751},{"step":146,"reward":-1.9148845672607422,"core":-0.47851476073265076,"cost":-0.16891001164913177,"kl":0.04121271055191755,"entropy":0.5204219222068787},{"step":147,"reward":-0.8133847713470459,"core":-0.21126578748226166,"cost":-0.013422285206615925,"kl":0.05245538055896759,"entropy":0.5257513001561165},{"step":148,"reward":-1.4763755798339844,"core":-0.3933868110179901,"cost":-0.09699990600347519,"kl":0.036135842092335224,"entropy":0.46007681638002396},{"step":149,"reward":-1.309221625328064,"core":-0.38159939646720886,"cost":-0.010491366498172283,"kl":0.0382732143625617,"entropy":0.5719642490148544},{"step":150,"reward":-2.1611757278442383,"core":-0.6790690422058105,"cost":-0.07086265832185745,"kl":0.0440499703399837,"entropy":0.5043991357088089},{"step":151,"reward":-2.201347827911377,"core":-0.6441351771354675,"cost":-0.12836016714572906,"kl":0.04169363109394908,"entropy":0.602887749671936},{"step":152,"reward":-1.958264946937561,"core":-0.4956416189670563,"cost":-0.15866470336914062,"kl":0.04214617516845465,"entropy":0.6064857095479965},{"step":153,"reward":-1.911770224571228,"core":-0.5585699081420898,"cost":-0.1065373420715332,"kl":0.03468718891963363,"entropy":0.4818182438611984},{"step":154,"reward":-1.1972190141677856,"core":-0.3545617163181305,"cost":-0.007761400192975998,"kl":0.025104181841015816,"entropy":0.5257758796215057},{"step":155,"reward":-0.945858895778656,"core":-0.23747707903385162,"cost":-0.03020610846579075,"kl":0.024725202471017838,"entropy":0.4818975478410721},{"step":156,"reward":-2.480168104171753,"core":-0.7239038944244385,"cost":-0.18633484840393066,"kl":0.036509792786091566,"entropy":0.4701046943664551},{"step":157,"reward":-2.7848002910614014,"core":-0.808879554271698,"cost":-0.1759851723909378,"kl":0.03883510595187545,"entropy":0.45664142072200775},{"step":158,"reward":-2.5820233821868896,"core":-0.7464540600776672,"cost":-0.1949481964111328,"kl":0.036288573406636715,"entropy":0.5211409851908684},{"step":159,"reward":-1.2250462770462036,"core":-0.3017663061618805,"cost":-0.0727577805519104,"kl":0.05945082660764456,"entropy":0.5226495340466499},{"step":160,"reward":-1.410044550895691,"core":-0.37649187445640564,"cost":-0.032461851835250854,"kl":0.04412583913654089,"entropy":0.4963560327887535},{"step":161,"reward":-1.6037869453430176,"core":-0.46409741044044495,"cost":-0.10142717510461807,"kl":0.041008473839610815,"entropy":0.45995914191007614},{"step":162,"reward":-1.1025972366333008,"core":-0.2825500965118408,"cost":-0.017910288646817207,"kl":0.04994418006390333,"entropy":0.5695390999317169},{"step":163,"reward":-2.5545103549957275,"core":-0.6766800880432129,"cost":-0.2729949951171875,"kl":0.048000019043684006,"entropy":0.5413224250078201},{"step":164,"reward":-2.148076057434082,"core":-0.6494308710098267,"cost":-0.14426589012145996,"kl":0.04328054841607809,"entropy":0.445263035595417},{"step":165,"reward":-0.7090676426887512,"core":-0.039151184260845184,"cost":-0.0619322769343853,"kl":0.04527767188847065,"entropy":0.4971519336104393},{"step":166,"reward":-1.753989577293396,"core":-0.5330433249473572,"cost":-0.10609182715415955,"kl":0.04364787973463535,"entropy":0.43819499015808105},{"step":167,"reward":-1.6842812299728394,"core":-0.5203526616096497,"cost":-0.02589035965502262,"kl":0.033122438471764326,"entropy":0.4297536537051201},{"step":168,"reward":-2.089564085006714,"core":-0.6164278388023376,"cost":-0.09584219008684158,"kl":0.03977143205702305,"entropy":0.5290933921933174},{"step":169,"reward":-0.9882173538208008,"core":-0.3071129620075226,"cost":0.0,"kl":0.032039019744843245,"entropy":0.4870501607656479},{"step":170,"reward":-2.279606580734253,"core":-0.6554690003395081,"cost":-0.11044120043516159,"kl":0.038066070061177015,"entropy":0.5084326267242432},{"step":171,"reward":-2.126521110534668,"core":-0.6690587997436523,"cost":-0.0301988422870636,"kl":0.04270030837506056,"entropy":0.5167096257209778},{"step":172,"reward":-1.0941381454467773,"core":-0.24506354331970215,"cost":-0.041714493185281754,"kl":0.03481235168874264,"entropy":0.47125859558582306},{"step":173,"reward":-2.6140353679656982,"core":-0.7956860661506653,"cost":-0.11532144993543625,"kl":0.03907797345891595,"entropy":0.49762599915266037},{"step":174,"reward":-1.2896062135696411,"core":-0.3418678343296051,"cost":-0.03887967765331268,"kl":0.04962729103863239,"entropy":0.4081578627228737},{"step":175,"reward":-1.9139732122421265,"core":-0.5825616717338562,"cost":-0.08950045704841614,"kl":0.057450476102530956,"entropy":0.5255044102668762},{"step":176,"reward":-1.5298062562942505,"core":-0.4144175946712494,"cost":-0.046357426792383194,"kl":0.03573377709835768,"entropy":0.4577096328139305},{"step":177,"reward":-1.111075520515442,"core":-0.2690523564815521,"cost":-0.05195843055844307,"kl":0.04683049116283655,"entropy":0.5889363437891006},{"step":178,"reward":-1.7254232168197632,"core":-0.48005422949790955,"cost":-0.06301222741603851,"kl":0.056711052544415,"entropy":0.5111922323703766},{"step":179,"reward":-1.788792610168457,"core":-0.5367498993873596,"cost":-0.07549933344125748,"kl":0.04613354988396168,"entropy":0.5515349060297012},{"step":180,"reward":-0.5334793925285339,"core":-0.05174204334616661,"cost":-0.05199960991740227,"kl":0.06174402870237827,"entropy":0.49639052152633667},{"step":181,"reward":-1.9984325170516968,"core":-0.5477374792098999,"cost":-0.1431635171175003,"kl":0.053249492309987545,"entropy":0.5117484107613564},{"step":182,"reward":-1.6596482992172241,"core":-0.4540916979312897,"cost":-0.09323420375585556,"kl":0.029234146233648062,"entropy":0.4554065465927124},{"step":183,"reward":-1.2594927549362183,"core":-0.3648504912853241,"cost":-0.027299994602799416,"kl":0.05151416314765811,"entropy":0.5186193585395813},{"step":184,"reward":-1.8723154067993164,"core":-0.5559692978858948,"cost":-0.08287317305803299,"kl":0.05168693698942661,"entropy":0.5793407261371613},{"step":185,"reward":-1.5003634691238403,"core":-0.3877560794353485,"cost":-0.029395269230008125,"kl":0.04970232117921114,"entropy":0.5274239853024483},{"step":186,"reward":-2.2315962314605713,"core":-0.5990477204322815,"cost":-0.20576685667037964,"kl":0.036588591523468494,"entropy":0.37650103867053986},{"step":187,"reward":-0.3655007779598236,"core":0.012411962263286114,"cost":-0.016438523307442665,"kl":0.050369614735245705,"entropy":0.5654818192124367},{"step":188,"reward":-0.4630074203014374,"core":-0.06613025069236755,"cost":0.0,"kl":0.05021501146256924,"entropy":0.518939197063446},{"step":189,"reward":-1.647727131843567,"core":-0.45652762055397034,"cost":-0.09589579701423645,"kl":0.05619007535278797,"entropy":0.596095085144043},{"step":190,"reward":-2.2693347930908203,"core":-0.7021069526672363,"cost":-0.09006287902593613,"kl":0.035041666589677334,"entropy":0.48257531225681305},{"step":191,"reward":-2.2686119079589844,"core":-0.7095221877098083,"cost":-0.007610150147229433,"kl":0.025711005087941885,"entropy":0.43089232593774796},{"step":192,"reward":-1.0518594980239868,"core":-0.21572618186473846,"cost":-0.04258406534790993,"kl":0.06265760399401188,"entropy":0.5570673793554306},{"step":193,"reward":-0.4396805763244629,"core":0.02085278369486332,"cost":-0.012436025775969028,"kl":0.051610520109534264,"entropy":0.5549077391624451},{"step":194,"reward":-2.4869983196258545,"core":-0.7558137774467468,"cost":-0.13347947597503662,"kl":0.09119207877665758,"entropy":0.5707646757364273},{"step":195,"reward":-1.063534140586853,"core":-0.2319256067276001,"cost":-0.040807515382766724,"kl":0.05342838540673256,"entropy":0.5017183646559715},{"step":196,"reward":-1.275577187538147,"core":-0.34527596831321716,"cost":-0.01773219369351864,"kl":0.053232972510159016,"entropy":0.500701293349266},{"step":197,"reward":-0.6466855406761169,"core":-0.10978706926107407,"cost":0.0,"kl":0.04419397935271263,"entropy":0.49861591309309006},{"step":198,"reward":-1.671899437904358,"core":-0.4487757384777069,"cost":-0.026789555326104164,"kl":0.03656411776319146,"entropy":0.44521414488554},{"step":199,"reward":-0.6196610927581787,"core":-0.08435239642858505,"cost":-0.033980634063482285,"kl":0.07894671615213156,"entropy":0.5297744870185852},{"step":200,"reward":-1.7076339721679688,"core":-0.48154401779174805,"cost":-0.04049306735396385,"kl":0.05217537097632885,"entropy":0.5010471493005753},{"step":201,"reward":-1.8571547269821167,"core":-0.5311425924301147,"cost":-0.0722011923789978,"kl":0.06112318392843008,"entropy":0.6086879968643188},{"step":202,"reward":-1.8132587671279907,"core":-0.5586836934089661,"cost":-0.03719378635287285,"kl":0.04096537549048662,"entropy":0.5164606273174286},{"step":203,"reward":-1.5732053518295288,"core":-0.45382487773895264,"cost":-0.0323016457259655,"kl":0.05552923306822777,"entropy":0.5481879711151123},{"step":204,"reward":-2.1921839714050293,"core":-0.654815137386322,"cost":-0.077290840446949,"kl":0.041718166787177324,"entropy":0.4735347628593445},{"step":205,"reward":-0.8974270820617676,"core":-0.19742567837238312,"cost":-0.028994610533118248,"kl":0.06262446939945221,"entropy":0.5565068125724792},{"step":206,"reward":-1.521781325340271,"core":-0.47259703278541565,"cost":-0.023234084248542786,"kl":0.04265961516648531,"entropy":0.6324272900819778},{"step":207,"reward":-1.5919828414916992,"core":-0.43255627155303955,"cost":-0.08514190465211868,"kl":0.05020248191431165,"entropy":0.5156587660312653},{"step":208,"reward":-1.5606015920639038,"core":-0.46448883414268494,"cost":-0.030614888295531273,"kl":0.04536101780831814,"entropy":0.5513234585523605},{"step":209,"reward":-1.4646447896957397,"core":-0.39711442589759827,"cost":-0.08811134099960327,"kl":0.035950591787695885,"entropy":0.483015812933445},{"step":210,"reward":-0.9264035224914551,"core":-0.1821521669626236,"cost":0.0,"kl":0.05143128242343664,"entropy":0.5325013995170593},{"step":211,"reward":-1.5885721445083618,"core":-0.5056578516960144,"cost":0.0,"kl":0.04322008555755019,"entropy":0.578865259885788},{"step":212,"reward":-2.1772565841674805,"core":-0.5657700896263123,"cost":-0.1957327127456665,"kl":0.08619091473519802,"entropy":0.4562637209892273},{"step":213,"reward":-1.2956535816192627,"core":-0.3432800769805908,"cost":-0.11771252006292343,"kl":0.049991932697594166,"entropy":0.4675370529294014},{"step":214,"reward":-1.1264839172363281,"core":-0.28303518891334534,"cost":-0.033460844308137894,"kl":0.05023485608398914,"entropy":0.5106493085622787},{"step":215,"reward":-1.604879379272461,"core":-0.4852595329284668,"cost":-0.04251068830490112,"kl":0.04801614675670862,"entropy":0.571051150560379},{"step":216,"reward":-2.1297285556793213,"core":-0.6205782294273376,"cost":-0.07815941423177719,"kl":0.054874228313565254,"entropy":0.576031818985939},{"step":217,"reward":-2.3888609409332275,"core":-0.6737391352653503,"cost":-0.13997675478458405,"kl":0.04330454673618078,"entropy":0.41371869295835495},{"step":218,"reward":-1.4206938743591309,"core":-0.3714517056941986,"cost":-0.055522602051496506,"kl":0.05223282752558589,"entropy":0.5170278623700142},{"step":219,"reward":-1.6937928199768066,"core":-0.403399795293808,"cost":-0.12985505163669586,"kl":0.0588147072121501,"entropy":0.5424451678991318},{"step":220,"reward":-3.7110707759857178,"core":-1.0575107336044312,"cost":-0.45176419615745544,"kl":0.055756138637661934,"entropy":0.4969850033521652},{"step":221,"reward":-1.6635074615478516,"core":-0.46857786178588867,"cost":-0.022570209577679634,"kl":0.050990681163966656,"entropy":0.488637238740921},{"step":222,"reward":-1.4906657934188843,"core":-0.43780967593193054,"cost":-0.09841787815093994,"kl":0.051903532817959785,"entropy":0.5796484053134918},{"step":223,"reward":-2.5117766857147217,"core":-0.7433169484138489,"cost":-0.2078533172607422,"kl":0.04647802747786045,"entropy":0.47561316192150116},{"step":224,"reward":-2.08378267288208,"core":-0.6196135878562927,"cost":-0.1379389464855194,"kl":0.07385965622961521,"entropy":0.49965517967939377},{"step":225,"reward":-1.0215301513671875,"core":-0.2790750563144684,"cost":0.0,"kl":0.06684829201549292,"entropy":0.5622880756855011},{"step":226,"reward":-2.929211378097534,"core":-0.8951205611228943,"cost":-0.1469879299402237,"kl":0.04672187194228172,"entropy":0.399949349462986},{"step":227,"reward":-1.7060190439224243,"core":-0.4479038715362549,"cost":-0.11289624124765396,"kl":0.058669078163802624,"entropy":0.5986448228359222},{"step":228,"reward":-2.230532646179199,"core":-0.6315669417381287,"cost":-0.09514794498682022,"kl":0.057073418982326984,"entropy":0.4719007685780525},{"step":229,"reward":-0.43390345573425293,"core":-0.019269432872533798,"cost":-0.008517544716596603,"kl":0.08267918787896633,"entropy":0.5215593725442886},{"step":230,"reward":-1.6425584554672241,"core":-0.48649099469184875,"cost":-0.037886399775743484,"kl":0.04777078051120043,"entropy":0.4979580193758011},{"step":231,"reward":-0.5218505263328552,"core":-0.09242425113916397,"cost":0.0,"kl":0.062241108156740665,"entropy":0.50031977891922},{"step":232,"reward":-1.7412115335464478,"core":-0.4939696490764618,"cost":-0.07760825008153915,"kl":0.06283297296613455,"entropy":0.47858593612909317},{"step":233,"reward":-2.942991256713867,"core":-0.8557217121124268,"cost":-0.24475181102752686,"kl":0.09712653048336506,"entropy":0.5256034880876541},{"step":234,"reward":-3.545212984085083,"core":-1.0793641805648804,"cost":-0.24345315992832184,"kl":0.06314524170011282,"entropy":0.5581411421298981},{"step":235,"reward":-1.9254652261734009,"core":-0.5223553776741028,"cost":-0.13878042995929718,"kl":0.06928504444658756,"entropy":0.5520646721124649},{"step":236,"reward":-2.6982767581939697,"core":-0.7694005966186523,"cost":-0.30532869696617126,"kl":0.07506162114441395,"entropy":0.5422471612691879},{"step":237,"reward":-1.2817848920822144,"core":-0.3489394187927246,"cost":-0.0948437824845314,"kl":0.09769578464329243,"entropy":0.5746345221996307},{"step":238,"reward":0.1054466962814331,"core":0.21467332541942596,"cost":0.0,"kl":0.06431716401129961,"entropy":0.500253714621067},{"step":239,"reward":-2.400165319442749,"core":-0.7099825739860535,"cost":-0.1369442492723465,"kl":0.08207260072231293,"entropy":0.42577748745679855},{"step":240,"reward":-0.5584749579429626,"core":-0.11388438940048218,"cost":-8.080956467892975e-05,"kl":0.051889981143176556,"entropy":0.47747012972831726},{"step":241,"reward":-0.8138454556465149,"core":-0.1424129158258438,"cost":-0.025053711608052254,"kl":0.0879319068044424,"entropy":0.527310699224472},{"step":242,"reward":-1.5310091972351074,"core":-0.45238617062568665,"cost":-0.016940372064709663,"kl":0.07611241843551397,"entropy":0.5951195359230042},{"step":243,"reward":-1.4310506582260132,"core":-0.4085456430912018,"cost":-0.006573010701686144,"kl":0.055641861632466316,"entropy":0.46985987573862076},{"step":244,"reward":-3.547766923904419,"core":-1.0781718492507935,"cost":-0.2240191251039505,"kl":0.06631986889988184,"entropy":0.5049320310354233},{"step":245,"reward":-1.0012037754058838,"core":-0.24350625276565552,"cost":-0.03110007382929325,"kl":0.055185469798743725,"entropy":0.4700998067855835},{"step":246,"reward":-0.6889750361442566,"core":-0.14117036759853363,"cost":0.0,"kl":0.050905474461615086,"entropy":0.44345559924840927},{"step":247,"reward":-1.260155200958252,"core":-0.33393827080726624,"cost":-0.03270373120903969,"kl":0.08864832110702991,"entropy":0.5354894250631332},{"step":248,"reward":-1.166643500328064,"core":-0.2435205727815628,"cost":-0.029719481244683266,"kl":0.07485305331647396,"entropy":0.5284587070345879},{"step":249,"reward":-2.2896783351898193,"core":-0.6087990403175354,"cost":-0.20404721796512604,"kl":0.09182457439601421,"entropy":0.6851475983858109},{"step":250,"reward":-1.4285935163497925,"core":-0.41830873489379883,"cost":-0.046762410551309586,"kl":0.05580398440361023,"entropy":0.5329142659902573},{"step":251,"reward":-1.4265365600585938,"core":-0.39461883902549744,"cost":-0.0954194888472557,"kl":0.08795084059238434,"entropy":0.5957340151071548},{"step":252,"reward":-0.3610812723636627,"core":0.0031426448840647936,"cost":-0.028034694492816925,"kl":0.06928119249641895,"entropy":0.4576331600546837},{"step":253,"reward":-1.652825951576233,"core":-0.4531271755695343,"cost":-0.11275657266378403,"kl":0.06085479166358709,"entropy":0.503224328160286},{"step":254,"reward":-1.3019208908081055,"core":-0.33576712012290955,"cost":-0.10766920447349548,"kl":0.08714175224304199,"entropy":0.5887280702590942},{"step":255,"reward":-1.804491400718689,"core":-0.5186957716941833,"cost":-0.04399867728352547,"kl":0.07527711242437363,"entropy":0.56082883477211},{"step":256,"reward":-2.298532247543335,"core":-0.5905722975730896,"cost":-0.23648659884929657,"kl":0.08592472784221172,"entropy":0.6219925284385681},{"step":257,"reward":-2.1486656665802,"core":-0.6150329113006592,"cost":-0.16305609047412872,"kl":0.0833673533052206,"entropy":0.48658110946416855},{"step":258,"reward":-1.3329252004623413,"core":-0.3759138882160187,"cost":-0.09161257743835449,"kl":0.11454677954316139,"entropy":0.6136042773723602},{"step":259,"reward":-2.2908742427825928,"core":-0.6994277834892273,"cost":-0.0781199112534523,"kl":0.06076886784285307,"entropy":0.6177183240652084},{"step":260,"reward":-1.9014710187911987,"core":-0.5215187668800354,"cost":-0.15013782680034637,"kl":0.066450459882617,"entropy":0.5014851167798042},{"step":261,"reward":-2.2741506099700928,"core":-0.6847249865531921,"cost":-0.09212180972099304,"kl":0.09535912424325943,"entropy":0.5410813391208649},{"step":262,"reward":-2.0602738857269287,"core":-0.623727560043335,"cost":-0.056580960750579834,"kl":0.07811853475868702,"entropy":0.5598636865615845},{"step":263,"reward":-0.8354699015617371,"core":-0.20628495514392853,"cost":-0.01855931058526039,"kl":0.07780182361602783,"entropy":0.5903552919626236},{"step":264,"reward":-1.2731889486312866,"core":-0.3304367959499359,"cost":-0.023760246112942696,"kl":0.07237277086824179,"entropy":0.5617560148239136},{"step":265,"reward":-2.6413116455078125,"core":-0.7874035239219666,"cost":-0.10640212148427963,"kl":0.06799034401774406,"entropy":0.5569972395896912},{"step":266,"reward":-1.45614755153656,"core":-0.411419153213501,"cost":-0.05993078649044037,"kl":0.07183277327567339,"entropy":0.5673278421163559},{"step":267,"reward":-2.889509916305542,"core":-0.836533784866333,"cost":-0.17519502341747284,"kl":0.06929501798003912,"entropy":0.6035037785768509},{"step":268,"reward":-1.0891221761703491,"core":-0.2386317402124405,"cost":-0.049488574266433716,"kl":0.07943293172866106,"entropy":0.6293973475694656},{"step":269,"reward":-1.1529382467269897,"core":-0.28212276101112366,"cost":-0.028693636879324913,"kl":0.06487694568932056,"entropy":0.6040018647909164},{"step":270,"reward":-1.1755338907241821,"core":-0.3353976309299469,"cost":-0.01551121473312378,"kl":0.06994283385574818,"entropy":0.5812961459159851},{"step":271,"reward":-1.439653754234314,"core":-0.3849279582500458,"cost":-0.026488447561860085,"kl":0.09498397074639797,"entropy":0.5236003324389458},{"step":272,"reward":-1.0205532312393188,"core":-0.1724759191274643,"cost":-0.06594216078519821,"kl":0.10328394174575806,"entropy":0.5368244349956512},{"step":273,"reward":-2.788396120071411,"core":-0.7566990256309509,"cost":-0.35730576515197754,"kl":0.06774121802300215,"entropy":0.5224339514970779},{"step":274,"reward":-1.049761176109314,"core":-0.27504321932792664,"cost":0.0,"kl":0.058667803183197975,"entropy":0.4787627086043358},{"step":275,"reward":-1.969422459602356,"core":-0.5635052919387817,"cost":-0.11998122930526733,"kl":0.07846770249307156,"entropy":0.5864869356155396},{"step":276,"reward":-1.7645221948623657,"core":-0.4754582941532135,"cost":-0.13156147301197052,"kl":0.048249357379972935,"entropy":0.525999054312706},{"step":277,"reward":-2.2519967555999756,"core":-0.6607593894004822,"cost":-0.10379818826913834,"kl":0.06930791959166527,"entropy":0.5478402674198151},{"step":278,"reward":-1.6856845617294312,"core":-0.4658820331096649,"cost":-0.0908215269446373,"kl":0.07049603387713432,"entropy":0.6284660845994949},{"step":279,"reward":-0.7986266016960144,"core":-0.1401640772819519,"cost":-0.040450140833854675,"kl":0.09048894140869379,"entropy":0.625972643494606},{"step":280,"reward":-1.7003353834152222,"core":-0.43518343567848206,"cost":-0.05854323133826256,"kl":0.11186783201992512,"entropy":0.615846186876297},{"step":281,"reward":-1.6428827047348022,"core":-0.4574872553348541,"cost":-0.03454479202628136,"kl":0.08052907325327396,"entropy":0.6582830846309662},{"step":282,"reward":-1.472918152809143,"core":-0.4255286455154419,"cost":-0.055675625801086426,"kl":0.0564681189134717,"entropy":0.5095196813344955},{"step":283,"reward":-2.2314112186431885,"core":-0.654926061630249,"cost":-0.12519513070583344,"kl":0.07327324151992798,"entropy":0.6096293330192566},{"step":284,"reward":-2.275779962539673,"core":-0.7138751149177551,"cost":-0.048185959458351135,"kl":0.06518073193728924,"entropy":0.5934710502624512},{"step":285,"reward":-1.033530831336975,"core":-0.25017669796943665,"cost":-0.03654405102133751,"kl":0.0696746101602912,"entropy":0.5851100087165833},{"step":286,"reward":-1.0889091491699219,"core":-0.22053520381450653,"cost":-0.059560615569353104,"kl":0.0640507098287344,"entropy":0.5465236082673073},{"step":287,"reward":-1.6632490158081055,"core":-0.4449861943721771,"cost":-0.06855051964521408,"kl":0.051891000010073185,"entropy":0.5801744312047958},{"step":288,"reward":-2.0266945362091064,"core":-0.5637138485908508,"cost":-0.06916510313749313,"kl":0.08678936772048473,"entropy":0.6473463773727417},{"step":289,"reward":-1.4015251398086548,"core":-0.39106234908103943,"cost":-0.010397193022072315,"kl":0.06145017687231302,"entropy":0.5482346415519714},{"step":290,"reward":-1.7058366537094116,"core":-0.4503333270549774,"cost":-0.08749818056821823,"kl":0.0885465070605278,"entropy":0.5527619570493698},{"step":291,"reward":-1.4367138147354126,"core":-0.3705965578556061,"cost":-0.10850030183792114,"kl":0.0913483239710331,"entropy":0.6025430560112},{"step":292,"reward":-1.3136852979660034,"core":-0.31083300709724426,"cost":-0.015499897301197052,"kl":0.06516359839588404,"entropy":0.6013837158679962},{"step":293,"reward":-2.1416265964508057,"core":-0.6112580895423889,"cost":-0.17667603492736816,"kl":0.0865451730787754,"entropy":0.6497423052787781},{"step":294,"reward":-1.7611554861068726,"core":-0.5101354122161865,"cost":-0.08872152119874954,"kl":0.06178789399564266,"entropy":0.4600997269153595},{"step":295,"reward":-0.6744655966758728,"core":-0.15267953276634216,"cost":0.0,"kl":0.05565296811982989,"entropy":0.5309076383709908},{"step":296,"reward":-1.1248162984848022,"core":-0.3083236813545227,"cost":-0.03089509718120098,"kl":0.05560723040252924,"entropy":0.5224493369460106},{"step":297,"reward":-1.598069190979004,"core":-0.472182035446167,"cost":-0.04406275972723961,"kl":0.0641411105170846,"entropy":0.6252999007701874},{"step":298,"reward":-2.506352663040161,"core":-0.7375323176383972,"cost":-0.21098339557647705,"kl":0.061885710805654526,"entropy":0.5619473904371262},{"step":299,"reward":-0.8330473303794861,"core":-0.18175257742404938,"cost":-0.019540490582585335,"kl":0.06495212484151125,"entropy":0.6039683520793915},{"step":300,"reward":-1.6887359619140625,"core":-0.4256950318813324,"cost":-0.12575101852416992,"kl":0.07787315361201763,"entropy":0.6103832274675369},{"step":301,"reward":-1.8983577489852905,"core":-0.548782229423523,"cost":-0.09683945029973984,"kl":0.09751512296497822,"entropy":0.6233939975500107},{"step":302,"reward":-1.0684016942977905,"core":-0.26452919840812683,"cost":-0.027274517342448235,"kl":0.08478083834052086,"entropy":0.6471479684114456},{"step":303,"reward":-2.7768070697784424,"core":-0.8469223976135254,"cost":-0.16162903606891632,"kl":0.06716080382466316,"entropy":0.5595730170607567},{"step":304,"reward":-2.0986616611480713,"core":-0.5810444951057434,"cost":-0.10347872227430344,"kl":0.06284415256232023,"entropy":0.5471995696425438},{"step":305,"reward":-0.17269177734851837,"core":0.15805189311504364,"cost":-0.02098008245229721,"kl":0.06782680563628674,"entropy":0.4965778812766075},{"step":306,"reward":-1.822452187538147,"core":-0.519244909286499,"cost":-0.04780559614300728,"kl":0.04315322544425726,"entropy":0.506349690258503},{"step":307,"reward":-1.3564058542251587,"core":-0.36501458287239075,"cost":-0.023124782368540764,"kl":0.05435607675462961,"entropy":0.5137658566236496},{"step":308,"reward":-0.9201064705848694,"core":-0.16725383698940277,"cost":-0.025519229471683502,"kl":0.054362062364816666,"entropy":0.5446485802531242},{"step":309,"reward":-2.5176143646240234,"core":-0.7404963374137878,"cost":-0.2046567052602768,"kl":0.07033550646156073,"entropy":0.6051910817623138},{"step":310,"reward":-0.619391918182373,"core":-0.0404939167201519,"cost":-0.020407063886523247,"kl":0.08583861030638218,"entropy":0.5192616358399391},{"step":311,"reward":-0.8072357773780823,"core":-0.1459284871816635,"cost":-0.029809528961777687,"kl":0.06884016748517752,"entropy":0.4979887008666992},{"step":312,"reward":-0.7283157706260681,"core":-0.1568342000246048,"cost":-0.031249411404132843,"kl":0.07070004194974899,"entropy":0.558951735496521},{"step":313,"reward":-2.8334147930145264,"core":-0.8919251561164856,"cost":-0.1114012822508812,"kl":0.049544806592166424,"entropy":0.47463608533143997},{"step":314,"reward":-1.4093629121780396,"core":-0.3910697400569916,"cost":-0.03670429065823555,"kl":0.060918545350432396,"entropy":0.5501358062028885},{"step":315,"reward":-1.8238941431045532,"core":-0.5462351441383362,"cost":-0.05764535069465637,"kl":0.05333199352025986,"entropy":0.5267983451485634},{"step":316,"reward":-2.8059444427490234,"core":-0.8198339343070984,"cost":-0.252979040145874,"kl":0.056312535889446735,"entropy":0.4887831285595894},{"step":317,"reward":-1.4604803323745728,"core":-0.4196207523345947,"cost":-0.026536494493484497,"kl":0.0679734218865633,"entropy":0.5111366957426071},{"step":318,"reward":-1.6223300695419312,"core":-0.42771852016448975,"cost":-0.11946173757314682,"kl":0.07349005527794361,"entropy":0.5485407114028931},{"step":319,"reward":-1.6068164110183716,"core":-0.389195054769516,"cost":-0.09720417112112045,"kl":0.08628090284764767,"entropy":0.5277336090803146},{"step":320,"reward":-1.7746397256851196,"core":-0.49659785628318787,"cost":-0.11587879806756973,"kl":0.09864052757620811,"entropy":0.5282533392310143},{"step":321,"reward":-1.9430979490280151,"core":-0.5518143177032471,"cost":-0.07267782837152481,"kl":0.07475795410573483,"entropy":0.548317477107048},{"step":322,"reward":-1.8380804061889648,"core":-0.5241191983222961,"cost":-0.09548769146203995,"kl":0.1288712192326784,"entropy":0.6609980463981628},{"step":323,"reward":-1.4508556127548218,"core":-0.37826672196388245,"cost":-0.07507310062646866,"kl":0.07120146043598652,"entropy":0.5671798586845398},{"step":324,"reward":-0.3082037568092346,"core":0.07963743060827255,"cost":-0.012699979357421398,"kl":0.09678476862609386,"entropy":0.531658798456192},{"step":325,"reward":-0.7678082585334778,"core":-0.16094164550304413,"cost":-0.021613361313939095,"kl":0.08331689424812794,"entropy":0.5702159851789474},{"step":326,"reward":-1.556609034538269,"core":-0.3715815246105194,"cost":-0.17782187461853027,"kl":0.1026476789265871,"entropy":0.5834482610225677},{"step":327,"reward":-0.8702929615974426,"core":-0.23986296355724335,"cost":0.0,"kl":0.09232327807694674,"entropy":0.588936984539032},{"step":328,"reward":-0.76417475938797,"core":-0.1512887328863144,"cost":0.0,"kl":0.08835779502987862,"entropy":0.5235328748822212},{"step":329,"reward":-0.0572863407433033,"core":0.11213131994009018,"cost":-0.04578538239002228,"kl":0.11259935609996319,"entropy":0.5570340752601624},{"step":330,"reward":-1.6237373352050781,"core":-0.4044017493724823,"cost":-0.0789535716176033,"kl":0.07659224607050419,"entropy":0.533218115568161},{"step":331,"reward":-1.6896265745162964,"core":-0.49352049827575684,"cost":-0.07298483699560165,"kl":0.09157229401171207,"entropy":0.6063170731067657},{"step":332,"reward":-2.0792410373687744,"core":-0.6436198353767395,"cost":-0.008508334867656231,"kl":0.09494409710168839,"entropy":0.5059293210506439},{"step":333,"reward":-0.5655887126922607,"core":-0.042763758450746536,"cost":-0.006906085181981325,"kl":0.09929689951241016,"entropy":0.5259321257472038},{"step":334,"reward":-1.2463610172271729,"core":-0.3394625186920166,"cost":-0.015352572314441204,"kl":0.07563811168074608,"entropy":0.5511491298675537},{"step":335,"reward":-0.9641647338867188,"core":-0.23634739220142365,"cost":-0.02629409171640873,"kl":0.09593606740236282,"entropy":0.49031862616539},{"step":336,"reward":-1.6184860467910767,"core":-0.4414879083633423,"cost":-0.10810703784227371,"kl":0.10374544374644756,"entropy":0.5492321252822876},{"step":337,"reward":-1.9591541290283203,"core":-0.48471590876579285,"cost":-0.16880004107952118,"kl":0.1083335354924202,"entropy":0.5536152273416519},{"step":338,"reward":-1.4408973455429077,"core":-0.3658616542816162,"cost":-0.1507508009672165,"kl":0.1132427304983139,"entropy":0.521285705268383},{"step":339,"reward":-0.6142082810401917,"core":-0.09202951192855835,"cost":-0.009289060719311237,"kl":0.08351876959204674,"entropy":0.5667688399553299},{"step":340,"reward":-1.3575206995010376,"core":-0.32387489080429077,"cost":-0.030902167782187462,"kl":0.14237014390528202,"entropy":0.5909299552440643},{"step":341,"reward":-0.7412455677986145,"core":-0.11926132440567017,"cost":-0.029570000246167183,"kl":0.09549309778958559,"entropy":0.43687670677900314},{"step":342,"reward":0.2735549211502075,"core":0.3210676610469818,"cost":0.0,"kl":0.14071271941065788,"entropy":0.5466847270727158},{"step":343,"reward":-0.4174693524837494,"core":-0.007141163107007742,"cost":0.0,"kl":0.12034164555370808,"entropy":0.5446251407265663},{"step":344,"reward":-0.581206738948822,"core":-0.04531632363796234,"cost":-0.02743031643331051,"kl":0.09299109689891338,"entropy":0.540509819984436},{"step":345,"reward":-0.7422740459442139,"core":-0.1635415107011795,"cost":-0.05593746900558472,"kl":0.09203469939529896,"entropy":0.5363598614931107},{"step":346,"reward":-1.6344013214111328,"core":-0.48175325989723206,"cost":-0.01665499620139599,"kl":0.07914685644209385,"entropy":0.46677185595035553},{"step":347,"reward":-0.887563169002533,"core":-0.21036075055599213,"cost":-0.005022907629609108,"kl":0.08811941184103489,"entropy":0.49932605773210526},{"step":348,"reward":-2.3034827709198,"core":-0.6405927538871765,"cost":-0.21640263497829437,"kl":0.07946615293622017,"entropy":0.5164642333984375},{"step":349,"reward":0.10525316745042801,"core":0.2198195904493332,"cost":0.0,"kl":0.08629216812551022,"entropy":0.510485902428627},{"step":350,"reward":-2.662794828414917,"core":-0.7828860282897949,"cost":-0.1999518871307373,"kl":0.1245739534497261,"entropy":0.6140792816877365},{"step":351,"reward":-0.46854665875434875,"core":-0.0832129642367363,"cost":-0.0010445626685395837,"kl":0.0818305853754282,"entropy":0.5883145779371262},{"step":352,"reward":-1.8490486145019531,"core":-0.5287882089614868,"cost":-0.11917539685964584,"kl":0.10729866474866867,"entropy":0.6373395770788193},{"step":353,"reward":-1.5826069116592407,"core":-0.45391249656677246,"cost":-0.0695771649479866,"kl":0.0700703077018261,"entropy":0.4569885581731796},{"step":354,"reward":-1.4301601648330688,"core":-0.3958854675292969,"cost":-0.027779191732406616,"kl":0.09316973201930523,"entropy":0.5282879024744034},{"step":355,"reward":-1.4400895833969116,"core":-0.3727504014968872,"cost":-0.10591999441385269,"kl":0.0759150292724371,"entropy":0.4636857137084007},{"step":356,"reward":-0.8911409974098206,"core":-0.17693065106868744,"cost":0.0,"kl":0.1025456041097641,"entropy":0.5915640890598297},{"step":357,"reward":-0.9911008477210999,"core":-0.2547188997268677,"cost":-0.05169105529785156,"kl":0.12877484411001205,"entropy":0.5529535263776779},{"step":358,"reward":-1.8365983963012695,"core":-0.46293747425079346,"cost":-0.2093149870634079,"kl":0.08663756586611271,"entropy":0.5006202384829521},{"step":359,"reward":-0.5544300675392151,"core":-0.09752506017684937,"cost":-0.006699430290609598,"kl":0.08125325199216604,"entropy":0.45859789103269577},{"step":360,"reward":-1.7185264825820923,"core":-0.4447108805179596,"cost":-0.11564157158136368,"kl":0.1137571930885315,"entropy":0.5355140566825867},{"step":361,"reward":-2.0987534523010254,"core":-0.6073179244995117,"cost":-0.05173344910144806,"kl":0.0768904210999608,"entropy":0.5040950104594231},{"step":362,"reward":-1.1856380701065063,"core":-0.31835171580314636,"cost":-0.011900939978659153,"kl":0.08839940652251244,"entropy":0.5920946449041367},{"step":363,"reward":-1.1312682628631592,"core":-0.2395949512720108,"cost":-0.03990907594561577,"kl":0.08437606692314148,"entropy":0.5639157891273499},{"step":364,"reward":-3.00356125831604,"core":-0.8946507573127747,"cost":-0.2369220107793808,"kl":0.07534476462751627,"entropy":0.5287485793232918},{"step":365,"reward":-1.6256386041641235,"core":-0.4245121479034424,"cost":-0.15839022397994995,"kl":0.10573675110936165,"entropy":0.5274813026189804},{"step":366,"reward":-1.0974594354629517,"core":-0.27603763341903687,"cost":-0.042053889483213425,"kl":0.07532125897705555,"entropy":0.500742107629776},{"step":367,"reward":-1.513895034790039,"core":-0.4162464439868927,"cost":-0.07430178672075272,"kl":0.06330604013055563,"entropy":0.49029046297073364},{"step":368,"reward":-0.7448094487190247,"core":-0.09605605155229568,"cost":-0.0649871677160263,"kl":0.0661610197275877,"entropy":0.4899819940328598},{"step":369,"reward":0.21641536056995392,"core":0.20678742229938507,"cost":0.0,"kl":0.10270466841757298,"entropy":0.5247152373194695},{"step":370,"reward":-1.1036792993545532,"core":-0.2749479115009308,"cost":-0.058082085102796555,"kl":0.05308647360652685,"entropy":0.47609688341617584},{"step":371,"reward":-0.6613367199897766,"core":-0.16641268134117126,"cost":-0.01992659457027912,"kl":0.08499318733811378,"entropy":0.5441625863313675},{"step":372,"reward":-1.2329820394515991,"core":-0.31595858931541443,"cost":-0.0402916856110096,"kl":0.08163681998848915,"entropy":0.5195420458912849},{"step":373,"reward":-3.0354416370391846,"core":-0.9094502329826355,"cost":-0.24607038497924805,"kl":0.07335513457655907,"entropy":0.5091366767883301},{"step":374,"reward":-0.28080984950065613,"core":0.0566420704126358,"cost":-0.018587112426757812,"kl":0.07684742193669081,"entropy":0.4316621422767639},{"step":375,"reward":-1.3343466520309448,"core":-0.37615370750427246,"cost":-0.06554431468248367,"kl":0.06320162024348974,"entropy":0.5228423103690147},{"step":376,"reward":-1.0541197061538696,"core":-0.2665749192237854,"cost":-0.011804583482444286,"kl":0.07090324349701405,"entropy":0.47220373898744583},{"step":377,"reward":-1.5267601013183594,"core":-0.4021267890930176,"cost":-0.08892292529344559,"kl":0.07027987111359835,"entropy":0.4633055776357651},{"step":378,"reward":-1.0943905115127563,"core":-0.2851952016353607,"cost":0.0,"kl":0.07761777378618717,"entropy":0.5027717426419258},{"step":379,"reward":-2.000152349472046,"core":-0.5886936187744141,"cost":-0.037289734929800034,"kl":0.1005149558186531,"entropy":0.4975813329219818},{"step":380,"reward":-1.709739089012146,"core":-0.4769248068332672,"cost":-0.12644799053668976,"kl":0.09877669624984264,"entropy":0.546432763338089},{"step":381,"reward":-0.7034425735473633,"core":-0.10618770867586136,"cost":-0.016062472015619278,"kl":0.08017020300030708,"entropy":0.49395910650491714},{"step":382,"reward":-0.12725181877613068,"core":0.14895416796207428,"cost":-0.030068740248680115,"kl":0.08017922006547451,"entropy":0.5193340256810188},{"step":383,"reward":-1.472910761833191,"core":-0.3977006673812866,"cost":-0.06416189670562744,"kl":0.0899193286895752,"entropy":0.5886830240488052},{"step":384,"reward":-0.4836517870426178,"core":-0.07598940283060074,"cost":0.0,"kl":0.061961231753230095,"entropy":0.3798609524965286},{"step":385,"reward":-1.4084991216659546,"core":-0.4087400436401367,"cost":-0.08526363223791122,"kl":0.07794580422341824,"entropy":0.533007487654686},{"step":386,"reward":-0.6966979503631592,"core":-0.06617653369903564,"cost":-0.052894461899995804,"kl":0.07911147829145193,"entropy":0.4692842587828636},{"step":387,"reward":-1.3555234670639038,"core":-0.35012635588645935,"cost":-0.05725269392132759,"kl":0.09854794014245272,"entropy":0.4822516366839409},{"step":388,"reward":-1.4055761098861694,"core":-0.41386470198631287,"cost":-0.017885083332657814,"kl":0.0819711359217763,"entropy":0.4744168817996979},{"step":389,"reward":-0.585353672504425,"core":-0.12169010192155838,"cost":-0.042788952589035034,"kl":0.08348524756729603,"entropy":0.5462234616279602},{"step":390,"reward":-0.6843269467353821,"core":-0.09720313549041748,"cost":-0.019172795116901398,"kl":0.1158656757324934,"entropy":0.4814569056034088},{"step":391,"reward":-0.8649027347564697,"core":-0.2098599076271057,"cost":-0.03412047401070595,"kl":0.07223558612167835,"entropy":0.5176068544387817},{"step":392,"reward":-2.3840131759643555,"core":-0.7470515370368958,"cost":-0.05972461774945259,"kl":0.0673898896202445,"entropy":0.5303749516606331},{"step":393,"reward":-0.08782254904508591,"core":0.13639777898788452,"cost":0.0,"kl":0.18237152881920338,"entropy":0.6222255676984787},{"step":394,"reward":-0.6936829090118408,"core":-0.1317732334136963,"cost":-0.0319165401160717,"kl":0.08888813853263855,"entropy":0.554283618927002},{"step":395,"reward":-1.3868188858032227,"core":-0.35442793369293213,"cost":-0.0999458059668541,"kl":0.06356312334537506,"entropy":0.4784036725759506},{"step":396,"reward":-0.8111495971679688,"core":-0.12908871471881866,"cost":-0.046533942222595215,"kl":0.08666159212589264,"entropy":0.45250875502824783},{"step":397,"reward":-1.0982249975204468,"core":-0.2822435796260834,"cost":-0.021335439756512642,"kl":0.07982211001217365,"entropy":0.5550811439752579},{"step":398,"reward":-1.4588419198989868,"core":-0.42132869362831116,"cost":-0.011706027202308178,"kl":0.09316875878721476,"entropy":0.5243427753448486},{"step":399,"reward":-1.0224418640136719,"core":-0.2919802963733673,"cost":-0.019390353932976723,"kl":0.0703121880069375,"entropy":0.4697185605764389},{"step":400,"reward":-0.7874175906181335,"core":-0.16512256860733032,"cost":-0.0506010502576828,"kl":0.08913809806108475,"entropy":0.514363057911396},{"step":401,"reward":-2.6439154148101807,"core":-0.8217064738273621,"cost":-0.10467791557312012,"kl":0.12025666795670986,"entropy":0.6026499718427658},{"step":402,"reward":-1.1845476627349854,"core":-0.17677398025989532,"cost":-0.20031118392944336,"kl":0.08052713982760906,"entropy":0.5269459187984467},{"step":403,"reward":-1.827375054359436,"core":-0.558358371257782,"cost":-0.029869742691516876,"kl":0.07348810229450464,"entropy":0.5191583186388016},{"step":404,"reward":-3.0830955505371094,"core":-0.961545467376709,"cost":-0.13077150285243988,"kl":0.06798488646745682,"entropy":0.5515476912260056},{"step":405,"reward":-0.24326114356517792,"core":0.11145371198654175,"cost":0.0,"kl":0.09096886590123177,"entropy":0.5613632202148438},{"step":406,"reward":-0.6606130599975586,"core":-0.12943421304225922,"cost":-0.0016524008242413402,"kl":0.09002817794680595,"entropy":0.5401446223258972},{"step":407,"reward":-2.2732911109924316,"core":-0.6565282940864563,"cost":-0.1383390575647354,"kl":0.09014795906841755,"entropy":0.49135827273130417},{"step":408,"reward":-0.9379045367240906,"core":-0.23410440981388092,"cost":-0.04917153716087341,"kl":0.13624281622469425,"entropy":0.576189786195755},{"step":409,"reward":-2.0651559829711914,"core":-0.6262404918670654,"cost":-0.07723600417375565,"kl":0.08033493719995022,"entropy":0.5402515530586243},{"step":410,"reward":-1.090627670288086,"core":-0.1917133927345276,"cost":-0.07123978435993195,"kl":0.0835893303155899,"entropy":0.49636316299438477},{"step":411,"reward":-0.8671180605888367,"core":-0.18339787423610687,"cost":-0.1036686822772026,"kl":0.09927078895270824,"entropy":0.483444020152092},{"step":412,"reward":-1.661842942237854,"core":-0.4723902642726898,"cost":-0.06309909373521805,"kl":0.07973700854927301,"entropy":0.5928179621696472},{"step":413,"reward":-0.2658195197582245,"core":-0.00332312542013824,"cost":-0.03213632106781006,"kl":0.14023976773023605,"entropy":0.5113681629300117},{"step":414,"reward":-1.984406590461731,"core":-0.5800034403800964,"cost":-0.045193497091531754,"kl":0.11304044537246227,"entropy":0.5583366453647614},{"step":415,"reward":-2.8297348022460938,"core":-0.8496506214141846,"cost":-0.2382921725511551,"kl":0.08107459358870983,"entropy":0.5736142992973328},{"step":416,"reward":-1.090377926826477,"core":-0.23974664509296417,"cost":-0.07682517170906067,"kl":0.10558601096272469,"entropy":0.5873842835426331},{"step":417,"reward":-0.8805468082427979,"core":-0.16325287520885468,"cost":-0.02892971970140934,"kl":0.12072544544935226,"entropy":0.5467613786458969},{"step":418,"reward":-1.5902007818222046,"core":-0.4154721200466156,"cost":-0.1026231124997139,"kl":0.08066903613507748,"entropy":0.504797026515007},{"step":419,"reward":-1.8267227411270142,"core":-0.5595319867134094,"cost":-0.029260428622364998,"kl":0.07314048148691654,"entropy":0.5119954571127892},{"step":420,"reward":-1.796628475189209,"core":-0.5066750645637512,"cost":-0.10747355222702026,"kl":0.10742738842964172,"entropy":0.5640438795089722},{"step":421,"reward":-1.7096725702285767,"core":-0.5054572224617004,"cost":-0.0333893746137619,"kl":0.06561709567904472,"entropy":0.4515727609395981},{"step":422,"reward":-1.007568597793579,"core":-0.2680448591709137,"cost":-0.026428116485476494,"kl":0.07200808171182871,"entropy":0.5076609402894974},{"step":423,"reward":0.03849155828356743,"core":0.2089468240737915,"cost":0.0,"kl":0.05806966871023178,"entropy":0.449109710752964},{"step":424,"reward":-2.2328898906707764,"core":-0.7015175819396973,"cost":-0.05518549680709839,"kl":0.04206643719226122,"entropy":0.45153599232435226},{"step":425,"reward":-1.2837042808532715,"core":-0.28576043248176575,"cost":-0.11113431304693222,"kl":0.08742711134254932,"entropy":0.46914640814065933},{"step":426,"reward":-1.4247459173202515,"core":-0.4184138774871826,"cost":-0.030544660985469818,"kl":0.07239756174385548,"entropy":0.6132333725690842},{"step":427,"reward":-1.2688089609146118,"core":-0.3585294485092163,"cost":-0.025332463905215263,"kl":0.07283450942486525,"entropy":0.6146295964717865},{"step":428,"reward":-1.6660579442977905,"core":-0.44876912236213684,"cost":-0.13499058783054352,"kl":0.07249599508941174,"entropy":0.40310338884592056},{"step":429,"reward":-0.3946651518344879,"core":0.01883436180651188,"cost":0.0,"kl":0.092611039057374,"entropy":0.4639754816889763},{"step":430,"reward":-0.8521277904510498,"core":-0.2289568930864334,"cost":-0.02350912056863308,"kl":0.08276729099452496,"entropy":0.5863946676254272},{"step":431,"reward":-1.3750358819961548,"core":-0.34572675824165344,"cost":-0.05960483476519585,"kl":0.07351395208388567,"entropy":0.5530588030815125},{"step":432,"reward":-0.8080897331237793,"core":-0.16147302091121674,"cost":-0.03199506178498268,"kl":0.06522161047905684,"entropy":0.5301886051893234},{"step":433,"reward":-1.0915071964263916,"core":-0.23608337342739105,"cost":-0.03709736093878746,"kl":0.06248121336102486,"entropy":0.4620179235935211},{"step":434,"reward":-2.7746334075927734,"core":-0.833275318145752,"cost":-0.200453519821167,"kl":0.08458259142935276,"entropy":0.5333742648363113},{"step":435,"reward":-1.305037260055542,"core":-0.33032992482185364,"cost":-0.04943567514419556,"kl":0.057829925790429115,"entropy":0.5329144150018692},{"step":436,"reward":-1.5439707040786743,"core":-0.3978423774242401,"cost":-0.08922415226697922,"kl":0.056552265770733356,"entropy":0.4785086140036583},{"step":437,"reward":-0.7791640162467957,"core":-0.15680496394634247,"cost":0.0,"kl":0.07627368345856667,"entropy":0.6178846955299377},{"step":438,"reward":-2.037580728530884,"core":-0.5932562947273254,"cost":-0.12811708450317383,"kl":0.08319388702511787,"entropy":0.5711559504270554},{"step":439,"reward":-1.6447696685791016,"core":-0.3625471591949463,"cost":-0.20990979671478271,"kl":0.0928691029548645,"entropy":0.5712525844573975},{"step":440,"reward":-0.010237743146717548,"core":0.12407609820365906,"cost":-0.02421599067747593,"kl":0.1215341854840517,"entropy":0.49311093240976334},{"step":441,"reward":-1.5276541709899902,"core":-0.3991645872592926,"cost":-0.03729599341750145,"kl":0.07858974486589432,"entropy":0.5628267079591751},{"step":442,"reward":-1.6973119974136353,"core":-0.4668750762939453,"cost":-0.13348902761936188,"kl":0.054049122147262096,"entropy":0.6179219186306},{"step":443,"reward":-0.9248718619346619,"core":-0.24881957471370697,"cost":-0.030509531497955322,"kl":0.07880420424044132,"entropy":0.5673758387565613},{"step":444,"reward":-2.552079916000366,"core":-0.6619558930397034,"cost":-0.3114722669124603,"kl":0.09837506897747517,"entropy":0.486754834651947},{"step":445,"reward":-0.523815393447876,"core":-0.07573732733726501,"cost":-0.011078392155468464,"kl":0.08407656103372574,"entropy":0.5064221397042274},{"step":446,"reward":-2.8190574645996094,"core":-0.8334441184997559,"cost":-0.2302999049425125,"kl":0.07252869196236134,"entropy":0.5701417624950409},{"step":447,"reward":-2.9374234676361084,"core":-0.9090330600738525,"cost":-0.13927583396434784,"kl":0.05728909559547901,"entropy":0.4804355502128601},{"step":448,"reward":-0.28420934081077576,"core":0.0210068728774786,"cost":-0.01768592745065689,"kl":0.059587420895695686,"entropy":0.5818616896867752},{"step":449,"reward":-2.0189316272735596,"core":-0.5237966179847717,"cost":-0.20115363597869873,"kl":0.07153832074254751,"entropy":0.5139005407691002},{"step":450,"reward":-1.1245522499084473,"core":-0.2933352589607239,"cost":-0.04571206867694855,"kl":0.06209997460246086,"entropy":0.4810241311788559},{"step":451,"reward":-1.0109925270080566,"core":-0.18835027515888214,"cost":-0.025579648092389107,"kl":0.07547716796398163,"entropy":0.5031109005212784},{"step":452,"reward":-0.8763143420219421,"core":-0.2062111645936966,"cost":-0.011146929115056992,"kl":0.060865587554872036,"entropy":0.5357615873217583},{"step":453,"reward":-0.5893561244010925,"core":-0.09371340274810791,"cost":-0.01115157175809145,"kl":0.07201450690627098,"entropy":0.5235129818320274},{"step":454,"reward":-1.1314820051193237,"core":-0.31610116362571716,"cost":0.0,"kl":0.05941990856081247,"entropy":0.5556639432907104},{"step":455,"reward":-1.7143354415893555,"core":-0.5136303901672363,"cost":-0.047000203281641006,"kl":0.049106343649327755,"entropy":0.5301478356122971},{"step":456,"reward":-3.092411756515503,"core":-0.8635108470916748,"cost":-0.2929009199142456,"kl":0.07795786019414663,"entropy":0.5254295021295547},{"step":457,"reward":-0.7553086876869202,"core":-0.12112373113632202,"cost":-0.08358266949653625,"kl":0.09181065298616886,"entropy":0.4963580518960953},{"step":458,"reward":-0.29382166266441345,"core":0.06467989832162857,"cost":-0.0021604602225124836,"kl":0.06602640729397535,"entropy":0.4844232127070427},{"step":459,"reward":-1.7103263139724731,"core":-0.4770181179046631,"cost":-0.12524139881134033,"kl":0.0710251983255148,"entropy":0.5762492269277573},{"step":460,"reward":-1.4865692853927612,"core":-0.42765215039253235,"cost":-0.06524747610092163,"kl":0.078890735283494,"entropy":0.5476478934288025},{"step":461,"reward":-1.541072964668274,"core":-0.40476706624031067,"cost":-0.052733033895492554,"kl":0.036370244808495045,"entropy":0.566019594669342},{"step":462,"reward":-0.978658139705658,"core":-0.21984195709228516,"cost":-0.11688875406980515,"kl":0.10186219215393066,"entropy":0.5741131007671356},{"step":463,"reward":-1.6060247421264648,"core":-0.4887784421443939,"cost":-0.021989963948726654,"kl":0.04466073028743267,"entropy":0.48934880644083023},{"step":464,"reward":-0.9239484667778015,"core":-0.2360781878232956,"cost":-0.010378665290772915,"kl":0.05902272742241621,"entropy":0.5830855518579483},{"step":465,"reward":-0.7374565005302429,"core":-0.08054938167333603,"cost":-0.03206774219870567,"kl":0.07827969454228878,"entropy":0.5070541054010391},{"step":466,"reward":-2.171449899673462,"core":-0.6086061000823975,"cost":-0.16665154695510864,"kl":0.06769376527518034,"entropy":0.5076169446110725},{"step":467,"reward":-1.9912227392196655,"core":-0.5668753981590271,"cost":-0.0913272276520729,"kl":0.054779606871306896,"entropy":0.4868171960115433},{"step":468,"reward":-1.749725341796875,"core":-0.46903088688850403,"cost":-0.1226278766989708,"kl":0.08692214358597994,"entropy":0.5186139941215515},{"step":469,"reward":-0.8191153407096863,"core":-0.1596006602048874,"cost":-0.021664753556251526,"kl":0.0712883211672306,"entropy":0.5152203887701035},{"step":470,"reward":-1.2872533798217773,"core":-0.32607269287109375,"cost":-0.04612100124359131,"kl":0.05952595733106136,"entropy":0.5464275255799294},{"step":471,"reward":-1.4023054838180542,"core":-0.35654816031455994,"cost":-0.03304162248969078,"kl":0.06332204677164555,"entropy":0.4511486887931824},{"step":472,"reward":-0.8237060904502869,"core":-0.08105248212814331,"cost":-0.09326977282762527,"kl":0.08749681152403355,"entropy":0.54448102414608},{"step":473,"reward":-1.3665355443954468,"core":-0.38221216201782227,"cost":-0.05161513388156891,"kl":0.06458488013595343,"entropy":0.547236904501915},{"step":474,"reward":-1.2151867151260376,"core":-0.31761643290519714,"cost":-0.0304106455296278,"kl":0.06490770634263754,"entropy":0.48715055733919144},{"step":475,"reward":-0.21087606251239777,"core":0.05084845423698425,"cost":0.0,"kl":0.09463405050337315,"entropy":0.5175473764538765},{"step":476,"reward":-2.8182709217071533,"core":-0.8117174506187439,"cost":-0.2771652042865753,"kl":0.08776314742863178,"entropy":0.5799212157726288},{"step":477,"reward":-0.5310723185539246,"core":-0.037369776517152786,"cost":-0.05643855407834053,"kl":0.08958579413592815,"entropy":0.51662328094244},{"step":478,"reward":-2.401939630508423,"core":-0.7559680938720703,"cost":-0.05706368759274483,"kl":0.058063194155693054,"entropy":0.6017701774835587},{"step":479,"reward":-0.38098421692848206,"core":0.00881357491016388,"cost":-0.005457345396280289,"kl":0.055985864251852036,"entropy":0.4919666200876236},{"step":480,"reward":-1.98893404006958,"core":-0.5130638480186462,"cost":-0.1317358762025833,"kl":0.10946862865239382,"entropy":0.4914521649479866},{"step":481,"reward":-1.4610233306884766,"core":-0.32058703899383545,"cost":-0.07327578216791153,"kl":0.11978397145867348,"entropy":0.5285719931125641},{"step":482,"reward":-1.7487601041793823,"core":-0.5051238536834717,"cost":-0.049002159386873245,"kl":0.07790138013660908,"entropy":0.5451806038618088},{"step":483,"reward":-1.4715975522994995,"core":-0.4285382032394409,"cost":-0.011126396246254444,"kl":0.057659913785755634,"entropy":0.44906413555145264},{"step":484,"reward":-0.977393627166748,"core":-0.2615779936313629,"cost":-0.014711341820657253,"kl":0.08096565213054419,"entropy":0.5490427911281586},{"step":485,"reward":-1.52996826171875,"core":-0.4219801425933838,"cost":-0.07863032817840576,"kl":0.07927128113806248,"entropy":0.6156946122646332},{"step":486,"reward":-0.7800566554069519,"core":-0.14452801644802094,"cost":0.0,"kl":0.057804470881819725,"entropy":0.546047255396843},{"step":487,"reward":-1.15471351146698,"core":-0.2815002501010895,"cost":-0.03809503838419914,"kl":0.08150635287165642,"entropy":0.513124905526638},{"step":488,"reward":-0.9053339958190918,"core":-0.11406585574150085,"cost":-0.04157894104719162,"kl":0.06254110625013709,"entropy":0.5579905211925507},{"step":489,"reward":-2.1289141178131104,"core":-0.6681761741638184,"cost":-0.04558178782463074,"kl":0.06659696530550718,"entropy":0.5810939520597458},{"step":490,"reward":-1.7361688613891602,"core":-0.5263201594352722,"cost":-0.07526200264692307,"kl":0.07011689338833094,"entropy":0.5884190425276756},{"step":491,"reward":-1.328154444694519,"core":-0.3121413290500641,"cost":-0.08238967508077621,"kl":0.07130437344312668,"entropy":0.6208921670913696},{"step":492,"reward":-2.2341954708099365,"core":-0.6614070534706116,"cost":-0.16783170402050018,"kl":0.051517317071557045,"entropy":0.5825081765651703},{"step":493,"reward":-2.0543630123138428,"core":-0.6148165464401245,"cost":-0.13727092742919922,"kl":0.05774580594152212,"entropy":0.5543619841337204},{"step":494,"reward":-0.16721107065677643,"core":0.03847856819629669,"cost":-0.016689425334334373,"kl":0.05589038971811533,"entropy":0.5427295863628387},{"step":495,"reward":-1.4878616333007812,"core":-0.4334143102169037,"cost":-0.06037580966949463,"kl":0.05303787812590599,"entropy":0.5844718962907791},{"step":496,"reward":-1.6169523000717163,"core":-0.47505465149879456,"cost":-0.07835555821657181,"kl":0.0453390059992671,"entropy":0.48531200736761093},{"step":497,"reward":-1.6475614309310913,"core":-0.49920210242271423,"cost":-0.031992923468351364,"kl":0.05697830580174923,"entropy":0.55279441177845},{"step":498,"reward":-1.3263095617294312,"core":-0.3991231918334961,"cost":-0.035929176956415176,"kl":0.065080382861197,"entropy":0.5415823757648468},{"step":499,"reward":-2.29797101020813,"core":-0.6802255511283875,"cost":-0.13615499436855316,"kl":0.05802634637802839,"entropy":0.5563626289367676},{"step":500,"reward":-2.628493547439575,"core":-0.7132031917572021,"cost":-0.25951334834098816,"kl":0.06544632092118263,"entropy":0.6110702008008957},{"step":501,"reward":-1.2631946802139282,"core":-0.34037843346595764,"cost":-0.039263006299734116,"kl":0.07647918164730072,"entropy":0.570259228348732},{"step":502,"reward":-1.0183106660842896,"core":-0.2591184675693512,"cost":0.0,"kl":0.04612663295120001,"entropy":0.5062588825821877},{"step":503,"reward":-1.7323726415634155,"core":-0.4763925075531006,"cost":-0.05427693948149681,"kl":0.06345029175281525,"entropy":0.5238076597452164},{"step":504,"reward":-0.37609902024269104,"core":0.012633562088012695,"cost":-0.04294649884104729,"kl":0.08257243037223816,"entropy":0.534516230225563},{"step":505,"reward":-2.0335986614227295,"core":-0.6278373599052429,"cost":-0.0772792175412178,"kl":0.06555342208594084,"entropy":0.5313642546534538},{"step":506,"reward":-2.043566942214966,"core":-0.6025710105895996,"cost":-0.12692315876483917,"kl":0.09450450912117958,"entropy":0.6055238842964172},{"step":507,"reward":-1.767969012260437,"core":-0.525467038154602,"cost":-0.08289416879415512,"kl":0.05263539869338274,"entropy":0.5279589965939522},{"step":508,"reward":-1.8415099382400513,"core":-0.5054388642311096,"cost":-0.1308947205543518,"kl":0.06156019400805235,"entropy":0.5527131110429764},{"step":509,"reward":-0.9226898550987244,"core":-0.18060417473316193,"cost":-0.030418751761317253,"kl":0.08507492952048779,"entropy":0.5667644888162613},{"step":510,"reward":-0.3286018669605255,"core":0.007163832429796457,"cost":-0.043810829520225525,"kl":0.08793786354362965,"entropy":0.48469826579093933},{"step":511,"reward":-1.08800208568573,"core":-0.2227010726928711,"cost":-0.03186611086130142,"kl":0.07035164721310139,"entropy":0.5711694955825806},{"step":512,"reward":-0.4294019937515259,"core":-0.060631800442934036,"cost":-0.007321231532841921,"kl":0.06562915910035372,"entropy":0.564895823597908},{"step":513,"reward":0.1325002908706665,"core":0.20287789404392242,"cost":-0.028121890500187874,"kl":0.08432150445878506,"entropy":0.5163640975952148},{"step":514,"reward":-1.7772706747055054,"core":-0.46345511078834534,"cost":-0.11669174581766129,"kl":0.10187131725251675,"entropy":0.5900004953145981},{"step":515,"reward":-1.3654308319091797,"core":-0.3501357138156891,"cost":-0.09572430700063705,"kl":0.07670443691313267,"entropy":0.5258008986711502},{"step":516,"reward":-1.5848156213760376,"core":-0.3774667978286743,"cost":-0.13777120411396027,"kl":0.07216867431998253,"entropy":0.553843080997467},{"step":517,"reward":-1.1490355730056763,"core":-0.31176993250846863,"cost":-0.020601557567715645,"kl":0.08249076921492815,"entropy":0.5318500250577927},{"step":518,"reward":-1.307512879371643,"core":-0.34698522090911865,"cost":-0.05916318669915199,"kl":0.11530423536896706,"entropy":0.541835680603981},{"step":519,"reward":-1.560092568397522,"core":-0.4543665945529938,"cost":-0.03809358552098274,"kl":0.04798890836536884,"entropy":0.543815016746521},{"step":520,"reward":-1.6847076416015625,"core":-0.49880146980285645,"cost":-0.025951765477657318,"kl":0.07083006389439106,"entropy":0.5343438535928726},{"step":521,"reward":-1.5784631967544556,"core":-0.45983681082725525,"cost":-0.05034749209880829,"kl":0.0691624479368329,"entropy":0.5143636018037796},{"step":522,"reward":-2.403903007507324,"core":-0.7075209617614746,"cost":-0.18266141414642334,"kl":0.06286666169762611,"entropy":0.4607674777507782},{"step":523,"reward":-0.8778241276741028,"core":-0.21399252116680145,"cost":-0.03352685645222664,"kl":0.05544182565063238,"entropy":0.5490670651197433},{"step":524,"reward":-1.3967819213867188,"core":-0.3983306884765625,"cost":-0.09529919177293777,"kl":0.08868276886641979,"entropy":0.5883709341287613},{"step":525,"reward":-0.6051799654960632,"core":-0.09796881675720215,"cost":-0.012832019478082657,"kl":0.0934242345392704,"entropy":0.4925842136144638},{"step":526,"reward":-1.7083983421325684,"core":-0.43878355622291565,"cost":-0.10778429359197617,"kl":0.06793271377682686,"entropy":0.48733381927013397},{"step":527,"reward":-0.7216865420341492,"core":-0.15150219202041626,"cost":-0.00734354555606842,"kl":0.059661117382347584,"entropy":0.48890361934900284},{"step":528,"reward":-1.7885271310806274,"core":-0.5233430862426758,"cost":-0.10971995443105698,"kl":0.0677706291899085,"entropy":0.5431029051542282},{"step":529,"reward":-1.7029290199279785,"core":-0.531741201877594,"cost":-0.017113154754042625,"kl":0.06390819977968931,"entropy":0.3941689804196358},{"step":530,"reward":-2.0837020874023438,"core":-0.5932409167289734,"cost":-0.1608661562204361,"kl":0.0757842380553484,"entropy":0.515081025660038},{"step":531,"reward":-0.5159663558006287,"core":-0.10883752256631851,"cost":-0.015350793488323689,"kl":0.08600704558193684,"entropy":0.5700702667236328},{"step":532,"reward":-1.3181885480880737,"core":-0.3426623046398163,"cost":-0.0966290757060051,"kl":0.06504646874964237,"entropy":0.495039239525795},{"step":533,"reward":-0.3468639850616455,"core":-0.012240871787071228,"cost":-0.014906536787748337,"kl":0.07962299138307571,"entropy":0.5718352347612381},{"step":534,"reward":-1.4881749153137207,"core":-0.4095593988895416,"cost":-0.018610656261444092,"kl":0.06775310635566711,"entropy":0.5695811957120895},{"step":535,"reward":-1.5684460401535034,"core":-0.3321523666381836,"cost":-0.2341863065958023,"kl":0.09168932028114796,"entropy":0.49304722249507904},{"step":536,"reward":-1.799166202545166,"core":-0.5531226992607117,"cost":-0.05781838670372963,"kl":0.07783273980021477,"entropy":0.49079157412052155},{"step":537,"reward":-0.10302704572677612,"core":0.10145614296197891,"cost":-0.014467552304267883,"kl":0.07435549888759851,"entropy":0.47789230942726135},{"step":538,"reward":0.16094815731048584,"core":0.2527301609516144,"cost":-0.03441309556365013,"kl":0.06662212684750557,"entropy":0.5317956060171127},{"step":539,"reward":-2.099264144897461,"core":-0.6366586089134216,"cost":-0.055362943559885025,"kl":0.09030701033771038,"entropy":0.5925275832414627},{"step":540,"reward":-0.9016494750976562,"core":-0.14185011386871338,"cost":-0.06353303790092468,"kl":0.08351518400013447,"entropy":0.5111784711480141},{"step":541,"reward":-0.052912160754203796,"core":0.11262432485818863,"cost":-0.014418520964682102,"kl":0.07152458559721708,"entropy":0.5233386233448982},{"step":542,"reward":-1.2199236154556274,"core":-0.3050987720489502,"cost":-0.06766154617071152,"kl":0.09947543032467365,"entropy":0.5494555234909058},{"step":543,"reward":-0.6244180798530579,"core":-0.08548200130462646,"cost":-0.01652744598686695,"kl":0.09998384863138199,"entropy":0.5184425190091133},{"step":544,"reward":-2.2419321537017822,"core":-0.6615628600120544,"cost":-0.07081414014101028,"kl":0.06260378286242485,"entropy":0.5321355313062668},{"step":545,"reward":-2.769638776779175,"core":-0.790003776550293,"cost":-0.19514454901218414,"kl":0.0686387550085783,"entropy":0.6154752969741821},{"step":546,"reward":-0.6757280826568604,"core":-0.01179556455463171,"cost":-0.07774031907320023,"kl":0.07282464485615492,"entropy":0.48503342270851135},{"step":547,"reward":-0.16941814124584198,"core":0.11341450363397598,"cost":0.0,"kl":0.07588279992341995,"entropy":0.5844723582267761},{"step":548,"reward":-1.1729093790054321,"core":-0.3200007379055023,"cost":-0.0037878069560974836,"kl":0.05525184981524944,"entropy":0.5194242745637894},{"step":549,"reward":-1.799087643623352,"core":-0.5094682574272156,"cost":-0.10043304413557053,"kl":0.04932946152985096,"entropy":0.49027782678604126},{"step":550,"reward":-0.7636216282844543,"core":-0.10687028616666794,"cost":-0.023357145488262177,"kl":0.06398242991417646,"entropy":0.5134144574403763},{"step":551,"reward":-1.248258113861084,"core":-0.32107236981391907,"cost":-0.16016797721385956,"kl":0.08555192686617374,"entropy":0.5109900608658791},{"step":552,"reward":-1.9116896390914917,"core":-0.46583375334739685,"cost":-0.21883954107761383,"kl":0.08899137936532497,"entropy":0.5754934847354889},{"step":553,"reward":-0.6954910159111023,"core":-0.14642010629177094,"cost":0.0,"kl":0.1107892356812954,"entropy":0.5852017849683762},{"step":554,"reward":-0.8611621856689453,"core":-0.16565169394016266,"cost":-0.02754085324704647,"kl":0.08677368424832821,"entropy":0.5487206876277924},{"step":555,"reward":-1.8458245992660522,"core":-0.5263435244560242,"cost":-0.11640366911888123,"kl":0.08304722234606743,"entropy":0.49291346967220306},{"step":556,"reward":-1.0283104181289673,"core":-0.2770576477050781,"cost":-0.004925295244902372,"kl":0.06328535173088312,"entropy":0.5894822329282761},{"step":557,"reward":-1.7741092443466187,"core":-0.509499728679657,"cost":-0.11899730563163757,"kl":0.09315475262701511,"entropy":0.5337792932987213},{"step":558,"reward":-0.13853155076503754,"core":0.10628651827573776,"cost":-0.05750586465001106,"kl":0.09819017630070448,"entropy":0.5158702880144119},{"step":559,"reward":-2.702512502670288,"core":-0.8267782330513,"cost":-0.1347266286611557,"kl":0.04282485041767359,"entropy":0.45628612488508224},{"step":560,"reward":-2.8302135467529297,"core":-0.8566675186157227,"cost":-0.19280298054218292,"kl":0.0437607541680336,"entropy":0.525649406015873},{"step":561,"reward":-1.3836568593978882,"core":-0.389789342880249,"cost":-0.06244460120797157,"kl":0.05055796913802624,"entropy":0.48895205557346344},{"step":562,"reward":-0.9413502812385559,"core":-0.207331582903862,"cost":-0.04831201210618019,"kl":0.077079595066607,"entropy":0.5724730938673019},{"step":563,"reward":-1.852953314781189,"core":-0.5352061986923218,"cost":-0.020391106605529785,"kl":0.07761191949248314,"entropy":0.5627006143331528},{"step":564,"reward":-1.6244487762451172,"core":-0.46392807364463806,"cost":-0.03985753655433655,"kl":0.08659128658473492,"entropy":0.486959345638752},{"step":565,"reward":-0.9359516501426697,"core":-0.2074543684720993,"cost":-0.06078404560685158,"kl":0.06077786721289158,"entropy":0.5189192742109299},{"step":566,"reward":-0.6407301425933838,"core":-0.1319064348936081,"cost":0.0,"kl":0.05645353626459837,"entropy":0.5746306777000427},{"step":567,"reward":-0.13608574867248535,"core":0.1009124219417572,"cost":-0.023265354335308075,"kl":0.11267323046922684,"entropy":0.4867963641881943},{"step":568,"reward":-1.8257060050964355,"core":-0.5608975291252136,"cost":-0.0424874983727932,"kl":0.0618644654750824,"entropy":0.6046473234891891},{"step":569,"reward":-2.219236373901367,"core":-0.6970908641815186,"cost":-0.029222100973129272,"kl":0.08376096747815609,"entropy":0.5187279284000397},{"step":570,"reward":-2.1061177253723145,"core":-0.6119030117988586,"cost":-0.11269140243530273,"kl":0.06880614720284939,"entropy":0.5070173889398575},{"step":571,"reward":-0.9121264815330505,"core":-0.1696615368127823,"cost":-0.01818615384399891,"kl":0.0841225516051054,"entropy":0.516069196164608},{"step":572,"reward":-0.6068680286407471,"core":-0.06901592761278152,"cost":-0.026982827112078667,"kl":0.057910770177841187,"entropy":0.48218535631895065},{"step":573,"reward":-1.8593670129776,"core":-0.5561381578445435,"cost":-0.07995810359716415,"kl":0.06352472491562366,"entropy":0.4908350929617882},{"step":574,"reward":-1.5821040868759155,"core":-0.46643170714378357,"cost":-0.023196307942271233,"kl":0.038727994076907635,"entropy":0.40908506512641907},{"step":575,"reward":-2.5805327892303467,"core":-0.8126263618469238,"cost":-0.07490948587656021,"kl":0.05018948670476675,"entropy":0.4167659059166908},{"step":576,"reward":-2.2580296993255615,"core":-0.6899779438972473,"cost":-0.10846269130706787,"kl":0.07845878228545189,"entropy":0.5210245326161385},{"step":577,"reward":-0.8930268883705139,"core":-0.23903973400592804,"cost":-0.031173765659332275,"kl":0.08049461618065834,"entropy":0.5253824889659882},{"step":578,"reward":-0.23467665910720825,"core":0.0828772559762001,"cost":0.0,"kl":0.06607557833194733,"entropy":0.4915064126253128},{"step":579,"reward":-1.9863799810409546,"core":-0.591733992099762,"cost":-0.05067775025963783,"kl":0.07712114043533802,"entropy":0.5319293886423111},{"step":580,"reward":-0.0026199370622634888,"core":0.1284419149160385,"cost":-0.008745518513023853,"kl":0.06863124947994947,"entropy":0.47144615650177},{"step":581,"reward":-2.563647985458374,"core":-0.8051084876060486,"cost":-0.10175570100545883,"kl":0.05291796289384365,"entropy":0.4395769089460373},{"step":582,"reward":-1.4413918256759644,"core":-0.37637242674827576,"cost":-0.06992950290441513,"kl":0.0633945045992732,"entropy":0.5123835802078247},{"step":583,"reward":-2.184713125228882,"core":-0.6590937972068787,"cost":-0.11874926090240479,"kl":0.09745602123439312,"entropy":0.5763951539993286},{"step":584,"reward":-1.8566021919250488,"core":-0.511905312538147,"cost":-0.10787408798933029,"kl":0.06079707480967045,"entropy":0.4697476923465729},{"step":585,"reward":-1.0598397254943848,"core":-0.2924517095088959,"cost":0.0,"kl":0.06526198145002127,"entropy":0.4629756137728691},{"step":586,"reward":-0.9050093293190002,"core":-0.20029999315738678,"cost":-0.017153145745396614,"kl":0.06506290100514889,"entropy":0.5196897089481354},{"step":587,"reward":-0.978492259979248,"core":-0.18626034259796143,"cost":-0.04079129174351692,"kl":0.0835481584072113,"entropy":0.5060087144374847},{"step":588,"reward":-1.4432315826416016,"core":-0.26804980635643005,"cost":-0.15034131705760956,"kl":0.07726935856044292,"entropy":0.5583579242229462},{"step":589,"reward":-1.7671399116516113,"core":-0.5399807095527649,"cost":-0.021083420142531395,"kl":0.06314197275787592,"entropy":0.49436740577220917},{"step":590,"reward":-1.274682641029358,"core":-0.33938297629356384,"cost":-0.031045757234096527,"kl":0.09427970089018345,"entropy":0.5373968929052353},{"step":591,"reward":-1.1536208391189575,"core":-0.2765507698059082,"cost":-0.016113687306642532,"kl":0.07410974055528641,"entropy":0.49108101427555084},{"step":592,"reward":-2.359748125076294,"core":-0.6762627959251404,"cost":-0.12279310077428818,"kl":0.090096615254879,"entropy":0.4751279726624489},{"step":593,"reward":-0.36657118797302246,"core":0.030824458226561546,"cost":0.0,"kl":0.07093353755772114,"entropy":0.5614902675151825},{"step":594,"reward":-2.8238255977630615,"core":-0.848198413848877,"cost":-0.13500593602657318,"kl":0.040494476445019245,"entropy":0.4321022480726242},{"step":595,"reward":-1.4454641342163086,"core":-0.42680224776268005,"cost":-0.06300501525402069,"kl":0.07527728285640478,"entropy":0.44472262263298035},{"step":596,"reward":-2.020951986312866,"core":-0.5517826080322266,"cost":-0.13713203370571136,"kl":0.07465282082557678,"entropy":0.6079055517911911},{"step":597,"reward":-0.7763527035713196,"core":-0.16412024199962616,"cost":-0.027602558955550194,"kl":0.08793911896646023,"entropy":0.5384587049484253},{"step":598,"reward":-1.464737057685852,"core":-0.4311300218105316,"cost":-0.0590900294482708,"kl":0.06822375860065222,"entropy":0.5425691455602646},{"step":599,"reward":-2.2052762508392334,"core":-0.6394557356834412,"cost":-0.19678430259227753,"kl":0.06718053016811609,"entropy":0.5029816254973412},{"step":600,"reward":-1.1066670417785645,"core":-0.32007303833961487,"cost":-0.0350746214389801,"kl":0.0965172778815031,"entropy":0.5647031366825104}],"nonzero_advantage":[{"step":1,"nonzero":0.5416666666666666},{"step":3,"nonzero":0.2708333333333333},{"step":5,"nonzero":0.4375},{"step":7,"nonzero":0.125},{"step":9,"nonzero":0.0},{"step":11,"nonzero":0.5416666666666666},{"step":13,"nonzero":0.0},{"step":15,"nonzero":0.125},{"step":17,"nonzero":0.375},{"step":19,"nonzero":0.20833333333333334},{"step":21,"nonzero":0.4375},{"step":23,"nonzero":0.4791666666666667},{"step":25,"nonzero":0.3958333333333333},{"step":27,"nonzero":0.625},{"step":29,"nonzero":0.3958333333333333},{"step":31,"nonzero":0.3333333333333333},{"step":33,"nonzero":0.14583333333333334},{"step":35,"nonzero":0.1875},{"step":37,"nonzero":0.20833333333333334},{"step":39,"nonzero":0.5416666666666666},{"step":41,"nonzero":0.25},{"step":43,"nonzero":0.4166666666666667},{"step":45,"nonzero":0.5416666666666666},{"step":47,"nonzero":0.6458333333333334},{"step":49,"nonzero":0.125},{"step":51,"nonzero":0.4375},{"step":53,"nonzero":0.5833333333333334},{"step":55,"nonzero":0.3333333333333333},{"step":57,"nonzero":0.7708333333333334},{"step":59,"nonzero":0.4791666666666667},{"step":61,"nonzero":0.3958333333333333},{"step":63,"nonzero":0.5},{"step":65,"nonzero":0.5833333333333334},{"step":67,"nonzero":0.6666666666666666},{"step":69,"nonzero":0.4166666666666667},{"step":71,"nonzero":0.125},{"step":73,"nonzero":0.4791666666666667},{"step":75,"nonzero":0.6458333333333334},{"step":77,"nonzero":0.6458333333333334},{"step":79,"nonzero":0.6875},{"step":81,"nonzero":0.4166666666666667},{"step":83,"nonzero":0.6458333333333334},{"step":85,"nonzero":0.5},{"step":87,"nonzero":0.1875},{"step":89,"nonzero":0.5208333333333334},{"step":91,"nonzero":0.2916666666666667},{"step":93,"nonzero":0.4166666666666667},{"step":95,"nonzero":0.5416666666666666},{"step":97,"nonzero":0.8125},{"step":99,"nonzero":0.625},{"step":101,"nonzero":0.125},{"step":103,"nonzero":0.5833333333333334},{"step":105,"nonzero":0.625},{"step":107,"nonzero":0.375},{"step":109,"nonzero":0.5625},{"step":111,"nonzero":0.6875},{"step":113,"nonzero":0.3333333333333333},{"step":115,"nonzero":0.4375},{"step":117,"nonzero":0.8125},{"step":119,"nonzero":0.6458333333333334},{"step":121,"nonzero":0.3958333333333333},{"step":123,"nonzero":0.6875},{"step":125,"nonzero":0.10416666666666667},{"step":127,"nonzero":0.5208333333333334},{"step":129,"nonzero":0.6458333333333334},{"step":131,"nonzero":0.4583333333333333},{"step":133,"nonzero":0.25},{"step":135,"nonzero":0.4583333333333333},{"step":137,"nonzero":0.6458333333333334},{"step":139,"nonzero":0.5208333333333334},{"step":141,"nonzero":0.22916666666666666},{"step":143,"nonzero":0.7916666666666666},{"step":145,"nonzero":0.6458333333333334},{"step":147,"nonzero":0.5625},{"step":149,"nonzero":0.25},{"step":151,"nonzero":0.7708333333333334},{"step":153,"nonzero":0.4375},{"step":155,"nonzero":0.3333333333333333},{"step":157,"nonzero":0.5625},{"step":159,"nonzero":0.375},{"step":161,"nonzero":0.3958333333333333},{"step":163,"nonzero":0.5208333333333334},{"step":165,"nonzero":0.22916666666666666},{"step":167,"nonzero":0.7083333333333334},{"step":169,"nonzero":0.6875},{"step":171,"nonzero":0.4166666666666667},{"step":173,"nonzero":0.75},{"step":175,"nonzero":0.4375},{"step":177,"nonzero":0.625},{"step":179,"nonzero":0.4791666666666667},{"step":181,"nonzero":0.4375},{"step":183,"nonzero":0.6666666666666666},{"step":185,"nonzero":0.2916666666666667},{"step":187,"nonzero":0.5416666666666666},{"step":189,"nonzero":0.6041666666666666},{"step":191,"nonzero":0.5416666666666666},{"step":193,"nonzero":0.20833333333333334},{"step":195,"nonzero":0.6458333333333334},{"step":197,"nonzero":0.3333333333333333},{"step":199,"nonzero":0.875},{"step":201,"nonzero":0.5833333333333334},{"step":203,"nonzero":0.14583333333333334},{"step":205,"nonzero":0.7708333333333334},{"step":207,"nonzero":0.7916666666666666},{"step":209,"nonzero":0.8125},{"step":211,"nonzero":0.6458333333333334},{"step":213,"nonzero":0.5416666666666666},{"step":215,"nonzero":0.3541666666666667},{"step":217,"nonzero":0.5208333333333334},{"step":219,"nonzero":0.8333333333333334},{"step":221,"nonzero":0.4166666666666667},{"step":223,"nonzero":0.5416666666666666},{"step":225,"nonzero":0.3958333333333333},{"step":227,"nonzero":0.3958333333333333},{"step":229,"nonzero":0.4583333333333333},{"step":231,"nonzero":0.9166666666666666},{"step":233,"nonzero":0.4583333333333333},{"step":235,"nonzero":0.3541666666666667},{"step":237,"nonzero":0.6875},{"step":239,"nonzero":0.4166666666666667},{"step":241,"nonzero":0.2916666666666667},{"step":243,"nonzero":0.6041666666666666},{"step":245,"nonzero":0.5416666666666666},{"step":247,"nonzero":0.3958333333333333},{"step":249,"nonzero":0.6666666666666666},{"step":251,"nonzero":0.7291666666666666},{"step":253,"nonzero":0.5416666666666666},{"step":255,"nonzero":0.6458333333333334},{"step":257,"nonzero":0.4166666666666667},{"step":259,"nonzero":0.875},{"step":261,"nonzero":0.2916666666666667},{"step":263,"nonzero":0.4375},{"step":265,"nonzero":0.4791666666666667},{"step":267,"nonzero":0.3333333333333333},{"step":269,"nonzero":0.4166666666666667},{"step":271,"nonzero":0.5416666666666666},{"step":273,"nonzero":0.6875},{"step":275,"nonzero":0.22916666666666666},{"step":277,"nonzero":0.4166666666666667},{"step":279,"nonzero":0.7916666666666666},{"step":281,"nonzero":0.4791666666666667},{"step":283,"nonzero":0.625},{"step":285,"nonzero":0.6041666666666666},{"step":287,"nonzero":0.625},{"step":289,"nonzero":0.5625},{"step":291,"nonzero":0.625},{"step":293,"nonzero":0.6875},{"step":295,"nonzero":0.6458333333333334},{"step":297,"nonzero":0.6875},{"step":299,"nonzero":0.4375},{"step":301,"nonzero":0.5625},{"step":303,"nonzero":0.6666666666666666},{"step":305,"nonzero":0.5833333333333334},{"step":307,"nonzero":0.5833333333333334},{"step":309,"nonzero":0.7083333333333334},{"step":311,"nonzero":0.5},{"step":313,"nonzero":0.5},{"step":315,"nonzero":0.4166666666666667},{"step":317,"nonzero":0.5833333333333334},{"step":319,"nonzero":0.6041666666666666},{"step":321,"nonzero":0.3333333333333333},{"step":323,"nonzero":0.7083333333333334},{"step":325,"nonzero":0.4166666666666667},{"step":327,"nonzero":0.4166666666666667},{"step":329,"nonzero":0.75},{"step":331,"nonzero":0.4791666666666667},{"step":333,"nonzero":0.75},{"step":335,"nonzero":0.5208333333333334},{"step":337,"nonzero":0.7291666666666666},{"step":339,"nonzero":0.4166666666666667},{"step":341,"nonzero":0.4791666666666667},{"step":343,"nonzero":0.6875},{"step":345,"nonzero":0.22916666666666666},{"step":347,"nonzero":0.6666666666666666},{"step":349,"nonzero":0.7916666666666666},{"step":351,"nonzero":0.5208333333333334},{"step":353,"nonzero":0.7708333333333334},{"step":355,"nonzero":0.75},{"step":357,"nonzero":0.5833333333333334},{"step":359,"nonzero":0.8958333333333334},{"step":361,"nonzero":0.5416666666666666},{"step":363,"nonzero":0.6041666666666666},{"step":365,"nonzero":0.6666666666666666},{"step":367,"nonzero":0.5833333333333334},{"step":369,"nonzero":0.6041666666666666},{"step":371,"nonzero":0.4791666666666667},{"step":373,"nonzero":0.9375},{"step":375,"nonzero":0.75},{"step":377,"nonzero":0.6666666666666666},{"step":379,"nonzero":0.3541666666666667},{"step":381,"nonzero":0.4791666666666667},{"step":383,"nonzero":0.8125},{"step":385,"nonzero":0.8958333333333334},{"step":387,"nonzero":0.3958333333333333},{"step":389,"nonzero":0.7083333333333334},{"step":391,"nonzero":0.5208333333333334},{"step":393,"nonzero":0.8958333333333334},{"step":395,"nonzero":0.75},{"step":397,"nonzero":0.7708333333333334},{"step":399,"nonzero":0.5833333333333334},{"step":401,"nonzero":0.7291666666666666},{"step":403,"nonzero":0.4583333333333333},{"step":405,"nonzero":0.6875},{"step":407,"nonzero":0.3958333333333333},{"step":409,"nonzero":0.6666666666666666},{"step":411,"nonzero":0.7916666666666666},{"step":413,"nonzero":0.6875},{"step":415,"nonzero":0.5416666666666666},{"step":417,"nonzero":0.7916666666666666},{"step":419,"nonzero":0.7083333333333334},{"step":421,"nonzero":0.5208333333333334},{"step":423,"nonzero":0.5625},{"step":425,"nonzero":0.625},{"step":427,"nonzero":0.9166666666666666},{"step":429,"nonzero":0.4791666666666667},{"step":431,"nonzero":0.4375},{"step":433,"nonzero":0.7708333333333334},{"step":435,"nonzero":0.75},{"step":437,"nonzero":0.5833333333333334},{"step":439,"nonzero":0.5},{"step":441,"nonzero":0.3333333333333333},{"step":443,"nonzero":0.7291666666666666},{"step":445,"nonzero":0.4166666666666667},{"step":447,"nonzero":0.5625},{"step":449,"nonzero":0.7083333333333334},{"step":451,"nonzero":0.3333333333333333},{"step":453,"nonzero":0.6041666666666666},{"step":455,"nonzero":0.5416666666666666},{"step":457,"nonzero":0.7916666666666666},{"step":459,"nonzero":0.5416666666666666},{"step":461,"nonzero":0.8541666666666666},{"step":463,"nonzero":0.5416666666666666},{"step":465,"nonzero":0.5416666666666666},{"step":467,"nonzero":0.0},{"step":469,"nonzero":0.4791666666666667},{"step":471,"nonzero":0.3958333333333333},{"step":473,"nonzero":0.6041666666666666},{"step":475,"nonzero":0.9791666666666666},{"step":477,"nonzero":0.3541666666666667},{"step":479,"nonzero":0.6666666666666666},{"step":481,"nonzero":0.8333333333333334},{"step":483,"nonzero":0.3125},{"step":485,"nonzero":0.6458333333333334},{"step":487,"nonzero":0.125},{"step":489,"nonzero":0.7291666666666666},{"step":491,"nonzero":0.6041666666666666},{"step":493,"nonzero":0.7083333333333334},{"step":495,"nonzero":0.6666666666666666},{"step":497,"nonzero":0.6041666666666666},{"step":499,"nonzero":0.8541666666666666},{"step":501,"nonzero":0.6666666666666666},{"step":503,"nonzero":0.9583333333333334},{"step":505,"nonzero":0.625},{"step":507,"nonzero":0.7708333333333334},{"step":509,"nonzero":0.5208333333333334},{"step":511,"nonzero":0.5833333333333334},{"step":513,"nonzero":0.5625},{"step":515,"nonzero":0.7708333333333334},{"step":517,"nonzero":0.4166666666666667},{"step":519,"nonzero":0.7708333333333334},{"step":521,"nonzero":0.4583333333333333},{"step":523,"nonzero":0.5208333333333334},{"step":525,"nonzero":0.8541666666666666},{"step":527,"nonzero":0.5833333333333334},{"step":529,"nonzero":0.2916666666666667},{"step":531,"nonzero":0.4791666666666667},{"step":533,"nonzero":0.5833333333333334},{"step":535,"nonzero":0.5833333333333334},{"step":537,"nonzero":0.6875},{"step":539,"nonzero":0.6458333333333334},{"step":541,"nonzero":0.8125},{"step":543,"nonzero":0.6458333333333334},{"step":545,"nonzero":0.375},{"step":547,"nonzero":0.8958333333333334},{"step":549,"nonzero":0.5416666666666666},{"step":551,"nonzero":0.6666666666666666},{"step":553,"nonzero":0.6875},{"step":555,"nonzero":0.5833333333333334},{"step":557,"nonzero":0.875},{"step":559,"nonzero":0.5625},{"step":561,"nonzero":0.5625},{"step":563,"nonzero":0.5416666666666666},{"step":565,"nonzero":0.4166666666666667},{"step":567,"nonzero":0.4375},{"step":569,"nonzero":0.8333333333333334},{"step":571,"nonzero":0.7083333333333334},{"step":573,"nonzero":0.625},{"step":575,"nonzero":0.3958333333333333},{"step":577,"nonzero":0.5},{"step":579,"nonzero":0.5833333333333334},{"step":581,"nonzero":0.7708333333333334},{"step":583,"nonzero":0.6041666666666666},{"step":585,"nonzero":0.5833333333333334},{"step":587,"nonzero":0.375},{"step":589,"nonzero":0.8125},{"step":591,"nonzero":0.7083333333333334},{"step":593,"nonzero":0.5},{"step":595,"nonzero":0.5208333333333334},{"step":597,"nonzero":0.7708333333333334},{"step":599,"nonzero":0.7708333333333334},{"step":601,"nonzero":0.3958333333333333},{"step":603,"nonzero":0.8958333333333334},{"step":605,"nonzero":0.3125},{"step":607,"nonzero":0.6875},{"step":609,"nonzero":0.8333333333333334},{"step":611,"nonzero":0.5208333333333334},{"step":613,"nonzero":0.5},{"step":615,"nonzero":0.5416666666666666},{"step":617,"nonzero":0.5},{"step":619,"nonzero":0.8541666666666666},{"step":621,"nonzero":0.7916666666666666},{"step":623,"nonzero":0.7083333333333334},{"step":625,"nonzero":0.25},{"step":627,"nonzero":0.5625},{"step":629,"nonzero":0.625},{"step":631,"nonzero":0.4375},{"step":633,"nonzero":0.5208333333333334},{"step":635,"nonzero":0.6666666666666666},{"step":637,"nonzero":0.75},{"step":639,"nonzero":0.4375},{"step":641,"nonzero":0.6458333333333334},{"step":643,"nonzero":0.5208333333333334},{"step":645,"nonzero":0.6458333333333334},{"step":647,"nonzero":0.9375},{"step":649,"nonzero":0.6458333333333334},{"step":651,"nonzero":0.5625},{"step":653,"nonzero":0.8333333333333334},{"step":655,"nonzero":0.6666666666666666},{"step":657,"nonzero":0.8958333333333334},{"step":659,"nonzero":0.5},{"step":661,"nonzero":0.6458333333333334},{"step":663,"nonzero":0.625},{"step":665,"nonzero":0.8958333333333334},{"step":667,"nonzero":0.6875},{"step":669,"nonzero":0.7916666666666666},{"step":671,"nonzero":0.7291666666666666},{"step":673,"nonzero":0.5833333333333334},{"step":675,"nonzero":0.875},{"step":677,"nonzero":0.6875},{"step":679,"nonzero":0.7708333333333334},{"step":681,"nonzero":0.7708333333333334},{"step":683,"nonzero":0.875},{"step":685,"nonzero":0.8958333333333334},{"step":687,"nonzero":0.875},{"step":689,"nonzero":0.7291666666666666},{"step":691,"nonzero":0.5625},{"step":693,"nonzero":0.8958333333333334},{"step":695,"nonzero":0.625},{"step":697,"nonzero":0.875},{"step":699,"nonzero":0.5},{"step":701,"nonzero":0.8541666666666666},{"step":703,"nonzero":0.75},{"step":705,"nonzero":0.4791666666666667},{"step":707,"nonzero":0.6041666666666666},{"step":709,"nonzero":0.8541666666666666},{"step":711,"nonzero":0.8958333333333334},{"step":713,"nonzero":0.7916666666666666},{"step":715,"nonzero":0.625},{"step":717,"nonzero":0.5833333333333334},{"step":719,"nonzero":0.5625},{"step":721,"nonzero":0.4791666666666667},{"step":723,"nonzero":0.7291666666666666},{"step":725,"nonzero":0.6458333333333334},{"step":727,"nonzero":0.4583333333333333},{"step":729,"nonzero":0.6875},{"step":731,"nonzero":0.5},{"step":733,"nonzero":0.5},{"step":735,"nonzero":0.8333333333333334},{"step":737,"nonzero":0.9583333333333334},{"step":739,"nonzero":0.8333333333333334},{"step":741,"nonzero":0.8333333333333334},{"step":743,"nonzero":0.8541666666666666},{"step":745,"nonzero":0.4791666666666667},{"step":747,"nonzero":0.6666666666666666},{"step":749,"nonzero":0.4791666666666667},{"step":751,"nonzero":0.6666666666666666},{"step":753,"nonzero":0.4791666666666667},{"step":755,"nonzero":0.7916666666666666},{"step":757,"nonzero":0.5833333333333334},{"step":759,"nonzero":0.5625},{"step":761,"nonzero":0.625},{"step":763,"nonzero":0.9583333333333334},{"step":765,"nonzero":0.4583333333333333},{"step":767,"nonzero":0.875},{"step":769,"nonzero":0.5833333333333334},{"step":771,"nonzero":0.8125},{"step":773,"nonzero":0.7083333333333334},{"step":775,"nonzero":0.6041666666666666},{"step":777,"nonzero":0.7291666666666666},{"step":779,"nonzero":0.7916666666666666},{"step":781,"nonzero":0.7916666666666666},{"step":783,"nonzero":0.4583333333333333},{"step":785,"nonzero":0.9583333333333334},{"step":787,"nonzero":0.625},{"step":789,"nonzero":0.6041666666666666},{"step":791,"nonzero":0.8125},{"step":793,"nonzero":0.5},{"step":795,"nonzero":0.4583333333333333},{"step":797,"nonzero":0.4583333333333333},{"step":799,"nonzero":0.625},{"step":801,"nonzero":0.4791666666666667},{"step":803,"nonzero":0.9375},{"step":805,"nonzero":0.3958333333333333},{"step":807,"nonzero":0.1875},{"step":809,"nonzero":0.8333333333333334},{"step":811,"nonzero":0.8333333333333334},{"step":813,"nonzero":0.5833333333333334},{"step":815,"nonzero":0.875},{"step":817,"nonzero":0.5416666666666666},{"step":819,"nonzero":0.6875},{"step":821,"nonzero":0.6875},{"step":823,"nonzero":0.6458333333333334},{"step":825,"nonzero":0.9166666666666666},{"step":827,"nonzero":0.5625},{"step":829,"nonzero":0.4791666666666667},{"step":831,"nonzero":0.8958333333333334},{"step":833,"nonzero":0.8125},{"step":835,"nonzero":0.7291666666666666},{"step":837,"nonzero":0.7083333333333334},{"step":839,"nonzero":0.8125},{"step":841,"nonzero":0.4166666666666667},{"step":843,"nonzero":0.625},{"step":845,"nonzero":0.7083333333333334},{"step":847,"nonzero":0.5},{"step":849,"nonzero":0.8958333333333334},{"step":851,"nonzero":0.4375},{"step":853,"nonzero":0.7708333333333334},{"step":855,"nonzero":0.5833333333333334},{"step":857,"nonzero":0.875},{"step":859,"nonzero":0.7291666666666666},{"step":861,"nonzero":0.4583333333333333},{"step":863,"nonzero":0.6666666666666666},{"step":865,"nonzero":0.6875},{"step":867,"nonzero":0.3958333333333333},{"step":869,"nonzero":0.6875},{"step":871,"nonzero":0.6666666666666666},{"step":873,"nonzero":0.75},{"step":875,"nonzero":0.7291666666666666},{"step":877,"nonzero":0.75},{"step":879,"nonzero":0.9375},{"step":881,"nonzero":0.625},{"step":883,"nonzero":0.6041666666666666},{"step":885,"nonzero":0.7083333333333334},{"step":887,"nonzero":0.4791666666666667},{"step":889,"nonzero":0.875},{"step":891,"nonzero":0.3958333333333333},{"step":893,"nonzero":0.125},{"step":895,"nonzero":0.8958333333333334},{"step":897,"nonzero":0.5416666666666666},{"step":899,"nonzero":0.7291666666666666},{"step":901,"nonzero":0.75},{"step":903,"nonzero":0.7291666666666666},{"step":905,"nonzero":0.7083333333333334},{"step":907,"nonzero":0.625},{"step":909,"nonzero":0.6041666666666666},{"step":911,"nonzero":0.4791666666666667},{"step":913,"nonzero":0.8958333333333334},{"step":915,"nonzero":0.9166666666666666},{"step":917,"nonzero":0.5416666666666666},{"step":919,"nonzero":0.7708333333333334},{"step":921,"nonzero":0.4791666666666667},{"step":923,"nonzero":0.7916666666666666},{"step":925,"nonzero":0.4375},{"step":927,"nonzero":0.7083333333333334},{"step":929,"nonzero":0.8333333333333334},{"step":931,"nonzero":0.5},{"step":933,"nonzero":0.625},{"step":935,"nonzero":0.625},{"step":937,"nonzero":0.8541666666666666},{"step":939,"nonzero":0.6666666666666666},{"step":941,"nonzero":0.5833333333333334},{"step":943,"nonzero":0.75},{"step":945,"nonzero":0.6875},{"step":947,"nonzero":0.5208333333333334},{"step":949,"nonzero":0.875},{"step":951,"nonzero":0.4791666666666667},{"step":953,"nonzero":0.875},{"step":955,"nonzero":0.3333333333333333},{"step":957,"nonzero":0.8541666666666666},{"step":959,"nonzero":0.5833333333333334},{"step":961,"nonzero":0.6666666666666666},{"step":963,"nonzero":0.7083333333333334},{"step":965,"nonzero":0.6666666666666666},{"step":967,"nonzero":0.6458333333333334},{"step":969,"nonzero":0.7083333333333334},{"step":971,"nonzero":0.7083333333333334},{"step":973,"nonzero":0.7291666666666666},{"step":975,"nonzero":0.75},{"step":977,"nonzero":0.625},{"step":979,"nonzero":0.625},{"step":981,"nonzero":0.7291666666666666},{"step":983,"nonzero":0.6041666666666666},{"step":985,"nonzero":0.5625},{"step":987,"nonzero":0.8541666666666666},{"step":989,"nonzero":0.6041666666666666},{"step":991,"nonzero":0.5833333333333334},{"step":993,"nonzero":0.6041666666666666},{"step":995,"nonzero":0.6458333333333334},{"step":997,"nonzero":0.3958333333333333},{"step":999,"nonzero":0.5416666666666666},{"step":1001,"nonzero":0.6041666666666666},{"step":1003,"nonzero":0.6666666666666666},{"step":1005,"nonzero":0.3541666666666667},{"step":1007,"nonzero":0.7916666666666666},{"step":1009,"nonzero":0.5833333333333334},{"step":1011,"nonzero":0.5},{"step":1013,"nonzero":0.6875},{"step":1015,"nonzero":0.6666666666666666},{"step":1017,"nonzero":0.8125},{"step":1019,"nonzero":0.8958333333333334},{"step":1021,"nonzero":0.8333333333333334},{"step":1023,"nonzero":0.9166666666666666},{"step":1025,"nonzero":0.9375},{"step":1027,"nonzero":0.6458333333333334},{"step":1029,"nonzero":0.8125},{"step":1031,"nonzero":0.6458333333333334},{"step":1033,"nonzero":0.7291666666666666},{"step":1035,"nonzero":0.6041666666666666},{"step":1037,"nonzero":0.5},{"step":1039,"nonzero":0.4583333333333333},{"step":1041,"nonzero":0.6666666666666666},{"step":1043,"nonzero":0.5416666666666666},{"step":1045,"nonzero":0.8958333333333334},{"step":1047,"nonzero":0.6458333333333334},{"step":1049,"nonzero":0.8541666666666666},{"step":1051,"nonzero":0.6666666666666666},{"step":1053,"nonzero":0.8958333333333334},{"step":1055,"nonzero":0.4583333333333333},{"step":1057,"nonzero":0.2708333333333333},{"step":1059,"nonzero":0.7708333333333334},{"step":1061,"nonzero":0.8333333333333334},{"step":1063,"nonzero":0.8125},{"step":1065,"nonzero":0.8333333333333334},{"step":1067,"nonzero":0.6666666666666666},{"step":1069,"nonzero":0.8958333333333334},{"step":1071,"nonzero":0.5},{"step":1073,"nonzero":0.9375},{"step":1075,"nonzero":1.0},{"step":1077,"nonzero":0.375},{"step":1079,"nonzero":0.7291666666666666},{"step":1081,"nonzero":0.9166666666666666},{"step":1083,"nonzero":0.7708333333333334},{"step":1085,"nonzero":0.875},{"step":1087,"nonzero":0.3333333333333333},{"step":1089,"nonzero":0.4375},{"step":1091,"nonzero":0.6875},{"step":1093,"nonzero":0.9791666666666666},{"step":1095,"nonzero":0.6875},{"step":1097,"nonzero":0.4166666666666667},{"step":1099,"nonzero":0.8541666666666666},{"step":1101,"nonzero":0.6666666666666666},{"step":1103,"nonzero":0.6041666666666666},{"step":1105,"nonzero":0.875},{"step":1107,"nonzero":0.7083333333333334},{"step":1109,"nonzero":0.6875},{"step":1111,"nonzero":0.7708333333333334},{"step":1113,"nonzero":0.5416666666666666},{"step":1115,"nonzero":0.9791666666666666},{"step":1117,"nonzero":0.3541666666666667},{"step":1119,"nonzero":0.3333333333333333},{"step":1121,"nonzero":0.8125},{"step":1123,"nonzero":0.875},{"step":1125,"nonzero":0.625},{"step":1127,"nonzero":0.6666666666666666},{"step":1129,"nonzero":0.7291666666666666},{"step":1131,"nonzero":0.875},{"step":1133,"nonzero":0.8958333333333334},{"step":1135,"nonzero":0.4166666666666667},{"step":1137,"nonzero":0.4791666666666667},{"step":1139,"nonzero":0.3541666666666667},{"step":1141,"nonzero":0.8541666666666666},{"step":1143,"nonzero":0.8125},{"step":1145,"nonzero":0.4791666666666667},{"step":1147,"nonzero":0.4166666666666667},{"step":1149,"nonzero":0.14583333333333334},{"step":1151,"nonzero":0.5},{"step":1153,"nonzero":0.8333333333333334},{"step":1155,"nonzero":0.7083333333333334},{"step":1157,"nonzero":0.6458333333333334},{"step":1159,"nonzero":0.875},{"step":1161,"nonzero":0.5833333333333334},{"step":1163,"nonzero":0.5},{"step":1165,"nonzero":0.7083333333333334},{"step":1167,"nonzero":0.3541666666666667},{"step":1169,"nonzero":0.4791666666666667},{"step":1171,"nonzero":0.6458333333333334},{"step":1173,"nonzero":0.7291666666666666},{"step":1175,"nonzero":0.6666666666666666},{"step":1177,"nonzero":0.8333333333333334},{"step":1179,"nonzero":0.6875},{"step":1181,"nonzero":0.625},{"step":1183,"nonzero":0.5},{"step":1185,"nonzero":0.8125},{"step":1187,"nonzero":0.5},{"step":1189,"nonzero":0.4583333333333333},{"step":1191,"nonzero":0.5833333333333334},{"step":1193,"nonzero":0.9375},{"step":1195,"nonzero":0.5833333333333334},{"step":1197,"nonzero":0.5416666666666666},{"step":1199,"nonzero":0.5}]},{"id":"cispo-lora-500","label":"CISPO \u00b7 LoRA \u00b7 500 prompts","run_ids":["1790891170"],"loss":"CISPO","adapter":"LoRA","prompts":500,"repo_r":false,"holdout_by_checkpoint":[{"step":0,"holdout":-1.3664,"train_core":-0.9836,"valid_rate":0.9},{"step":100,"holdout":-0.8768,"train_core":-0.5768,"valid_rate":0.92},{"step":200,"holdout":-0.8437,"train_core":-0.4679,"valid_rate":0.96},{"step":300,"holdout":-0.357,"train_core":-0.2878,"valid_rate":0.98},{"step":400,"holdout":-0.411,"train_core":-0.3064,"valid_rate":0.96},{"step":500,"holdout":-0.0731,"train_core":-0.1393,"valid_rate":0.96},{"step":600,"holdout":-0.0238,"train_core":-0.0759,"valid_rate":0.96}],"selected_step":600,"selected_holdout":-0.0238,"train_steps":[{"step":1,"reward":-2.2583541870117188,"core":-0.7187981605529785,"cost":-0.08142795413732529,"kl":0.0,"entropy":0.5175154954195023},{"step":2,"reward":-2.9991142749786377,"core":-0.9022471308708191,"cost":-0.18774215877056122,"kl":0.00023456673079635948,"entropy":0.45070403814315796},{"step":3,"reward":-2.94317364692688,"core":-0.8581757545471191,"cost":-0.27271509170532227,"kl":0.000425292324507609,"entropy":0.5004923269152641},{"step":4,"reward":-3.0975046157836914,"core":-0.9354011416435242,"cost":-0.22742819786071777,"kl":0.0005059328104835004,"entropy":0.5383268296718597},{"step":5,"reward":-4.004680633544922,"core":-1.1581145524978638,"cost":-0.4358718693256378,"kl":0.0005266924054012634,"entropy":0.5259639397263527},{"step":6,"reward":-2.5278589725494385,"core":-0.7521414160728455,"cost":-0.16183467209339142,"kl":0.0009191778808599338,"entropy":0.50031328946352},{"step":7,"reward":-4.841067790985107,"core":-1.3986352682113647,"cost":-0.599328339099884,"kl":0.001311467683990486,"entropy":0.49095258861780167},{"step":8,"reward":-3.139941930770874,"core":-0.9154273867607117,"cost":-0.2994728982448578,"kl":0.0018470350769348443,"entropy":0.5438341051340103},{"step":9,"reward":-1.6290712356567383,"core":-0.5010430216789246,"cost":-0.01457308977842331,"kl":0.00244742093491368,"entropy":0.5471763908863068},{"step":10,"reward":-2.357492446899414,"core":-0.6895873546600342,"cost":-0.17077083885669708,"kl":0.0033945890027098358,"entropy":0.5009930804371834},{"step":11,"reward":-2.321132183074951,"core":-0.7177125811576843,"cost":-0.07038591057062149,"kl":0.0034833898534998298,"entropy":0.493100106716156},{"step":12,"reward":-2.514181613922119,"core":-0.7229154706001282,"cost":-0.18155519664287567,"kl":0.006625834503211081,"entropy":0.4416109323501587},{"step":13,"reward":-2.3546254634857178,"core":-0.7146406173706055,"cost":-0.10580740123987198,"kl":0.007445836206898093,"entropy":0.43163925409317017},{"step":14,"reward":-2.347039222717285,"core":-0.693945586681366,"cost":-0.15158003568649292,"kl":0.007799963233992457,"entropy":0.4919782057404518},{"step":15,"reward":-3.3492679595947266,"core":-1.0118130445480347,"cost":-0.23822756111621857,"kl":0.010095543460920453,"entropy":0.5722168236970901},{"step":16,"reward":-2.1242129802703857,"core":-0.6202408671379089,"cost":-0.09469801932573318,"kl":0.014694149373099208,"entropy":0.5213876962661743},{"step":17,"reward":-4.125580310821533,"core":-1.250967264175415,"cost":-0.3097933530807495,"kl":0.009128592209890485,"entropy":0.43798091262578964},{"step":18,"reward":-3.4216129779815674,"core":-1.0159746408462524,"cost":-0.3077037036418915,"kl":0.020719473948702216,"entropy":0.6014263331890106},{"step":19,"reward":-1.5988178253173828,"core":-0.47336897253990173,"cost":-0.05405864119529724,"kl":0.017199151683598757,"entropy":0.48381806164979935},{"step":20,"reward":-2.158353567123413,"core":-0.6001566052436829,"cost":-0.1407698541879654,"kl":0.015791159588843584,"entropy":0.5061067268252373},{"step":21,"reward":-2.353634834289551,"core":-0.6661030054092407,"cost":-0.1612202227115631,"kl":0.015266490867361426,"entropy":0.5274627059698105},{"step":22,"reward":-2.612590789794922,"core":-0.7704262733459473,"cost":-0.12813201546669006,"kl":0.027201584074646235,"entropy":0.5484939515590668},{"step":23,"reward":-2.252194404602051,"core":-0.6637815237045288,"cost":-0.10293225198984146,"kl":0.023200371069833636,"entropy":0.4670744687318802},{"step":24,"reward":-1.5275869369506836,"core":-0.4495622217655182,"cost":-0.007322743069380522,"kl":0.023869932629168034,"entropy":0.45024122297763824},{"step":25,"reward":-4.006917953491211,"core":-1.1694248914718628,"cost":-0.4096197783946991,"kl":0.026827764231711626,"entropy":0.4176264777779579},{"step":26,"reward":-1.5641883611679077,"core":-0.39357879757881165,"cost":-0.08065969496965408,"kl":0.021842909511178732,"entropy":0.4393974170088768},{"step":27,"reward":-1.603045105934143,"core":-0.4496345520019531,"cost":-0.08229123800992966,"kl":0.033084319438785315,"entropy":0.41730059683322906},{"step":28,"reward":-2.448241949081421,"core":-0.6943225860595703,"cost":-0.24216778576374054,"kl":0.04017028212547302,"entropy":0.4199502244591713},{"step":29,"reward":-1.0827436447143555,"core":-0.2999538779258728,"cost":-0.05480246618390083,"kl":0.04681811109185219,"entropy":0.5536157339811325},{"step":30,"reward":-1.7301944494247437,"core":-0.4665854871273041,"cost":-0.06221264228224754,"kl":0.03342516301199794,"entropy":0.37121137976646423},{"step":31,"reward":-2.7696151733398438,"core":-0.8364599347114563,"cost":-0.10154404491186142,"kl":0.06482217554003,"entropy":0.4445219933986664},{"step":32,"reward":-1.4696842432022095,"core":-0.45207810401916504,"cost":-0.04505517706274986,"kl":0.04164008004590869,"entropy":0.5191213488578796},{"step":33,"reward":-1.1927956342697144,"core":-0.3372401297092438,"cost":-0.00011308125249342993,"kl":0.059791975654661655,"entropy":0.5364971831440926},{"step":34,"reward":-2.152059555053711,"core":-0.6250324845314026,"cost":-0.04935064911842346,"kl":0.086803681217134,"entropy":0.4806259572505951},{"step":35,"reward":-1.5104619264602661,"core":-0.4489598274230957,"cost":-0.043238651007413864,"kl":0.06447460688650608,"entropy":0.5318896695971489},{"step":36,"reward":-2.973501443862915,"core":-0.8716400265693665,"cost":-0.23383931815624237,"kl":0.034538077656179667,"entropy":0.4097753167152405},{"step":37,"reward":-2.297893524169922,"core":-0.6511683464050293,"cost":-0.13804404437541962,"kl":0.05197635293006897,"entropy":0.4224545508623123},{"step":38,"reward":-0.965893030166626,"core":-0.23233550786972046,"cost":-0.042388420552015305,"kl":0.035460010170936584,"entropy":0.43635641783475876},{"step":39,"reward":-1.5243902206420898,"core":-0.4379083216190338,"cost":-0.003782127285376191,"kl":0.031243789941072464,"entropy":0.37871213257312775},{"step":40,"reward":-1.4191880226135254,"core":-0.4193430244922638,"cost":-0.021512439474463463,"kl":0.055799282155930996,"entropy":0.4304047077894211},{"step":41,"reward":-2.53153657913208,"core":-0.7457740902900696,"cost":-0.15641812980175018,"kl":0.035535894334316254,"entropy":0.4329649806022644},{"step":42,"reward":-0.7707758545875549,"core":-0.19510018825531006,"cost":-0.03183431550860405,"kl":0.0340244653634727,"entropy":0.43957724422216415},{"step":43,"reward":-1.4265395402908325,"core":-0.39857926964759827,"cost":-0.050898969173431396,"kl":0.02978695509955287,"entropy":0.4506237432360649},{"step":44,"reward":-2.435507297515869,"core":-0.7525653839111328,"cost":-0.06107860431075096,"kl":0.03578806854784489,"entropy":0.45378996431827545},{"step":45,"reward":-1.542192816734314,"core":-0.47094082832336426,"cost":-0.03273021802306175,"kl":0.038636879064142704,"entropy":0.4651770293712616},{"step":46,"reward":-1.736007571220398,"core":-0.49900951981544495,"cost":-0.033286336809396744,"kl":0.04511739779263735,"entropy":0.5486686527729034},{"step":47,"reward":-1.8785828351974487,"core":-0.548435628414154,"cost":-0.03513540327548981,"kl":0.04205253068357706,"entropy":0.4808560982346535},{"step":48,"reward":-1.9177045822143555,"core":-0.5724199414253235,"cost":-0.09330185502767563,"kl":0.046363577246665955,"entropy":0.4574437811970711},{"step":49,"reward":-1.1565041542053223,"core":-0.31865254044532776,"cost":-0.017592763528227806,"kl":0.0461303167976439,"entropy":0.5025374069809914},{"step":50,"reward":-1.806195855140686,"core":-0.4848902225494385,"cost":-0.11245349794626236,"kl":0.04075142182409763,"entropy":0.4820035621523857},{"step":51,"reward":-2.7506141662597656,"core":-0.8227537274360657,"cost":-0.1433601677417755,"kl":0.04613098129630089,"entropy":0.4852048233151436},{"step":52,"reward":-2.4700262546539307,"core":-0.7426161766052246,"cost":-0.18684516847133636,"kl":0.03545901691541076,"entropy":0.49084296077489853},{"step":53,"reward":-1.330853819847107,"core":-0.39000579714775085,"cost":-0.04240061715245247,"kl":0.033176873344928026,"entropy":0.46140193194150925},{"step":54,"reward":-1.7625852823257446,"core":-0.49105677008628845,"cost":-0.11372623592615128,"kl":0.037156619131565094,"entropy":0.5582830458879471},{"step":55,"reward":-1.648734450340271,"core":-0.4946289360523224,"cost":-0.0335693396627903,"kl":0.045541259460151196,"entropy":0.5622713565826416},{"step":56,"reward":-1.8418124914169312,"core":-0.5522123575210571,"cost":-0.10282456129789352,"kl":0.03374938014894724,"entropy":0.46929628401994705},{"step":57,"reward":-1.799536108970642,"core":-0.5508273243904114,"cost":-0.0634634792804718,"kl":0.03340124385431409,"entropy":0.5427168607711792},{"step":58,"reward":-1.5904622077941895,"core":-0.49893733859062195,"cost":-0.011311796493828297,"kl":0.041368184611201286,"entropy":0.5044969096779823},{"step":59,"reward":-1.1815261840820312,"core":-0.33468905091285706,"cost":0.0,"kl":0.05739549174904823,"entropy":0.4841710478067398},{"step":60,"reward":-1.416251540184021,"core":-0.40412071347236633,"cost":-0.022854089736938477,"kl":0.03939502080902457,"entropy":0.5185195580124855},{"step":61,"reward":-1.7485116720199585,"core":-0.5455143451690674,"cost":-0.03841034695506096,"kl":0.05628044996410608,"entropy":0.4278881847858429},{"step":62,"reward":-1.992310643196106,"core":-0.6029263138771057,"cost":-0.08173162490129471,"kl":0.05089388135820627,"entropy":0.4822029545903206},{"step":63,"reward":-2.1744470596313477,"core":-0.6798768639564514,"cost":-0.05917662754654884,"kl":0.04589771665632725,"entropy":0.4962329864501953},{"step":64,"reward":-1.8618555068969727,"core":-0.5491837859153748,"cost":-0.06808013468980789,"kl":0.04914229176938534,"entropy":0.47633038461208344},{"step":65,"reward":-0.9148035049438477,"core":-0.19507597386837006,"cost":-0.026567906141281128,"kl":0.05381646752357483,"entropy":0.5091278329491615},{"step":66,"reward":-1.1954078674316406,"core":-0.3537144660949707,"cost":0.0,"kl":0.0433939453214407,"entropy":0.5079823881387711},{"step":67,"reward":-1.6708145141601562,"core":-0.5165652632713318,"cost":-0.05474551394581795,"kl":0.05041254311800003,"entropy":0.573804184794426},{"step":68,"reward":-1.4992613792419434,"core":-0.44019603729248047,"cost":-0.07394440472126007,"kl":0.05642862990498543,"entropy":0.5570868402719498},{"step":69,"reward":-1.6960467100143433,"core":-0.5039024949073792,"cost":-0.025365954264998436,"kl":0.04885501693934202,"entropy":0.5601024776697159},{"step":70,"reward":-2.486236095428467,"core":-0.76649409532547,"cost":-0.08528000116348267,"kl":0.06210832018405199,"entropy":0.5338591933250427},{"step":71,"reward":-2.717247724533081,"core":-0.8029139041900635,"cost":-0.1836339831352234,"kl":0.056481397710740566,"entropy":0.5652947425842285},{"step":72,"reward":-1.9564833641052246,"core":-0.6074568629264832,"cost":-0.05393711104989052,"kl":0.04936850070953369,"entropy":0.5129585266113281},{"step":73,"reward":-1.9362736940383911,"core":-0.6149352192878723,"cost":-0.025971723720431328,"kl":0.03677332494407892,"entropy":0.48009122908115387},{"step":74,"reward":-1.4171406030654907,"core":-0.39838483929634094,"cost":-0.023250319063663483,"kl":0.05279969982802868,"entropy":0.555828757584095},{"step":75,"reward":-2.4069693088531494,"core":-0.7492208480834961,"cost":-0.09870308637619019,"kl":0.030055664479732513,"entropy":0.444711335003376},{"step":76,"reward":-2.285128593444824,"core":-0.6710777878761292,"cost":-0.07369454205036163,"kl":0.034815758001059294,"entropy":0.4903194233775139},{"step":77,"reward":-2.820314407348633,"core":-0.8513617515563965,"cost":-0.17494072020053864,"kl":0.038212383165955544,"entropy":0.5226079970598221},{"step":78,"reward":-2.005127191543579,"core":-0.5858716368675232,"cost":-0.10373339056968689,"kl":0.04930805508047342,"entropy":0.5349173992872238},{"step":79,"reward":-2.055718183517456,"core":-0.6236664652824402,"cost":-0.0939568504691124,"kl":0.04625403834506869,"entropy":0.5827695578336716},{"step":80,"reward":-2.5562350749969482,"core":-0.6870430111885071,"cost":-0.26752355694770813,"kl":0.0341314603574574,"entropy":0.4848063960671425},{"step":81,"reward":-2.535755157470703,"core":-0.7692628502845764,"cost":-0.14406122267246246,"kl":0.05274519696831703,"entropy":0.5307983383536339},{"step":82,"reward":-2.8775691986083984,"core":-0.869098961353302,"cost":-0.1495072841644287,"kl":0.04678867431357503,"entropy":0.541705310344696},{"step":83,"reward":-3.0928733348846436,"core":-0.9132263660430908,"cost":-0.2837267518043518,"kl":0.026885895058512688,"entropy":0.5129286497831345},{"step":84,"reward":-2.1631758213043213,"core":-0.6754452586174011,"cost":-0.03657818213105202,"kl":0.07671176688745618,"entropy":0.5368999242782593},{"step":85,"reward":-1.5208377838134766,"core":-0.4186914265155792,"cost":-0.02571740746498108,"kl":0.028713728301227093,"entropy":0.5420590341091156},{"step":86,"reward":-2.1748239994049072,"core":-0.688098669052124,"cost":-0.03854861855506897,"kl":0.04754869267344475,"entropy":0.5818623155355453},{"step":87,"reward":-2.0121915340423584,"core":-0.5995033383369446,"cost":-0.06520330160856247,"kl":0.028013935778290033,"entropy":0.538110040128231},{"step":88,"reward":-2.7366116046905518,"core":-0.808701753616333,"cost":-0.21592281758785248,"kl":0.03823794238269329,"entropy":0.5516804158687592},{"step":89,"reward":-1.6367522478103638,"core":-0.5074589252471924,"cost":-0.0003227353736292571,"kl":0.042313829995691776,"entropy":0.4702178090810776},{"step":90,"reward":-1.8050593137741089,"core":-0.5521051287651062,"cost":-0.03310764208436012,"kl":0.0410268846899271,"entropy":0.602017879486084},{"step":91,"reward":-1.920030117034912,"core":-0.5670508742332458,"cost":-0.060836583375930786,"kl":0.0179561460390687,"entropy":0.5290157124400139},{"step":92,"reward":-2.1608738899230957,"core":-0.6290717720985413,"cost":-0.039827391505241394,"kl":0.022347761318087578,"entropy":0.48354264348745346},{"step":93,"reward":-3.344752311706543,"core":-0.9833682179450989,"cost":-0.2934519052505493,"kl":0.015722331125289202,"entropy":0.4551960974931717},{"step":94,"reward":-2.536074638366699,"core":-0.7525556683540344,"cost":-0.16610567271709442,"kl":0.026875109877437353,"entropy":0.4733629450201988},{"step":95,"reward":-2.633204936981201,"core":-0.7437620162963867,"cost":-0.2708788812160492,"kl":0.026878674048930407,"entropy":0.5126914530992508},{"step":96,"reward":-1.9257808923721313,"core":-0.5654457211494446,"cost":-0.049638886004686356,"kl":0.02950846659950912,"entropy":0.5124329403042793},{"step":97,"reward":-4.0513081550598145,"core":-1.1649580001831055,"cost":-0.4930900037288666,"kl":0.02147721406072378,"entropy":0.5547563582658768},{"step":98,"reward":-2.152437210083008,"core":-0.5817753672599792,"cost":-0.10927214473485947,"kl":0.020833547692745924,"entropy":0.45527835935354233},{"step":99,"reward":-2.8665878772735596,"core":-0.8502058982849121,"cost":-0.21619988977909088,"kl":0.02971508540213108,"entropy":0.3848009705543518},{"step":100,"reward":-1.3435406684875488,"core":-0.3497951030731201,"cost":-0.07811091095209122,"kl":0.02543912874534726,"entropy":0.5415019541978836},{"step":101,"reward":-0.9903298020362854,"core":-0.25087305903434753,"cost":0.0,"kl":0.029119090642780066,"entropy":0.5549716800451279},{"step":102,"reward":-3.0537478923797607,"core":-0.9045023918151855,"cost":-0.198406383395195,"kl":0.020724336616694927,"entropy":0.4594495967030525},{"step":103,"reward":-0.8839948773384094,"core":-0.26807162165641785,"cost":0.0,"kl":0.026113761588931084,"entropy":0.5651893466711044},{"step":104,"reward":-2.02720046043396,"core":-0.589650571346283,"cost":-0.1037505492568016,"kl":0.023793370462954044,"entropy":0.598775327205658},{"step":105,"reward":-2.6344664096832275,"core":-0.8023333549499512,"cost":-0.1328212022781372,"kl":0.019036197569221258,"entropy":0.5003297403454781},{"step":106,"reward":-3.226409912109375,"core":-0.9472334384918213,"cost":-0.3006919324398041,"kl":0.03756574634462595,"entropy":0.5269028022885323},{"step":107,"reward":-1.4515976905822754,"core":-0.4144751727581024,"cost":-0.0366324707865715,"kl":0.021871168166399002,"entropy":0.47313494235277176},{"step":108,"reward":-3.0488808155059814,"core":-0.9356820583343506,"cost":-0.17174053192138672,"kl":0.022076715249568224,"entropy":0.49278757721185684},{"step":109,"reward":-3.4745006561279297,"core":-1.0718103647232056,"cost":-0.19655096530914307,"kl":0.028795221354812384,"entropy":0.5041055753827095},{"step":110,"reward":-2.6743004322052,"core":-0.7337705492973328,"cost":-0.22210443019866943,"kl":0.030341952573508024,"entropy":0.5602424368262291},{"step":111,"reward":-3.316537618637085,"core":-0.9971718788146973,"cost":-0.2046273797750473,"kl":0.024103526026010513,"entropy":0.5388277620077133},{"step":112,"reward":-1.6047264337539673,"core":-0.48523783683776855,"cost":-0.015922345221042633,"kl":0.03149688011035323,"entropy":0.5659838616847992},{"step":113,"reward":-1.389552116394043,"core":-0.42623040080070496,"cost":-0.011710125021636486,"kl":0.02566712675616145,"entropy":0.5627131760120392},{"step":114,"reward":-2.3076930046081543,"core":-0.7072909474372864,"cost":-0.11802226305007935,"kl":0.02712908200919628,"entropy":0.4423419013619423},{"step":115,"reward":-1.7984918355941772,"core":-0.5323708057403564,"cost":-0.03820216655731201,"kl":0.021336206700652838,"entropy":0.5009709894657135},{"step":116,"reward":-0.7026667594909668,"core":-0.1314529925584793,"cost":-0.010310855694115162,"kl":0.022567223757505417,"entropy":0.4820537120103836},{"step":117,"reward":-1.8827053308486938,"core":-0.539319634437561,"cost":-0.12844274938106537,"kl":0.027527811471372843,"entropy":0.5077526494860649},{"step":118,"reward":-2.8266239166259766,"core":-0.8512284159660339,"cost":-0.2147049456834793,"kl":0.02706099394708872,"entropy":0.5418351590633392},{"step":119,"reward":-1.1949373483657837,"core":-0.30749815702438354,"cost":-0.02025640569627285,"kl":0.029951658099889755,"entropy":0.499752402305603},{"step":120,"reward":-2.260913372039795,"core":-0.6461188793182373,"cost":-0.13592179119586945,"kl":0.0264006108045578,"entropy":0.5454516410827637},{"step":121,"reward":-2.7626678943634033,"core":-0.8108994364738464,"cost":-0.20917153358459473,"kl":0.028575400356203318,"entropy":0.4670546054840088},{"step":122,"reward":-1.3366049528121948,"core":-0.33147865533828735,"cost":-0.01829642988741398,"kl":0.04071943927556276,"entropy":0.5340625643730164},{"step":123,"reward":-1.4817838668823242,"core":-0.45933791995048523,"cost":-0.006491074338555336,"kl":0.03344764420762658,"entropy":0.5585716068744659},{"step":124,"reward":-1.5556964874267578,"core":-0.425795316696167,"cost":-0.12972116470336914,"kl":0.028397899121046066,"entropy":0.4892457202076912},{"step":125,"reward":-2.9482429027557373,"core":-0.7738337516784668,"cost":-0.34989622235298157,"kl":0.03525213059037924,"entropy":0.509943313896656},{"step":126,"reward":-1.9153245687484741,"core":-0.5875182151794434,"cost":-0.04700538516044617,"kl":0.03857993893325329,"entropy":0.60124471783638},{"step":127,"reward":-1.9473146200180054,"core":-0.5423094630241394,"cost":-0.13141363859176636,"kl":0.03081320086494088,"entropy":0.4384761303663254},{"step":128,"reward":-1.1775342226028442,"core":-0.3512197434902191,"cost":0.0,"kl":0.035676886327564716,"entropy":0.5605151951313019},{"step":129,"reward":-2.066538095474243,"core":-0.5614450573921204,"cost":-0.16828922927379608,"kl":0.04229593649506569,"entropy":0.5154262781143188},{"step":130,"reward":-0.6208964586257935,"core":-0.13736015558242798,"cost":-0.010753609240055084,"kl":0.021586085204035044,"entropy":0.4930000379681587},{"step":131,"reward":-3.6641132831573486,"core":-1.0887359380722046,"cost":-0.35045066475868225,"kl":0.032551967073231936,"entropy":0.5043047145009041},{"step":132,"reward":-2.7382590770721436,"core":-0.8073033690452576,"cost":-0.23737406730651855,"kl":0.02295030513778329,"entropy":0.5104323923587799},{"step":133,"reward":-1.3705520629882812,"core":-0.4116494953632355,"cost":-0.005057236645370722,"kl":0.03511267714202404,"entropy":0.4885347932577133},{"step":134,"reward":-2.3960795402526855,"core":-0.7309104800224304,"cost":-0.06693489104509354,"kl":0.03251131111755967,"entropy":0.44254934787750244},{"step":135,"reward":-2.8656864166259766,"core":-0.8769223093986511,"cost":-0.11574289947748184,"kl":0.025062499567866325,"entropy":0.49659009277820587},{"step":136,"reward":-2.919987440109253,"core":-0.7381015419960022,"cost":-0.3779345750808716,"kl":0.027320904657244682,"entropy":0.4687683880329132},{"step":137,"reward":-0.9036582112312317,"core":-0.2196379452943802,"cost":-0.037703439593315125,"kl":0.029613880906254053,"entropy":0.511836089193821},{"step":138,"reward":-1.884840965270996,"core":-0.549620509147644,"cost":-0.051805246621370316,"kl":0.024108699522912502,"entropy":0.42922212928533554},{"step":139,"reward":-2.243663787841797,"core":-0.6598480343818665,"cost":-0.11739704757928848,"kl":0.03285122150555253,"entropy":0.5435660034418106},{"step":140,"reward":-1.277420163154602,"core":-0.3836840093135834,"cost":-0.03129367157816887,"kl":0.025515872985124588,"entropy":0.5213805139064789},{"step":141,"reward":-2.226731300354004,"core":-0.7016027569770813,"cost":-0.06295197457075119,"kl":0.026963074691593647,"entropy":0.5628555417060852},{"step":142,"reward":-2.228269338607788,"core":-0.6613473296165466,"cost":-0.12551794946193695,"kl":0.03327669622376561,"entropy":0.433190681040287},{"step":143,"reward":-1.830725073814392,"core":-0.5401341319084167,"cost":-0.060366351157426834,"kl":0.027597722597420216,"entropy":0.4732569456100464},{"step":144,"reward":-2.2086241245269775,"core":-0.5379993915557861,"cost":-0.18997378647327423,"kl":0.03664471302181482,"entropy":0.5110630393028259},{"step":145,"reward":-2.017554521560669,"core":-0.5587009787559509,"cost":-0.1831989735364914,"kl":0.026728822849690914,"entropy":0.43333493918180466},{"step":146,"reward":-2.1244077682495117,"core":-0.6032935976982117,"cost":-0.15767528116703033,"kl":0.02345361839979887,"entropy":0.47938090562820435},{"step":147,"reward":-1.3728915452957153,"core":-0.3895234167575836,"cost":-0.004173945635557175,"kl":0.02146235527470708,"entropy":0.48320654034614563},{"step":148,"reward":-1.6073317527770996,"core":-0.42604827880859375,"cost":-0.12361055612564087,"kl":0.02611095178872347,"entropy":0.47059765458106995},{"step":149,"reward":-1.7487821578979492,"core":-0.5281090140342712,"cost":-0.07059110701084137,"kl":0.0312846121378243,"entropy":0.5844947993755341},{"step":150,"reward":-1.8335590362548828,"core":-0.5362353324890137,"cost":-0.10346975922584534,"kl":0.04040884738788009,"entropy":0.49168475717306137},{"step":151,"reward":-2.3822829723358154,"core":-0.6949922442436218,"cost":-0.1729901283979416,"kl":0.023947790265083313,"entropy":0.5886896848678589},{"step":152,"reward":-2.2408206462860107,"core":-0.6201565861701965,"cost":-0.19074475765228271,"kl":0.02606105338782072,"entropy":0.5278346687555313},{"step":153,"reward":-1.9080390930175781,"core":-0.5744020938873291,"cost":-0.05066891387104988,"kl":0.025537286419421434,"entropy":0.4720648378133774},{"step":154,"reward":-1.6745270490646362,"core":-0.4883645474910736,"cost":-0.018216034397482872,"kl":0.021134566515684128,"entropy":0.5089857801795006},{"step":155,"reward":-1.0059798955917358,"core":-0.274768203496933,"cost":-0.008880587294697762,"kl":0.015276802936568856,"entropy":0.43552470952272415},{"step":156,"reward":-2.1396400928497314,"core":-0.6233047842979431,"cost":-0.14576946198940277,"kl":0.016568240942433476,"entropy":0.4706911966204643},{"step":157,"reward":-2.675687551498413,"core":-0.8125069737434387,"cost":-0.12517273426055908,"kl":0.028562662657350302,"entropy":0.4667792543768883},{"step":158,"reward":-2.3177988529205322,"core":-0.6835476756095886,"cost":-0.12810565531253815,"kl":0.017200627364218235,"entropy":0.5264142975211143},{"step":159,"reward":-1.2201286554336548,"core":-0.3515465259552002,"cost":-0.025284087285399437,"kl":0.025652569718658924,"entropy":0.5044577047228813},{"step":160,"reward":-1.2019404172897339,"core":-0.31491395831108093,"cost":-0.042347829788923264,"kl":0.025388538371771574,"entropy":0.45743072032928467},{"step":161,"reward":-2.0570335388183594,"core":-0.6091596484184265,"cost":-0.09287939220666885,"kl":0.017964005703106523,"entropy":0.42054642736911774},{"step":162,"reward":-1.5266224145889282,"core":-0.45457014441490173,"cost":-0.052976857870817184,"kl":0.028058908879756927,"entropy":0.5410914123058319},{"step":163,"reward":-1.7828830480575562,"core":-0.43386778235435486,"cost":-0.22945916652679443,"kl":0.030341610312461853,"entropy":0.5020367503166199},{"step":164,"reward":-1.8564438819885254,"core":-0.5773136019706726,"cost":-0.05009650066494942,"kl":0.031382736284285784,"entropy":0.46481815725564957},{"step":165,"reward":-1.5477218627929688,"core":-0.38629910349845886,"cost":-0.08358166366815567,"kl":0.026865076273679733,"entropy":0.5095436945557594},{"step":166,"reward":-1.969826579093933,"core":-0.5990589261054993,"cost":-0.10841678828001022,"kl":0.021595378406345844,"entropy":0.46350351721048355},{"step":167,"reward":-1.674239158630371,"core":-0.5049057006835938,"cost":-0.026268690824508667,"kl":0.025133201386779547,"entropy":0.416841559112072},{"step":168,"reward":-1.9462083578109741,"core":-0.5605883002281189,"cost":-0.11538509279489517,"kl":0.0300450106151402,"entropy":0.48599299788475037},{"step":169,"reward":-1.4728116989135742,"core":-0.45490503311157227,"cost":-0.019497143104672432,"kl":0.02193631650879979,"entropy":0.4705338776111603},{"step":170,"reward":-2.5837600231170654,"core":-0.7390691637992859,"cost":-0.16990332305431366,"kl":0.03042061533778906,"entropy":0.47242221236228943},{"step":171,"reward":-2.6883904933929443,"core":-0.8494067192077637,"cost":-0.07136038690805435,"kl":0.029681289568543434,"entropy":0.5114797577261925},{"step":172,"reward":-1.246322751045227,"core":-0.30157580971717834,"cost":-0.028734898194670677,"kl":0.03622463624924421,"entropy":0.46603407710790634},{"step":173,"reward":-2.2036330699920654,"core":-0.6520351767539978,"cost":-0.14334416389465332,"kl":0.029952417127788067,"entropy":0.46596287190914154},{"step":174,"reward":-1.7554548978805542,"core":-0.5037042498588562,"cost":-0.04818516969680786,"kl":0.024730269331485033,"entropy":0.36470842361450195},{"step":175,"reward":-2.1643941402435303,"core":-0.6732771396636963,"cost":-0.0877034068107605,"kl":0.05068644415587187,"entropy":0.48402200639247894},{"step":176,"reward":-2.153245210647583,"core":-0.6693282127380371,"cost":-0.04069645702838898,"kl":0.035793226677924395,"entropy":0.4801994040608406},{"step":177,"reward":-1.4314442873001099,"core":-0.38015270233154297,"cost":-0.0777333676815033,"kl":0.04730330128222704,"entropy":0.5685902833938599},{"step":178,"reward":-2.3462791442871094,"core":-0.657389223575592,"cost":-0.18793116509914398,"kl":0.04537553479894996,"entropy":0.4717562422156334},{"step":179,"reward":-2.393918037414551,"core":-0.7228301167488098,"cost":-0.12044110894203186,"kl":0.04284093156456947,"entropy":0.5572600066661835},{"step":180,"reward":-1.255611538887024,"core":-0.3326203525066376,"cost":-0.005507852882146835,"kl":0.03876721113920212,"entropy":0.4564656540751457},{"step":181,"reward":-2.419851779937744,"core":-0.6817237734794617,"cost":-0.19672615826129913,"kl":0.05667521059513092,"entropy":0.46553635597229004},{"step":182,"reward":-0.9657766819000244,"core":-0.2473093718290329,"cost":0.0,"kl":0.028367712628096342,"entropy":0.42731061577796936},{"step":183,"reward":-1.1400059461593628,"core":-0.32935571670532227,"cost":0.0,"kl":0.03673525433987379,"entropy":0.49691271781921387},{"step":184,"reward":-2.3560056686401367,"core":-0.6915225982666016,"cost":-0.12795107066631317,"kl":0.041093832813203335,"entropy":0.5657631605863571},{"step":185,"reward":-1.6890225410461426,"core":-0.49032068252563477,"cost":-0.030778750777244568,"kl":0.042909154668450356,"entropy":0.5176407247781754},{"step":186,"reward":-1.9343414306640625,"core":-0.566534698009491,"cost":-0.0880882516503334,"kl":0.03802277846261859,"entropy":0.3708833381533623},{"step":187,"reward":-0.12380354851484299,"core":0.131336972117424,"cost":-0.01438511535525322,"kl":0.04812741186469793,"entropy":0.49601513892412186},{"step":188,"reward":-1.1113072633743286,"core":-0.254010945558548,"cost":-0.05078675225377083,"kl":0.0692795580253005,"entropy":0.5304916799068451},{"step":189,"reward":-1.7150441408157349,"core":-0.45818808674812317,"cost":-0.12953434884548187,"kl":0.05767935607582331,"entropy":0.60445636510849},{"step":190,"reward":-2.0243637561798096,"core":-0.5851406455039978,"cost":-0.13256894052028656,"kl":0.037117659114301205,"entropy":0.4854714646935463},{"step":191,"reward":-3.1197662353515625,"core":-0.9421553611755371,"cost":-0.1722692996263504,"kl":0.06450201850384474,"entropy":0.455961138010025},{"step":192,"reward":-0.388564795255661,"core":0.03359111025929451,"cost":-0.01683473214507103,"kl":0.05158181115984917,"entropy":0.5605773329734802},{"step":193,"reward":-0.3875180184841156,"core":0.03604555130004883,"cost":-0.01354797650128603,"kl":0.05983384232968092,"entropy":0.4935913532972336},{"step":194,"reward":-1.9509159326553345,"core":-0.5429901480674744,"cost":-0.17987604439258575,"kl":0.05813014134764671,"entropy":0.5549500733613968},{"step":195,"reward":-1.0502501726150513,"core":-0.23711436986923218,"cost":-0.009264154359698296,"kl":0.05224758759140968,"entropy":0.5186235681176186},{"step":196,"reward":-1.5392494201660156,"core":-0.48308470845222473,"cost":-0.01849391683936119,"kl":0.06204497721046209,"entropy":0.5043714940547943},{"step":197,"reward":-1.2550541162490845,"core":-0.3532737195491791,"cost":-0.031406644731760025,"kl":0.04004588071256876,"entropy":0.49780215322971344},{"step":198,"reward":-1.583387851715088,"core":-0.4164937734603882,"cost":-0.03972742334008217,"kl":0.03436767542734742,"entropy":0.4553638845682144},{"step":199,"reward":-0.7696678042411804,"core":-0.15617313981056213,"cost":0.0,"kl":0.06517649721354246,"entropy":0.520336426794529},{"step":200,"reward":-1.4344863891601562,"core":-0.33689531683921814,"cost":-0.11571284383535385,"kl":0.05687208566814661,"entropy":0.5057035312056541},{"step":201,"reward":-1.5258077383041382,"core":-0.42921802401542664,"cost":-0.02129174768924713,"kl":0.06475574150681496,"entropy":0.5825115591287613},{"step":202,"reward":-2.0927534103393555,"core":-0.6251853108406067,"cost":-0.09923145920038223,"kl":0.0492718992754817,"entropy":0.5528493523597717},{"step":203,"reward":-1.725911021232605,"core":-0.4998174011707306,"cost":-0.031105542555451393,"kl":0.056659105233848095,"entropy":0.5371628850698471},{"step":204,"reward":-1.9981374740600586,"core":-0.6096516251564026,"cost":-0.05583278834819794,"kl":0.04948738403618336,"entropy":0.45860298722982407},{"step":205,"reward":-1.4173043966293335,"core":-0.4040694534778595,"cost":-0.0029031375888735056,"kl":0.03824925236403942,"entropy":0.5595489665865898},{"step":206,"reward":-1.5911775827407837,"core":-0.4659377336502075,"cost":-0.07140371203422546,"kl":0.04692132491618395,"entropy":0.615773469209671},{"step":207,"reward":-2.298707962036133,"core":-0.6672770380973816,"cost":-0.12647511065006256,"kl":0.04824332194402814,"entropy":0.5201610997319221},{"step":208,"reward":-1.7032057046890259,"core":-0.5222143530845642,"cost":-0.07624126225709915,"kl":0.04911516048014164,"entropy":0.5369958207011223},{"step":209,"reward":-1.6001067161560059,"core":-0.4392869472503662,"cost":-0.1129433810710907,"kl":0.05052674189209938,"entropy":0.4921571910381317},{"step":210,"reward":-0.9047102332115173,"core":-0.1728886365890503,"cost":-0.01963864266872406,"kl":0.0433760667219758,"entropy":0.5137942135334015},{"step":211,"reward":-1.8073596954345703,"core":-0.5684123635292053,"cost":-2.327910351596074e-06,"kl":0.0441995020955801,"entropy":0.5729610621929169},{"step":212,"reward":-2.103231430053711,"core":-0.6004325747489929,"cost":-0.06544099748134613,"kl":0.07076869532465935,"entropy":0.4539628028869629},{"step":213,"reward":-1.000654935836792,"core":-0.23551297187805176,"cost":-0.07766187191009521,"kl":0.06244018115103245,"entropy":0.4962010607123375},{"step":214,"reward":-1.1072791814804077,"core":-0.27885639667510986,"cost":-0.02636067569255829,"kl":0.053009433671832085,"entropy":0.4954090267419815},{"step":215,"reward":-1.5821752548217773,"core":-0.48625972867012024,"cost":-0.029174556955695152,"kl":0.04462528182193637,"entropy":0.5615019202232361},{"step":216,"reward":-2.504409074783325,"core":-0.7014467120170593,"cost":-0.21479369699954987,"kl":0.060943651013076305,"entropy":0.5490754395723343},{"step":217,"reward":-2.6264307498931885,"core":-0.7150489687919617,"cost":-0.20654459297657013,"kl":0.0470743328332901,"entropy":0.4154406785964966},{"step":218,"reward":-1.7097620964050293,"core":-0.4527494013309479,"cost":-0.0785321593284607,"kl":0.04615671560168266,"entropy":0.528721034526825},{"step":219,"reward":-1.4577537775039673,"core":-0.3929692208766937,"cost":-0.054233551025390625,"kl":0.07019221410155296,"entropy":0.5458190217614174},{"step":220,"reward":-2.8897202014923096,"core":-0.8496911525726318,"cost":-0.24385206401348114,"kl":0.04137931298464537,"entropy":0.47431690990924835},{"step":221,"reward":-1.6067436933517456,"core":-0.4609549045562744,"cost":-0.03541065379977226,"kl":0.05737217701971531,"entropy":0.49155669659376144},{"step":222,"reward":-1.7378209829330444,"core":-0.49888288974761963,"cost":-0.13993822038173676,"kl":0.0768469450995326,"entropy":0.5545647442340851},{"step":223,"reward":-2.075338363647461,"core":-0.6246170997619629,"cost":-0.07160555571317673,"kl":0.05969944782555103,"entropy":0.48852119594812393},{"step":224,"reward":-1.8458722829818726,"core":-0.5463187098503113,"cost":-0.05721402168273926,"kl":0.06064356304705143,"entropy":0.5050204917788506},{"step":225,"reward":-1.3025339841842651,"core":-0.3647746741771698,"cost":-0.011439907364547253,"kl":0.05681648477911949,"entropy":0.578161895275116},{"step":226,"reward":-2.5831127166748047,"core":-0.7661369442939758,"cost":-0.14027169346809387,"kl":0.05699076224118471,"entropy":0.4324440732598305},{"step":227,"reward":-1.5703048706054688,"core":-0.42319929599761963,"cost":-0.07237759977579117,"kl":0.056216864846646786,"entropy":0.6071477383375168},{"step":228,"reward":-1.833608627319336,"core":-0.45587727427482605,"cost":-0.09406688809394836,"kl":0.05996490549296141,"entropy":0.450373999774456},{"step":229,"reward":-0.49191156029701233,"core":0.02191823720932007,"cost":-0.006352432072162628,"kl":0.09455376863479614,"entropy":0.532821536064148},{"step":230,"reward":-1.4489601850509644,"core":-0.3849440813064575,"cost":-0.07659932971000671,"kl":0.07443248014897108,"entropy":0.5372883528470993},{"step":231,"reward":-1.108323335647583,"core":-0.29512959718704224,"cost":0.0,"kl":0.0516680683940649,"entropy":0.49207162111997604},{"step":232,"reward":-1.337180495262146,"core":-0.3872688114643097,"cost":-0.0798022523522377,"kl":0.07999767176806927,"entropy":0.5009090155363083},{"step":233,"reward":-1.9682331085205078,"core":-0.5319228768348694,"cost":-0.18712015450000763,"kl":0.08665361441671848,"entropy":0.5674163103103638},{"step":234,"reward":-4.011560440063477,"core":-1.2081876993179321,"cost":-0.3641993999481201,"kl":0.0667739873751998,"entropy":0.6235606074333191},{"step":235,"reward":-2.018263578414917,"core":-0.5847188830375671,"cost":-0.12462272495031357,"kl":0.07028352189809084,"entropy":0.5738946199417114},{"step":236,"reward":-3.2861177921295166,"core":-0.9633855819702148,"cost":-0.3202749192714691,"kl":0.11395158804953098,"entropy":0.6103540062904358},{"step":237,"reward":-0.11251429468393326,"core":0.0551840178668499,"cost":0.0,"kl":0.10995684377849102,"entropy":0.5655318200588226},{"step":238,"reward":-0.37735262513160706,"core":0.06505685299634933,"cost":-0.015368160791695118,"kl":0.062208835035562515,"entropy":0.5279335528612137},{"step":239,"reward":-2.0993826389312744,"core":-0.6157346367835999,"cost":-0.14357000589370728,"kl":0.06392673589289188,"entropy":0.4394459053874016},{"step":240,"reward":-1.4542665481567383,"core":-0.3971695899963379,"cost":-0.06421653181314468,"kl":0.062098992988467216,"entropy":0.4798080176115036},{"step":241,"reward":-0.9055662155151367,"core":-0.20648272335529327,"cost":-0.04004202410578728,"kl":0.08499390073120594,"entropy":0.5648714005947113},{"step":242,"reward":-1.6719530820846558,"core":-0.5047011375427246,"cost":-0.03033790923655033,"kl":0.07807615771889687,"entropy":0.582391545176506},{"step":243,"reward":-1.9273992776870728,"core":-0.6033204197883606,"cost":-0.009895884431898594,"kl":0.06396343372762203,"entropy":0.4698811247944832},{"step":244,"reward":-3.246030807495117,"core":-0.9823002815246582,"cost":-0.19340725243091583,"kl":0.11055816151201725,"entropy":0.5119816213846207},{"step":245,"reward":-0.9369518160820007,"core":-0.2570496201515198,"cost":-0.05478040874004364,"kl":0.06938374787569046,"entropy":0.46307532489299774},{"step":246,"reward":-0.36443856358528137,"core":-0.023324409499764442,"cost":0.0,"kl":0.06620158441364765,"entropy":0.4498114734888077},{"step":247,"reward":-0.47754791378974915,"core":-0.03282344341278076,"cost":-0.015844160690903664,"kl":0.06781987845897675,"entropy":0.5260661095380783},{"step":248,"reward":-1.0239381790161133,"core":-0.1889701932668686,"cost":-0.08984308689832687,"kl":0.07804948650300503,"entropy":0.48842255771160126},{"step":249,"reward":-1.9954047203063965,"core":-0.5412383079528809,"cost":-0.13955539464950562,"kl":0.10359455272555351,"entropy":0.6143104434013367},{"step":250,"reward":-1.654894232749939,"core":-0.49335089325904846,"cost":-0.05389728024601936,"kl":0.05446959938853979,"entropy":0.5096258446574211},{"step":251,"reward":-1.7891329526901245,"core":-0.5182462334632874,"cost":-0.06258958578109741,"kl":0.07721785642206669,"entropy":0.5974310040473938},{"step":252,"reward":-1.1306500434875488,"core":-0.317882239818573,"cost":-0.0004015765152871609,"kl":0.06705318298190832,"entropy":0.43821651488542557},{"step":253,"reward":-1.251397728919983,"core":-0.33988919854164124,"cost":-0.05025247111916542,"kl":0.08018677309155464,"entropy":0.4932965636253357},{"step":254,"reward":-2.112520217895508,"core":-0.5904767513275146,"cost":-0.15453441441059113,"kl":0.07250099908560514,"entropy":0.5844633877277374},{"step":255,"reward":-1.3821302652359009,"core":-0.33130231499671936,"cost":-0.08044234663248062,"kl":0.08274638094007969,"entropy":0.506053552031517},{"step":256,"reward":-2.1503913402557373,"core":-0.5954573750495911,"cost":-0.18365561962127686,"kl":0.08757438324391842,"entropy":0.5813568383455276},{"step":257,"reward":-1.5327978134155273,"core":-0.4165514409542084,"cost":-0.09929440170526505,"kl":0.0932882372289896,"entropy":0.5060934200882912},{"step":258,"reward":-0.7510437965393066,"core":-0.17139707505702972,"cost":-0.04632139578461647,"kl":0.10101534053683281,"entropy":0.5816768407821655},{"step":259,"reward":-1.5189796686172485,"core":-0.44026756286621094,"cost":-0.05268881842494011,"kl":0.09286579303443432,"entropy":0.5594303831458092},{"step":260,"reward":-1.1003895998001099,"core":-0.29437950253486633,"cost":-0.0378737598657608,"kl":0.08853121288120747,"entropy":0.49781888723373413},{"step":261,"reward":-1.9784183502197266,"core":-0.5976839661598206,"cost":-0.027670398354530334,"kl":0.10066254623234272,"entropy":0.479342982172966},{"step":262,"reward":-2.123394727706909,"core":-0.6369369626045227,"cost":-0.12609709799289703,"kl":0.07468047551810741,"entropy":0.5077934414148331},{"step":263,"reward":-0.5936346650123596,"core":-0.1394737958908081,"cost":-0.016977230086922646,"kl":0.0659872516989708,"entropy":0.533147320151329},{"step":264,"reward":-1.18816339969635,"core":-0.30992382764816284,"cost":-0.0731060728430748,"kl":0.0854877820238471,"entropy":0.5060160756111145},{"step":265,"reward":-2.5755252838134766,"core":-0.7846878170967102,"cost":-0.11136338859796524,"kl":0.05239597614854574,"entropy":0.50530756264925},{"step":266,"reward":-1.4288357496261597,"core":-0.40536415576934814,"cost":-0.04161857068538666,"kl":0.06025807838886976,"entropy":0.47265564650297165},{"step":267,"reward":-1.7555111646652222,"core":-0.4852429926395416,"cost":-0.11316099762916565,"kl":0.09717113245278597,"entropy":0.5630271732807159},{"step":268,"reward":-1.6746171712875366,"core":-0.4657115936279297,"cost":-0.03762846440076828,"kl":0.08370138145983219,"entropy":0.5813184380531311},{"step":269,"reward":-0.9367876648902893,"core":-0.26375994086265564,"cost":-0.005121144000440836,"kl":0.0944476630538702,"entropy":0.5491864234209061},{"step":270,"reward":-1.0482473373413086,"core":-0.25682321190834045,"cost":-0.01853948086500168,"kl":0.07689865678548813,"entropy":0.5443193167448044},{"step":271,"reward":-2.115629196166992,"core":-0.6314029693603516,"cost":-0.07540103048086166,"kl":0.11459728982299566,"entropy":0.47042009979486465},{"step":272,"reward":-1.303457260131836,"core":-0.2911817133426666,"cost":-0.05801577493548393,"kl":0.09240742586553097,"entropy":0.505341924726963},{"step":273,"reward":-2.9704341888427734,"core":-0.8403204083442688,"cost":-0.34097322821617126,"kl":0.07089098263531923,"entropy":0.44426295161247253},{"step":274,"reward":-1.3423243761062622,"core":-0.37428018450737,"cost":-0.021308697760105133,"kl":0.05114059057086706,"entropy":0.4563888609409332},{"step":275,"reward":-1.8132957220077515,"core":-0.5200125575065613,"cost":-0.058715809136629105,"kl":0.08658339455723763,"entropy":0.5403572767972946},{"step":276,"reward":-0.4043715298175812,"core":-0.03343624994158745,"cost":-0.015042878687381744,"kl":0.0680309534072876,"entropy":0.4537228047847748},{"step":277,"reward":-2.523029327392578,"core":-0.7606863379478455,"cost":-0.10443871468305588,"kl":0.06453497149050236,"entropy":0.44226909428834915},{"step":278,"reward":-1.1820684671401978,"core":-0.26880696415901184,"cost":-0.05529635027050972,"kl":0.09580239094793797,"entropy":0.5498565286397934},{"step":279,"reward":-1.2845131158828735,"core":-0.3159850537776947,"cost":-0.12998291850090027,"kl":0.07595380768179893,"entropy":0.5640584826469421},{"step":280,"reward":-1.6471818685531616,"core":-0.45563292503356934,"cost":-0.022261789068579674,"kl":0.08098606020212173,"entropy":0.5780359357595444},{"step":281,"reward":-1.1634312868118286,"core":-0.275915265083313,"cost":-0.04134027287364006,"kl":0.08512436598539352,"entropy":0.5587577223777771},{"step":282,"reward":-1.555533766746521,"core":-0.46636152267456055,"cost":-0.007320210803300142,"kl":0.06398975197225809,"entropy":0.46974650025367737},{"step":283,"reward":-1.3914095163345337,"core":-0.35592153668403625,"cost":-0.046081069856882095,"kl":0.06846574135124683,"entropy":0.5051184967160225},{"step":284,"reward":-1.4580408334732056,"core":-0.43344342708587646,"cost":-0.05365089699625969,"kl":0.08186055906116962,"entropy":0.5544497966766357},{"step":285,"reward":-1.4035781621932983,"core":-0.3611019551753998,"cost":-0.08828537911176682,"kl":0.09351494163274765,"entropy":0.5502182841300964},{"step":286,"reward":-1.2210139036178589,"core":-0.19989438354969025,"cost":-0.10774332284927368,"kl":0.08258733712136745,"entropy":0.5147165358066559},{"step":287,"reward":-1.4889217615127563,"core":-0.3292129933834076,"cost":-0.14724253118038177,"kl":0.062180256471037865,"entropy":0.5440818071365356},{"step":288,"reward":-1.8295551538467407,"core":-0.5227425694465637,"cost":-0.07428719848394394,"kl":0.13876168988645077,"entropy":0.5642928630113602},{"step":289,"reward":-1.6389662027359009,"core":-0.48078417778015137,"cost":-0.05491827055811882,"kl":0.09989194571971893,"entropy":0.5410897061228752},{"step":290,"reward":-1.2658913135528564,"core":-0.3206098973751068,"cost":-0.022372471168637276,"kl":0.1059187576174736,"entropy":0.4847605749964714},{"step":291,"reward":-0.4796532690525055,"core":-0.07439160346984863,"cost":0.0,"kl":0.098387710750103,"entropy":0.5475919544696808},{"step":292,"reward":-1.44244384765625,"core":-0.38175106048583984,"cost":-0.06499803811311722,"kl":0.07256211154162884,"entropy":0.5426244139671326},{"step":293,"reward":-1.8582748174667358,"core":-0.5083574652671814,"cost":-0.15780465304851532,"kl":0.11394314654171467,"entropy":0.6324319690465927},{"step":294,"reward":-1.9760767221450806,"core":-0.5915395617485046,"cost":-0.09429561346769333,"kl":0.07592350244522095,"entropy":0.4290265589952469},{"step":295,"reward":-0.7850744724273682,"core":-0.20858335494995117,"cost":0.0,"kl":0.08488570060580969,"entropy":0.46959274262189865},{"step":296,"reward":-1.1358393430709839,"core":-0.2915595471858978,"cost":-0.032638341188430786,"kl":0.06602549739181995,"entropy":0.500327430665493},{"step":297,"reward":-1.0104987621307373,"core":-0.2535550594329834,"cost":-0.06670267134904861,"kl":0.06803743727505207,"entropy":0.5497378408908844},{"step":298,"reward":-2.52364182472229,"core":-0.7579444050788879,"cost":-0.17910832166671753,"kl":0.08028428629040718,"entropy":0.4889873042702675},{"step":299,"reward":-0.826455295085907,"core":-0.1928756982088089,"cost":-0.02784380130469799,"kl":0.07401913031935692,"entropy":0.5352674424648285},{"step":300,"reward":-2.158764600753784,"core":-0.5677985548973083,"cost":-0.19328832626342773,"kl":0.0762348435819149,"entropy":0.5717165321111679},{"step":301,"reward":-1.4781211614608765,"core":-0.4291737973690033,"cost":-0.0578371100127697,"kl":0.07913618348538876,"entropy":0.5640188753604889},{"step":302,"reward":-1.0807746648788452,"core":-0.2797916829586029,"cost":-0.0009576538577675819,"kl":0.06285333912819624,"entropy":0.5562853068113327},{"step":303,"reward":-2.174326181411743,"core":-0.6613271236419678,"cost":-0.11867024749517441,"kl":0.07893064804375172,"entropy":0.5450351685285568},{"step":304,"reward":-2.31404709815979,"core":-0.6750467419624329,"cost":-0.10200333595275879,"kl":0.07224719040095806,"entropy":0.47165336459875107},{"step":305,"reward":-0.04759705066680908,"core":0.23978972434997559,"cost":-0.04964733123779297,"kl":0.13157262466847897,"entropy":0.4490516781806946},{"step":306,"reward":-1.6321066617965698,"core":-0.44977331161499023,"cost":-0.07619606703519821,"kl":0.05568886827677488,"entropy":0.4632858633995056},{"step":307,"reward":-1.1549090147018433,"core":-0.2946462333202362,"cost":-0.0019398547010496259,"kl":0.06794864218682051,"entropy":0.4685700610280037},{"step":308,"reward":-1.1650655269622803,"core":-0.25556761026382446,"cost":-0.06357206404209137,"kl":0.06703111156821251,"entropy":0.5158368498086929},{"step":309,"reward":-1.6491832733154297,"core":-0.43475255370140076,"cost":-0.15562790632247925,"kl":0.09141578525304794,"entropy":0.54827581346035},{"step":310,"reward":-0.8109005093574524,"core":-0.12898023426532745,"cost":-0.04693597927689552,"kl":0.08314559981226921,"entropy":0.5160216391086578},{"step":311,"reward":-0.778308629989624,"core":-0.1729750782251358,"cost":-0.0025699418038129807,"kl":0.06708116549998522,"entropy":0.4905237928032875},{"step":312,"reward":-0.7171952724456787,"core":-0.15992631018161774,"cost":-0.027824020013213158,"kl":0.06381923984736204,"entropy":0.4950517565011978},{"step":313,"reward":-3.1119766235351562,"core":-0.9618692994117737,"cost":-0.14890548586845398,"kl":0.057248231023550034,"entropy":0.4726683124899864},{"step":314,"reward":-1.8225799798965454,"core":-0.560977041721344,"cost":-0.04312142729759216,"kl":0.05649279523640871,"entropy":0.5277990773320198},{"step":315,"reward":-2.2149603366851807,"core":-0.6766635775566101,"cost":-0.09143122285604477,"kl":0.05386597849428654,"entropy":0.48929983377456665},{"step":316,"reward":-2.549936056137085,"core":-0.7596996426582336,"cost":-0.18749882280826569,"kl":0.0447469144128263,"entropy":0.4684395343065262},{"step":317,"reward":-1.4169336557388306,"core":-0.42343220114707947,"cost":-0.004070739261806011,"kl":0.06997896172106266,"entropy":0.3969884067773819},{"step":318,"reward":-1.2119017839431763,"core":-0.3146835267543793,"cost":-0.0980549082159996,"kl":0.07107395585626364,"entropy":0.5011698305606842},{"step":319,"reward":-0.8980665802955627,"core":-0.11939411610364914,"cost":-0.018394501879811287,"kl":0.0690997838973999,"entropy":0.4885415658354759},{"step":320,"reward":-2.03187894821167,"core":-0.6250798106193542,"cost":-0.058444440364837646,"kl":0.07566027715802193,"entropy":0.4709971845149994},{"step":321,"reward":-1.494912028312683,"core":-0.43200698494911194,"cost":-0.07920663803815842,"kl":0.06345850601792336,"entropy":0.5218782424926758},{"step":322,"reward":-2.3061952590942383,"core":-0.6739022135734558,"cost":-0.13592936098575592,"kl":0.07373396772891283,"entropy":0.5571757555007935},{"step":323,"reward":-1.008402705192566,"core":-0.20473188161849976,"cost":-0.07200061529874802,"kl":0.07890155352652073,"entropy":0.5555379465222359},{"step":324,"reward":0.2849259376525879,"core":0.2936612069606781,"cost":0.0,"kl":0.06545003969222307,"entropy":0.44179727137088776},{"step":325,"reward":-0.6071879267692566,"core":-0.08553718775510788,"cost":-0.03191528096795082,"kl":0.07012765295803547,"entropy":0.5282087177038193},{"step":326,"reward":-1.4773359298706055,"core":-0.41434797644615173,"cost":-0.04257172718644142,"kl":0.07710431888699532,"entropy":0.5376546233892441},{"step":327,"reward":-1.1711162328720093,"core":-0.3293016254901886,"cost":-0.040109943598508835,"kl":0.061071367003023624,"entropy":0.5270922034978867},{"step":328,"reward":-0.9610443711280823,"core":-0.24358196556568146,"cost":-0.007784134242683649,"kl":0.039312505163252354,"entropy":0.48148998618125916},{"step":329,"reward":-0.4921362102031708,"core":-0.03185568377375603,"cost":-0.07983087003231049,"kl":0.08823778852820396,"entropy":0.5139842256903648},{"step":330,"reward":-1.4585866928100586,"core":-0.36440959572792053,"cost":-0.08019015938043594,"kl":0.058679102919995785,"entropy":0.46964606642723083},{"step":331,"reward":-1.5626176595687866,"core":-0.419893354177475,"cost":-0.07258854061365128,"kl":0.08278321847319603,"entropy":0.527095340192318},{"step":332,"reward":-2.611119270324707,"core":-0.8035404682159424,"cost":-0.08231223374605179,"kl":0.08038883283734322,"entropy":0.4431702420115471},{"step":333,"reward":-0.21055591106414795,"core":0.06030351296067238,"cost":-0.017457779496908188,"kl":0.07083715964108706,"entropy":0.48095785081386566},{"step":334,"reward":-1.1898226737976074,"core":-0.3216598927974701,"cost":0.0,"kl":0.06850283220410347,"entropy":0.5090543925762177},{"step":335,"reward":-0.9521072506904602,"core":-0.24192695319652557,"cost":-0.02082035318017006,"kl":0.06129807233810425,"entropy":0.4449247568845749},{"step":336,"reward":-1.430171012878418,"core":-0.41440853476524353,"cost":-0.033281341195106506,"kl":0.05251995101571083,"entropy":0.5027471408247948},{"step":337,"reward":-1.9074658155441284,"core":-0.5083150267601013,"cost":-0.12497812509536743,"kl":0.08019577153027058,"entropy":0.5069059655070305},{"step":338,"reward":-2.7044899463653564,"core":-0.7992095947265625,"cost":-0.20088434219360352,"kl":0.11128082685172558,"entropy":0.5199423804879189},{"step":339,"reward":-1.2958348989486694,"core":-0.3574056625366211,"cost":-0.022606821730732918,"kl":0.08220200426876545,"entropy":0.4935293644666672},{"step":340,"reward":-1.1677926778793335,"core":-0.27268287539482117,"cost":-0.06607010215520859,"kl":0.0945141352713108,"entropy":0.5123341381549835},{"step":341,"reward":-0.49849510192871094,"core":-0.03633736073970795,"cost":0.0,"kl":0.07747702859342098,"entropy":0.3676202893257141},{"step":342,"reward":-0.6611848473548889,"core":-0.07110197097063065,"cost":-0.032789405435323715,"kl":0.09448019228875637,"entropy":0.49053288251161575},{"step":343,"reward":-0.6623417139053345,"core":-0.09993701428174973,"cost":-0.03469066321849823,"kl":0.08922134339809418,"entropy":0.5183812007308006},{"step":344,"reward":-0.9770529866218567,"core":-0.19227559864521027,"cost":-0.06163153424859047,"kl":0.08743950352072716,"entropy":0.5089007765054703},{"step":345,"reward":-0.6114751100540161,"core":-0.13867218792438507,"cost":0.0,"kl":0.06703065522015095,"entropy":0.48440171033143997},{"step":346,"reward":-1.4527376890182495,"core":-0.40127840638160706,"cost":-0.0770094022154808,"kl":0.09080224856734276,"entropy":0.45176398754119873},{"step":347,"reward":-0.7191283106803894,"core":-0.13174569606781006,"cost":-0.0196735430508852,"kl":0.08218813687562943,"entropy":0.45837948471307755},{"step":348,"reward":-2.6328115463256836,"core":-0.7832026481628418,"cost":-0.17395730316638947,"kl":0.0724088167771697,"entropy":0.5246376097202301},{"step":349,"reward":-0.8056984543800354,"core":-0.09859082847833633,"cost":-0.031488582491874695,"kl":0.08101021125912666,"entropy":0.4512298107147217},{"step":350,"reward":-2.312795400619507,"core":-0.7010132670402527,"cost":-0.12276751548051834,"kl":0.09019759111106396,"entropy":0.5364428162574768},{"step":351,"reward":0.07971138507127762,"core":0.1378469616174698,"cost":0.0,"kl":0.07823140546679497,"entropy":0.5554806143045425},{"step":352,"reward":-1.9614022970199585,"core":-0.5821466445922852,"cost":-0.10841532796621323,"kl":0.09551839530467987,"entropy":0.5683549493551254},{"step":353,"reward":-1.4077895879745483,"core":-0.4138886034488678,"cost":0.0,"kl":0.06623862683773041,"entropy":0.41712480038404465},{"step":354,"reward":-0.9579863548278809,"core":-0.253684401512146,"cost":-0.023681996390223503,"kl":0.08436302933841944,"entropy":0.48799359798431396},{"step":355,"reward":-1.1150776147842407,"core":-0.26421526074409485,"cost":-0.10366395860910416,"kl":0.052322668954730034,"entropy":0.39343807101249695},{"step":356,"reward":-0.4660421311855316,"core":-0.03598976507782936,"cost":0.0,"kl":0.07606211863458157,"entropy":0.5105770081281662},{"step":357,"reward":-1.3220676183700562,"core":-0.39421629905700684,"cost":-0.055277060717344284,"kl":0.08992809243500233,"entropy":0.5077780485153198},{"step":358,"reward":-2.03849720954895,"core":-0.5424208641052246,"cost":-0.20044958591461182,"kl":0.08164850436151028,"entropy":0.47032709419727325},{"step":359,"reward":-0.016841836273670197,"core":0.0817905142903328,"cost":0.0,"kl":0.035721730906516314,"entropy":0.40256332606077194},{"step":360,"reward":-1.660988450050354,"core":-0.4296962022781372,"cost":-0.1435176134109497,"kl":0.1051035039126873,"entropy":0.4748918116092682},{"step":361,"reward":-2.013897180557251,"core":-0.5460525155067444,"cost":-0.15355102717876434,"kl":0.06502903625369072,"entropy":0.46345674991607666},{"step":362,"reward":-1.2187820672988892,"core":-0.26553067564964294,"cost":-0.1464352160692215,"kl":0.0863938182592392,"entropy":0.5467359274625778},{"step":363,"reward":-0.6804326176643372,"core":-0.11340422183275223,"cost":-0.02280198596417904,"kl":0.11610491201281548,"entropy":0.5028996840119362},{"step":364,"reward":-2.692443609237671,"core":-0.7913562655448914,"cost":-0.20465803146362305,"kl":0.07134660985320807,"entropy":0.47420167177915573},{"step":365,"reward":-1.1096264123916626,"core":-0.2539112865924835,"cost":-0.06469281762838364,"kl":0.1207820437848568,"entropy":0.5412652492523193},{"step":366,"reward":-0.9274803996086121,"core":-0.24975548684597015,"cost":-0.010860849171876907,"kl":0.04601137572899461,"entropy":0.5047109350562096},{"step":367,"reward":-0.8533646464347839,"core":-0.18102145195007324,"cost":-0.05131431296467781,"kl":0.08944064285606146,"entropy":0.4632454290986061},{"step":368,"reward":-0.7252252697944641,"core":-0.05812373757362366,"cost":-0.058151621371507645,"kl":0.07561051845550537,"entropy":0.4723437875509262},{"step":369,"reward":-0.3571193218231201,"core":0.008481137454509735,"cost":0.0,"kl":0.07329977303743362,"entropy":0.5065039470791817},{"step":370,"reward":-1.2976182699203491,"core":-0.3793956935405731,"cost":-1.7164027667604387e-05,"kl":0.0718408701941371,"entropy":0.4596499502658844},{"step":371,"reward":-1.2572299242019653,"core":-0.36030492186546326,"cost":-0.043274153023958206,"kl":0.11007765494287014,"entropy":0.4952651783823967},{"step":372,"reward":-1.3402100801467896,"core":-0.37844118475914,"cost":-0.04912780597805977,"kl":0.11352895945310593,"entropy":0.5087628811597824},{"step":373,"reward":-3.115980863571167,"core":-0.9449427723884583,"cost":-0.22711181640625,"kl":0.054256098344922066,"entropy":0.42769143730401993},{"step":374,"reward":-0.4717280864715576,"core":-0.0205575879663229,"cost":-0.022001588717103004,"kl":0.07862803060561419,"entropy":0.4320860654115677},{"step":375,"reward":-2.283003568649292,"core":-0.6282913088798523,"cost":-0.25499391555786133,"kl":0.07743979152292013,"entropy":0.5241498500108719},{"step":376,"reward":-0.789438784122467,"core":-0.15661156177520752,"cost":-0.029149338603019714,"kl":0.06877179816365242,"entropy":0.49792449176311493},{"step":377,"reward":-1.207088828086853,"core":-0.32092154026031494,"cost":-0.0348355770111084,"kl":0.07389050908386707,"entropy":0.46694493293762207},{"step":378,"reward":-1.6492677927017212,"core":-0.46499142050743103,"cost":-0.062494587153196335,"kl":0.08024671953171492,"entropy":0.4637078121304512},{"step":379,"reward":-1.3947452306747437,"core":-0.3889031708240509,"cost":-0.04723663255572319,"kl":0.08466812036931515,"entropy":0.4524303674697876},{"step":380,"reward":-1.538422703742981,"core":-0.4175487458705902,"cost":-0.10389244556427002,"kl":0.11804446205496788,"entropy":0.5122064426541328},{"step":381,"reward":-0.4797835052013397,"core":-0.0013791719684377313,"cost":-0.029796751216053963,"kl":0.0848456621170044,"entropy":0.4656149670481682},{"step":382,"reward":0.22722755372524261,"core":0.23825573921203613,"cost":-0.03254430368542671,"kl":0.0740312859416008,"entropy":0.5082276314496994},{"step":383,"reward":-1.1960219144821167,"core":-0.32301440834999084,"cost":-0.06677663326263428,"kl":0.0713585764169693,"entropy":0.6004244610667229},{"step":384,"reward":0.15007339417934418,"core":0.15965867042541504,"cost":-0.03532973304390907,"kl":0.10155196487903595,"entropy":0.37839680910110474},{"step":385,"reward":-1.6939274072647095,"core":-0.48192358016967773,"cost":-0.09059075266122818,"kl":0.08313305862247944,"entropy":0.5174036398530006},{"step":386,"reward":-0.9003010392189026,"core":-0.15630872547626495,"cost":-0.031184589490294456,"kl":0.06666705664247274,"entropy":0.41926439851522446},{"step":387,"reward":-0.981864869594574,"core":-0.20083989202976227,"cost":-0.020068597048521042,"kl":0.07989169284701347,"entropy":0.4505435824394226},{"step":388,"reward":-1.6411558389663696,"core":-0.45177388191223145,"cost":-0.0829806849360466,"kl":0.09516195207834244,"entropy":0.48550552129745483},{"step":389,"reward":-0.540848433971405,"core":-0.13024552166461945,"cost":0.0,"kl":0.08220209460705519,"entropy":0.5364303141832352},{"step":390,"reward":-1.385790467262268,"core":-0.4045874774456024,"cost":-0.01598871499300003,"kl":0.09178275242447853,"entropy":0.47303294390439987},{"step":391,"reward":-0.6476923823356628,"core":-0.14718396961688995,"cost":-0.012185267172753811,"kl":0.07727831229567528,"entropy":0.5147607624530792},{"step":392,"reward":-2.796231508255005,"core":-0.8472391963005066,"cost":-0.13599930703639984,"kl":0.05907969316467643,"entropy":0.49506277590990067},{"step":393,"reward":-0.6371275782585144,"core":-0.11025408655405045,"cost":-0.025144392624497414,"kl":0.10814807936549187,"entropy":0.602827861905098},{"step":394,"reward":-1.1767593622207642,"core":-0.3198675215244293,"cost":-0.0744878500699997,"kl":0.07266918011009693,"entropy":0.5373873561620712},{"step":395,"reward":-0.8240408897399902,"core":-0.16341663897037506,"cost":-0.0341157466173172,"kl":0.08190000988543034,"entropy":0.5025494769215584},{"step":396,"reward":-1.1844862699508667,"core":-0.2254803627729416,"cost":-0.12448481470346451,"kl":0.080117616802454,"entropy":0.4259539097547531},{"step":397,"reward":-1.6252561807632446,"core":-0.43762311339378357,"cost":-0.07539565116167068,"kl":0.07947770319879055,"entropy":0.5487103909254074},{"step":398,"reward":-1.440515160560608,"core":-0.4395579993724823,"cost":-0.02632756531238556,"kl":0.09112974256277084,"entropy":0.5043728724122047},{"step":399,"reward":-0.9079292416572571,"core":-0.21860457956790924,"cost":-0.018455790355801582,"kl":0.07275571208447218,"entropy":0.4847797676920891},{"step":400,"reward":-0.8391215801239014,"core":-0.1727462261915207,"cost":-0.002612331649288535,"kl":0.09450438432395458,"entropy":0.47618452459573746},{"step":401,"reward":-2.0853068828582764,"core":-0.5968193411827087,"cost":-0.1527976244688034,"kl":0.11621525697410107,"entropy":0.5936218649148941},{"step":402,"reward":-1.1765121221542358,"core":-0.19356656074523926,"cost":-0.1612679660320282,"kl":0.07964665442705154,"entropy":0.5260930731892586},{"step":403,"reward":-0.6537521481513977,"core":-0.15802286565303802,"cost":-0.001704764668829739,"kl":0.045987196266651154,"entropy":0.4824412539601326},{"step":404,"reward":-2.515122413635254,"core":-0.7723521590232849,"cost":-0.08756276220083237,"kl":0.08797067403793335,"entropy":0.5696863085031509},{"step":405,"reward":-0.001451382995583117,"core":0.16350580751895905,"cost":-0.010145132429897785,"kl":0.09969648905098438,"entropy":0.5366759151220322},{"step":406,"reward":-0.5961540341377258,"core":-0.08289613574743271,"cost":-0.015428014099597931,"kl":0.09187102783471346,"entropy":0.5347377359867096},{"step":407,"reward":-1.4150872230529785,"core":-0.3736483156681061,"cost":-0.10465869307518005,"kl":0.07702436670660973,"entropy":0.4874034747481346},{"step":408,"reward":-1.0002233982086182,"core":-0.2374693900346756,"cost":-0.0686124861240387,"kl":0.117220439016819,"entropy":0.5796336680650711},{"step":409,"reward":-1.8248621225357056,"core":-0.5384307503700256,"cost":-0.027936778962612152,"kl":0.0738762617111206,"entropy":0.5250311344861984},{"step":410,"reward":-0.9311577677726746,"core":-0.17911343276500702,"cost":-0.024478064849972725,"kl":0.10004427190870047,"entropy":0.4846823066473007},{"step":411,"reward":-1.2162504196166992,"core":-0.32753828167915344,"cost":-0.07569731026887894,"kl":0.09780714847147465,"entropy":0.4721261337399483},{"step":412,"reward":-1.228188395500183,"core":-0.32255780696868896,"cost":-0.03338480740785599,"kl":0.10569658130407333,"entropy":0.5211627632379532},{"step":413,"reward":-0.7512831091880798,"core":-0.18197111785411835,"cost":0.0,"kl":0.12215324304997921,"entropy":0.4837137684226036},{"step":414,"reward":-1.4509159326553345,"core":-0.32190823554992676,"cost":-0.06966600567102432,"kl":0.08331096544861794,"entropy":0.5404750779271126},{"step":415,"reward":-2.834956169128418,"core":-0.8553380370140076,"cost":-0.20710985362529755,"kl":0.07820545136928558,"entropy":0.5410013943910599},{"step":416,"reward":-0.5923144221305847,"core":-0.06351002305746078,"cost":-0.04633256793022156,"kl":0.10673963092267513,"entropy":0.5761243402957916},{"step":417,"reward":-1.3493236303329468,"core":-0.3456272780895233,"cost":-0.01927683688700199,"kl":0.11417538113892078,"entropy":0.578049048781395},{"step":418,"reward":-1.5294138193130493,"core":-0.42357155680656433,"cost":-0.09545930474996567,"kl":0.09209842048585415,"entropy":0.5468129962682724},{"step":419,"reward":-1.5837087631225586,"core":-0.4748654365539551,"cost":-0.027147715911269188,"kl":0.1166652012616396,"entropy":0.5637549459934235},{"step":420,"reward":-1.7970842123031616,"core":-0.5224236249923706,"cost":-0.11754562705755234,"kl":0.08686438947916031,"entropy":0.5979985743761063},{"step":421,"reward":-1.6133699417114258,"core":-0.45432400703430176,"cost":-0.049182116985321045,"kl":0.07761076278984547,"entropy":0.4590234309434891},{"step":422,"reward":-1.0987387895584106,"core":-0.3203490078449249,"cost":-0.009017818607389927,"kl":0.08162257447838783,"entropy":0.522201657295227},{"step":423,"reward":0.3073086440563202,"core":0.2915309965610504,"cost":0.0,"kl":0.07420920394361019,"entropy":0.45753680169582367},{"step":424,"reward":-1.9311631917953491,"core":-0.5768976211547852,"cost":-0.07063117623329163,"kl":0.059455372393131256,"entropy":0.4907495826482773},{"step":425,"reward":-1.7361866235733032,"core":-0.4029308557510376,"cost":-0.20466004312038422,"kl":0.0794312646612525,"entropy":0.5078858584165573},{"step":426,"reward":-1.3782039880752563,"core":-0.3897894322872162,"cost":-0.008711117319762707,"kl":0.11690177768468857,"entropy":0.6336771547794342},{"step":427,"reward":-0.8454156517982483,"core":-0.19881534576416016,"cost":-0.04892253503203392,"kl":0.06946094427257776,"entropy":0.6016354560852051},{"step":428,"reward":-0.7931819558143616,"core":-0.1686374545097351,"cost":-0.014975913800299168,"kl":0.08427131175994873,"entropy":0.46739835292100906},{"step":429,"reward":-0.6488469243049622,"core":-0.07051654905080795,"cost":-0.016643594950437546,"kl":0.08043122477829456,"entropy":0.4773098826408386},{"step":430,"reward":-1.0180635452270508,"core":-0.3060341775417328,"cost":0.0,"kl":0.07523670606315136,"entropy":0.5927493274211884},{"step":431,"reward":-1.7372010946273804,"core":-0.39911338686943054,"cost":-0.15622597932815552,"kl":0.0988818034529686,"entropy":0.5757292956113815},{"step":432,"reward":-0.820755660533905,"core":-0.17325137555599213,"cost":-0.011986381374299526,"kl":0.08725344389677048,"entropy":0.5704189836978912},{"step":433,"reward":-1.1081935167312622,"core":-0.26798632740974426,"cost":-0.03195574879646301,"kl":0.06403028685599566,"entropy":0.47332054376602173},{"step":434,"reward":-2.3997488021850586,"core":-0.7503306269645691,"cost":-0.10211356729269028,"kl":0.08617900684475899,"entropy":0.5744409263134003},{"step":435,"reward":-0.6956642270088196,"core":-0.16062690317630768,"cost":-0.015232610516250134,"kl":0.06462812051177025,"entropy":0.5361119657754898},{"step":436,"reward":-2.098100423812866,"core":-0.5599535703659058,"cost":-0.12137287855148315,"kl":0.08405071496963501,"entropy":0.4924328178167343},{"step":437,"reward":-0.6791526675224304,"core":-0.10645153373479843,"cost":-0.038595881313085556,"kl":0.08635441400110722,"entropy":0.6275528818368912},{"step":438,"reward":-1.6518841981887817,"core":-0.45012831687927246,"cost":-0.11139160394668579,"kl":0.10085384175181389,"entropy":0.5584894120693207},{"step":439,"reward":-0.7514732480049133,"core":-0.1092630997300148,"cost":-0.08653751760721207,"kl":0.09766965545713902,"entropy":0.5392458140850067},{"step":440,"reward":0.14282731711864471,"core":0.270786315202713,"cost":-0.10157325863838196,"kl":0.09131434746086597,"entropy":0.4630071222782135},{"step":441,"reward":-0.7672123908996582,"core":-0.18321259319782257,"cost":-0.0021699846256524324,"kl":0.11015901900827885,"entropy":0.5084242299199104},{"step":442,"reward":-2.0061252117156982,"core":-0.5651083588600159,"cost":-0.13981856405735016,"kl":0.09237384237349033,"entropy":0.6108192801475525},{"step":443,"reward":-0.9494640827178955,"core":-0.2311217188835144,"cost":-0.008830695413053036,"kl":0.09097757190465927,"entropy":0.574689045548439},{"step":444,"reward":-1.9111379384994507,"core":-0.46745380759239197,"cost":-0.25751569867134094,"kl":0.08645360358059406,"entropy":0.4962887763977051},{"step":445,"reward":-0.981341540813446,"core":-0.23525726795196533,"cost":-0.03704024851322174,"kl":0.09221936017274857,"entropy":0.497174508869648},{"step":446,"reward":-2.459205389022827,"core":-0.7006197571754456,"cost":-0.1849886178970337,"kl":0.11121698096394539,"entropy":0.5629732459783554},{"step":447,"reward":-2.5505502223968506,"core":-0.7799253463745117,"cost":-0.13227377831935883,"kl":0.06884204689413309,"entropy":0.4576037898659706},{"step":448,"reward":-0.5658935308456421,"core":-0.061416421085596085,"cost":-0.0349392294883728,"kl":0.07293318770825863,"entropy":0.5528959780931473},{"step":449,"reward":-1.3588918447494507,"core":-0.33394655585289,"cost":-0.0430445559322834,"kl":0.10847119428217411,"entropy":0.5007417872548103},{"step":450,"reward":-0.7743465900421143,"core":-0.15986016392707825,"cost":-0.06645148247480392,"kl":0.08978847414255142,"entropy":0.4713207706809044},{"step":451,"reward":-0.5893430113792419,"core":-0.03259680047631264,"cost":-0.005739132408052683,"kl":0.08080585114657879,"entropy":0.4964793175458908},{"step":452,"reward":-0.5818027853965759,"core":-0.1192404255270958,"cost":0.0,"kl":0.125304713845253,"entropy":0.5262415558099747},{"step":453,"reward":-0.3551276922225952,"core":0.03249899670481682,"cost":0.0,"kl":0.13550793752074242,"entropy":0.5048826560378075},{"step":454,"reward":-0.8963688015937805,"core":-0.20351962745189667,"cost":0.0,"kl":0.08637824095785618,"entropy":0.49415964633226395},{"step":455,"reward":-1.3700599670410156,"core":-0.3561260998249054,"cost":-0.0668380856513977,"kl":0.0735253356397152,"entropy":0.5202484726905823},{"step":456,"reward":-3.788074493408203,"core":-1.1046786308288574,"cost":-0.3924529552459717,"kl":0.07293454464524984,"entropy":0.5339293330907822},{"step":457,"reward":-1.1455434560775757,"core":-0.2810392677783966,"cost":-0.10747966915369034,"kl":0.12906285002827644,"entropy":0.48291776329278946},{"step":458,"reward":0.20996247231960297,"core":0.26750484108924866,"cost":-0.04873964190483093,"kl":0.0807330347597599,"entropy":0.47200907766819},{"step":459,"reward":-1.5482879877090454,"core":-0.4150486886501312,"cost":-0.03593156859278679,"kl":0.10171138867735863,"entropy":0.5927760452032089},{"step":460,"reward":-2.1760318279266357,"core":-0.6493008136749268,"cost":-0.08372777700424194,"kl":0.10560708679258823,"entropy":0.5024382323026657},{"step":461,"reward":-1.3520593643188477,"core":-0.3804410398006439,"cost":-0.0004448190738912672,"kl":0.07256362494081259,"entropy":0.5349618196487427},{"step":462,"reward":-0.680981457233429,"core":-0.11591183394193649,"cost":-0.06934990733861923,"kl":0.11668875813484192,"entropy":0.5310159027576447},{"step":463,"reward":-1.2893422842025757,"core":-0.34706246852874756,"cost":-0.040044207125902176,"kl":0.0633445456624031,"entropy":0.5105588287115097},{"step":464,"reward":-0.7923936247825623,"core":-0.17220143973827362,"cost":-0.008228996768593788,"kl":0.11253839917480946,"entropy":0.5347117856144905},{"step":465,"reward":-0.8523672223091125,"core":-0.12456420063972473,"cost":-0.05000809207558632,"kl":0.1093048807233572,"entropy":0.47294868528842926},{"step":466,"reward":-1.4855756759643555,"core":-0.3842322528362274,"cost":-0.15731467306613922,"kl":0.12113482877612114,"entropy":0.5473478585481644},{"step":467,"reward":-1.6972875595092773,"core":-0.48042377829551697,"cost":-0.06524529308080673,"kl":0.09141307137906551,"entropy":0.47951317578554153},{"step":468,"reward":-1.5905650854110718,"core":-0.43976202607154846,"cost":-0.071173757314682,"kl":0.07645884342491627,"entropy":0.5102259367704391},{"step":469,"reward":-0.2740279734134674,"core":0.06138579919934273,"cost":-0.021262740716338158,"kl":0.09683677554130554,"entropy":0.48904483765363693},{"step":470,"reward":-0.9209302067756653,"core":-0.17447566986083984,"cost":-0.024210063740611076,"kl":0.0897282175719738,"entropy":0.5373504385352135},{"step":471,"reward":-0.9563754200935364,"core":-0.2033735066652298,"cost":-0.05075982213020325,"kl":0.10756690613925457,"entropy":0.48372122645378113},{"step":472,"reward":0.18844173848628998,"core":0.2926783263683319,"cost":-0.023851094767451286,"kl":0.10991809330880642,"entropy":0.4938278868794441},{"step":473,"reward":-1.280965805053711,"core":-0.2936624586582184,"cost":-0.10197225213050842,"kl":0.09585033357143402,"entropy":0.5465954765677452},{"step":474,"reward":-1.1934529542922974,"core":-0.294344961643219,"cost":-0.03402363881468773,"kl":0.07905527390539646,"entropy":0.4988240450620651},{"step":475,"reward":-0.17243631184101105,"core":0.049846943467855453,"cost":0.0,"kl":0.1120761539787054,"entropy":0.6026183590292931},{"step":476,"reward":-1.0044535398483276,"core":-0.23221206665039062,"cost":-0.0909096971154213,"kl":0.1460348181426525,"entropy":0.5286345034837723},{"step":477,"reward":-0.3728998899459839,"core":0.015252898447215557,"cost":-0.031524401158094406,"kl":0.10232031345367432,"entropy":0.5075204893946648},{"step":478,"reward":-1.8171740770339966,"core":-0.5487921833992004,"cost":-0.036916498094797134,"kl":0.09662852995097637,"entropy":0.5883067697286606},{"step":479,"reward":-0.3462810218334198,"core":0.0027000729460269213,"cost":-0.03950934112071991,"kl":0.0949687771499157,"entropy":0.44139526784420013},{"step":480,"reward":-1.4809985160827637,"core":-0.33301815390586853,"cost":-0.1287836879491806,"kl":0.09485846012830734,"entropy":0.4412897154688835},{"step":481,"reward":-1.243153691291809,"core":-0.23059029877185822,"cost":-0.08924809843301773,"kl":0.1374202761799097,"entropy":0.5059916973114014},{"step":482,"reward":-1.1249874830245972,"core":-0.3124774396419525,"cost":0.0,"kl":0.09123470447957516,"entropy":0.5304507240653038},{"step":483,"reward":-0.9496542811393738,"core":-0.21643595397472382,"cost":-0.006662419531494379,"kl":0.07605385687202215,"entropy":0.40140752494335175},{"step":484,"reward":-0.4717010259628296,"core":-0.0690896213054657,"cost":0.0,"kl":0.1316309403628111,"entropy":0.5319747254252434},{"step":485,"reward":-0.9173659682273865,"core":-0.21718724071979523,"cost":-0.0019418355077505112,"kl":0.10099169798195362,"entropy":0.5952599048614502},{"step":486,"reward":-0.6397719979286194,"core":-0.08609611541032791,"cost":-0.050467949360609055,"kl":0.08162531070411205,"entropy":0.4590989500284195},{"step":487,"reward":-0.25102707743644714,"core":0.07189179211854935,"cost":-0.018202556297183037,"kl":0.09644406661391258,"entropy":0.5196498557925224},{"step":488,"reward":-0.7296819686889648,"core":-0.08104640245437622,"cost":-0.06069560348987579,"kl":0.18558510206639767,"entropy":0.5301069915294647},{"step":489,"reward":-1.7313756942749023,"core":-0.5332539081573486,"cost":0.0,"kl":0.08947709575295448,"entropy":0.566972479224205},{"step":490,"reward":-1.2674895524978638,"core":-0.3415926396846771,"cost":-0.10463150590658188,"kl":0.09725966677069664,"entropy":0.5434724986553192},{"step":491,"reward":-1.113352656364441,"core":-0.2411016821861267,"cost":-0.07788452506065369,"kl":0.09277892857789993,"entropy":0.5736891031265259},{"step":492,"reward":-1.7646578550338745,"core":-0.4824031889438629,"cost":-0.13785366714000702,"kl":0.09518717508763075,"entropy":0.5453951507806778},{"step":493,"reward":-1.5838886499404907,"core":-0.4616617262363434,"cost":-0.04386930540204048,"kl":0.10666903294622898,"entropy":0.5137358084321022},{"step":494,"reward":0.6864328980445862,"core":0.389202743768692,"cost":0.0,"kl":0.09693196415901184,"entropy":0.5229435190558434},{"step":495,"reward":-0.5707074403762817,"core":-0.1324322372674942,"cost":-0.015169267542660236,"kl":0.1176458578556776,"entropy":0.5648062229156494},{"step":496,"reward":-1.6096853017807007,"core":-0.4677382707595825,"cost":-0.07540097087621689,"kl":0.0871381051838398,"entropy":0.4995994344353676},{"step":497,"reward":-0.8285346031188965,"core":-0.20781557261943817,"cost":-0.016056450083851814,"kl":0.1272369585931301,"entropy":0.5222591459751129},{"step":498,"reward":-0.8880600333213806,"core":-0.23374886810779572,"cost":-0.029215099290013313,"kl":0.11423942260444164,"entropy":0.515174612402916},{"step":499,"reward":-1.68618905544281,"core":-0.47038641571998596,"cost":-0.13919740915298462,"kl":0.13172760233283043,"entropy":0.5812691897153854},{"step":500,"reward":-2.189180612564087,"core":-0.6309966444969177,"cost":-0.1312384009361267,"kl":0.12173470482230186,"entropy":0.6005405336618423},{"step":501,"reward":-0.25645163655281067,"core":0.022657059133052826,"cost":-0.004168545827269554,"kl":0.1568477600812912,"entropy":0.5203981995582581},{"step":502,"reward":-1.3477686643600464,"core":-0.37502846121788025,"cost":-0.054743725806474686,"kl":0.07352038659155369,"entropy":0.5331188291311264},{"step":503,"reward":-1.077537178993225,"core":-0.28833186626434326,"cost":-0.010899621993303299,"kl":0.11940824240446091,"entropy":0.49967168271541595},{"step":504,"reward":-0.2897479832172394,"core":0.11846510320901871,"cost":-0.08536633104085922,"kl":0.13009834475815296,"entropy":0.5105736330151558},{"step":505,"reward":-1.7292979955673218,"core":-0.5164559483528137,"cost":-0.08832602947950363,"kl":0.0954919084906578,"entropy":0.5196563675999641},{"step":506,"reward":-1.508357048034668,"core":-0.4390939772129059,"cost":-0.064121313393116,"kl":0.12368964962661266,"entropy":0.5824526250362396},{"step":507,"reward":-1.4269005060195923,"core":-0.39183250069618225,"cost":-0.03997303545475006,"kl":0.07252413872629404,"entropy":0.5453974232077599},{"step":508,"reward":-1.314949631690979,"core":-0.3173080384731293,"cost":-0.09800710529088974,"kl":0.1001364104449749,"entropy":0.5098260939121246},{"step":509,"reward":-0.563866913318634,"core":-0.058632493019104004,"cost":-0.05173960700631142,"kl":0.13529988192021847,"entropy":0.5461648106575012},{"step":510,"reward":-0.05588005855679512,"core":0.11523190140724182,"cost":-0.033151667565107346,"kl":0.1311960630118847,"entropy":0.458479642868042},{"step":511,"reward":-0.9731065630912781,"core":-0.21977955102920532,"cost":-0.007770284544676542,"kl":0.07522768713533878,"entropy":0.5375932306051254},{"step":512,"reward":-0.38852033019065857,"core":-0.037752848118543625,"cost":0.0,"kl":0.08539772778749466,"entropy":0.5633696019649506},{"step":513,"reward":0.4722221791744232,"core":0.3158816397190094,"cost":0.0,"kl":0.10596591979265213,"entropy":0.48534538596868515},{"step":514,"reward":-1.3525727987289429,"core":-0.2907513678073883,"cost":-0.1101861223578453,"kl":0.16150056011974812,"entropy":0.5431166738271713},{"step":515,"reward":-0.6292324066162109,"core":-0.07493377476930618,"cost":-0.06884009391069412,"kl":0.10680198483169079,"entropy":0.5255637615919113},{"step":516,"reward":-1.2398202419281006,"core":-0.28741949796676636,"cost":-0.08944053202867508,"kl":0.09549429453909397,"entropy":0.5366977900266647},{"step":517,"reward":-0.23348182439804077,"core":0.009634449146687984,"cost":-0.010678793303668499,"kl":0.10501294024288654,"entropy":0.4947335869073868},{"step":518,"reward":-0.5750571489334106,"core":-0.09074338525533676,"cost":0.0,"kl":0.1483866199851036,"entropy":0.546957366168499},{"step":519,"reward":-1.2621458768844604,"core":-0.3040824830532074,"cost":-0.06845545768737793,"kl":0.07358501479029655,"entropy":0.516935370862484},{"step":520,"reward":-1.0520120859146118,"core":-0.2847078740596771,"cost":-0.010806240141391754,"kl":0.08978100121021271,"entropy":0.5442384332418442},{"step":521,"reward":-0.8822383880615234,"core":-0.2043117731809616,"cost":-0.024022290483117104,"kl":0.11711869016289711,"entropy":0.47934436053037643},{"step":522,"reward":-2.1146774291992188,"core":-0.6240101456642151,"cost":-0.11442041397094727,"kl":0.0897520799189806,"entropy":0.48394162952899933},{"step":523,"reward":-0.3831629455089569,"core":-0.045374322682619095,"cost":0.0,"kl":0.09242581203579903,"entropy":0.5303548127412796},{"step":524,"reward":-0.8457736968994141,"core":-0.2147688865661621,"cost":-0.009726994670927525,"kl":0.13316984102129936,"entropy":0.5673732608556747},{"step":525,"reward":-0.4140879213809967,"core":-0.03681798279285431,"cost":-0.005439750384539366,"kl":0.09918701648712158,"entropy":0.4995042085647583},{"step":526,"reward":-1.0424665212631226,"core":-0.16872084140777588,"cost":-0.10216835886240005,"kl":0.1148466020822525,"entropy":0.5075820684432983},{"step":527,"reward":-1.3930917978286743,"core":-0.3762465715408325,"cost":-0.035580020397901535,"kl":0.10235324688255787,"entropy":0.4967561811208725},{"step":528,"reward":-1.4601072072982788,"core":-0.3853329122066498,"cost":-0.12116546183824539,"kl":0.0895363911986351,"entropy":0.5236146673560143},{"step":529,"reward":-1.7039152383804321,"core":-0.5364053845405579,"cost":-0.011380307376384735,"kl":0.10048770345747471,"entropy":0.43249592185020447},{"step":530,"reward":-1.6188654899597168,"core":-0.4335486888885498,"cost":-0.16042782366275787,"kl":0.13650263845920563,"entropy":0.5137021839618683},{"step":531,"reward":-0.22907708585262299,"core":0.018917078152298927,"cost":-0.004071191884577274,"kl":0.17913737148046494,"entropy":0.5540052950382233},{"step":532,"reward":-1.068913459777832,"core":-0.2554034888744354,"cost":-0.055521607398986816,"kl":0.06996819004416466,"entropy":0.4761161431670189},{"step":533,"reward":0.10327204316854477,"core":0.19674791395664215,"cost":0.0,"kl":0.11783857271075249,"entropy":0.57998988032341},{"step":534,"reward":-1.1192423105239868,"core":-0.30340471863746643,"cost":-0.005188023205846548,"kl":0.12033580616116524,"entropy":0.5488191545009613},{"step":535,"reward":-1.3003684282302856,"core":-0.3107997477054596,"cost":-0.09104675054550171,"kl":0.11819721385836601,"entropy":0.5261701941490173},{"step":536,"reward":-1.187459945678711,"core":-0.30893445014953613,"cost":-0.054100241512060165,"kl":0.09161796886473894,"entropy":0.5062828660011292},{"step":537,"reward":0.20876772701740265,"core":0.24031801521778107,"cost":-0.01697174832224846,"kl":0.13982905633747578,"entropy":0.46852611005306244},{"step":538,"reward":0.3497699201107025,"core":0.306598037481308,"cost":0.0,"kl":0.08531833626329899,"entropy":0.5132170245051384},{"step":539,"reward":-1.8199542760849,"core":-0.5413870811462402,"cost":-0.04678747430443764,"kl":0.14232849329710007,"entropy":0.5964807420969009},{"step":540,"reward":0.14268352091312408,"core":0.25529494881629944,"cost":-0.02508430927991867,"kl":0.11694425903260708,"entropy":0.5386097729206085},{"step":541,"reward":-0.4862649440765381,"core":-0.0779188945889473,"cost":-0.02111079730093479,"kl":0.0902661383152008,"entropy":0.5451496988534927},{"step":542,"reward":-0.8415506482124329,"core":-0.173383429646492,"cost":-0.05604146048426628,"kl":0.14701642468571663,"entropy":0.5317720174789429},{"step":543,"reward":-0.8238606452941895,"core":-0.12939751148223877,"cost":-0.04231668636202812,"kl":0.11607901379466057,"entropy":0.5102442651987076},{"step":544,"reward":-2.0398077964782715,"core":-0.6082077026367188,"cost":-0.04726564884185791,"kl":0.1178312674164772,"entropy":0.5130321532487869},{"step":545,"reward":-0.8942604660987854,"core":-0.15409386157989502,"cost":-0.10881657153367996,"kl":0.10975260473787785,"entropy":0.5712285935878754},{"step":546,"reward":-0.6704864501953125,"core":-0.02665034681558609,"cost":-0.14542566239833832,"kl":0.12094700150191784,"entropy":0.4923350438475609},{"step":547,"reward":0.3969610631465912,"core":0.2820858061313629,"cost":0.0,"kl":0.09439359605312347,"entropy":0.5558464378118515},{"step":548,"reward":-0.7475998997688293,"core":-0.1554095298051834,"cost":0.0,"kl":0.11522960849106312,"entropy":0.5658338069915771},{"step":549,"reward":-1.5070186853408813,"core":-0.39448294043540955,"cost":-0.05141685530543327,"kl":0.0987109076231718,"entropy":0.4787280932068825},{"step":550,"reward":-0.38408446311950684,"core":0.013099034316837788,"cost":-0.010826664976775646,"kl":0.09769553504884243,"entropy":0.5135451257228851},{"step":551,"reward":-1.152065396308899,"core":-0.3058513402938843,"cost":-0.07607025653123856,"kl":0.12675642408430576,"entropy":0.4966162145137787},{"step":552,"reward":-1.119467854499817,"core":-0.26147934794425964,"cost":-0.06742919236421585,"kl":0.14292384684085846,"entropy":0.5724216848611832},{"step":553,"reward":-0.21072165668010712,"core":0.02507013827562332,"cost":0.0,"kl":0.1160886287689209,"entropy":0.5599284619092941},{"step":554,"reward":-0.483363538980484,"core":-0.01837935484945774,"cost":-0.09719163179397583,"kl":0.1605119351297617,"entropy":0.5808594673871994},{"step":555,"reward":-1.2011016607284546,"core":-0.32592055201530457,"cost":-0.024741241708397865,"kl":0.1073505524545908,"entropy":0.4706042632460594},{"step":556,"reward":-0.7483065724372864,"core":-0.12883536517620087,"cost":-0.02155573107302189,"kl":0.09479687362909317,"entropy":0.6166216284036636},{"step":557,"reward":-1.369019865989685,"core":-0.3791099488735199,"cost":-0.06435299664735794,"kl":0.10961133614182472,"entropy":0.5503186732530594},{"step":558,"reward":0.0892866924405098,"core":0.1687130331993103,"cost":-0.06371262669563293,"kl":0.15910930559039116,"entropy":0.4897504672408104},{"step":559,"reward":-1.5484458208084106,"core":-0.46500492095947266,"cost":-0.025728831067681313,"kl":0.11171210557222366,"entropy":0.490009106695652},{"step":560,"reward":-2.442721366882324,"core":-0.7644265294075012,"cost":-0.06796509027481079,"kl":0.10125443711876869,"entropy":0.5195553824305534},{"step":561,"reward":-0.5471568703651428,"core":-0.08685754984617233,"cost":0.0,"kl":0.09640808403491974,"entropy":0.49374841898679733},{"step":562,"reward":-1.2513126134872437,"core":-0.30022892355918884,"cost":-0.04611917957663536,"kl":0.09497961401939392,"entropy":0.5219277888536453},{"step":563,"reward":-1.619637131690979,"core":-0.4662679135799408,"cost":-0.04397809877991676,"kl":0.12455469742417336,"entropy":0.5675375312566757},{"step":564,"reward":-0.4904024600982666,"core":-0.057753439992666245,"cost":-0.008963586762547493,"kl":0.17883440107107162,"entropy":0.4920062720775604},{"step":565,"reward":-0.6139959692955017,"core":-0.06259480863809586,"cost":-0.050024788826704025,"kl":0.10135746747255325,"entropy":0.5049278140068054},{"step":566,"reward":-0.0429253987967968,"core":0.12443146109580994,"cost":-0.04571337625384331,"kl":0.09718213602900505,"entropy":0.5542252361774445},{"step":567,"reward":0.2940308749675751,"core":0.29309964179992676,"cost":-0.03612074628472328,"kl":0.14027801528573036,"entropy":0.4825427010655403},{"step":568,"reward":-0.8995038866996765,"core":-0.23787860572338104,"cost":-0.0073734670877456665,"kl":0.11780751869082451,"entropy":0.6161465346813202},{"step":569,"reward":-1.7926974296569824,"core":-0.5418530702590942,"cost":-0.03307646885514259,"kl":0.09962503239512444,"entropy":0.5250278636813164},{"step":570,"reward":-1.2511011362075806,"core":-0.32668617367744446,"cost":-0.07986672967672348,"kl":0.09794286452233791,"entropy":0.5097935348749161},{"step":571,"reward":-0.8195316791534424,"core":-0.14961017668247223,"cost":0.0,"kl":0.1184504758566618,"entropy":0.5120046362280846},{"step":572,"reward":-0.23896424472332,"core":0.0710858404636383,"cost":-0.021840348839759827,"kl":0.08380915503948927,"entropy":0.4842488467693329},{"step":573,"reward":-1.704689383506775,"core":-0.4805445373058319,"cost":-0.12678761780261993,"kl":0.08827898651361465,"entropy":0.5045935958623886},{"step":574,"reward":-0.899857223033905,"core":-0.22165381908416748,"cost":-0.04743888974189758,"kl":0.08496552240103483,"entropy":0.4157959967851639},{"step":575,"reward":-1.9128674268722534,"core":-0.5788195729255676,"cost":-0.024428056553006172,"kl":0.0744147477671504,"entropy":0.379735492169857},{"step":576,"reward":-1.1444852352142334,"core":-0.32075273990631104,"cost":-0.04846730828285217,"kl":0.10371582582592964,"entropy":0.5019004493951797},{"step":577,"reward":-0.6719551086425781,"core":-0.17004577815532684,"cost":-0.009697315283119678,"kl":0.12957492470741272,"entropy":0.5492395162582397},{"step":578,"reward":0.17819778621196747,"core":0.2672925889492035,"cost":-0.00501109566539526,"kl":0.10169702023267746,"entropy":0.5005062818527222},{"step":579,"reward":-1.7625080347061157,"core":-0.5049065351486206,"cost":-0.031129946932196617,"kl":0.12434366904199123,"entropy":0.5290268808603287},{"step":580,"reward":0.19204719364643097,"core":0.24990393221378326,"cost":-0.05340572074055672,"kl":0.1113484837114811,"entropy":0.4665995314717293},{"step":581,"reward":-2.393350124359131,"core":-0.7138784527778625,"cost":-0.169350266456604,"kl":0.09060618653893471,"entropy":0.4515553489327431},{"step":582,"reward":-0.7067964673042297,"core":-0.15748511254787445,"cost":-0.02274641953408718,"kl":0.10765066742897034,"entropy":0.4905715510249138},{"step":583,"reward":-1.7013330459594727,"core":-0.4791642129421234,"cost":-0.07436511665582657,"kl":0.1285195481032133,"entropy":0.5511090308427811},{"step":584,"reward":-1.087964653968811,"core":-0.26870056986808777,"cost":-0.03987045958638191,"kl":0.09993059001863003,"entropy":0.5071726366877556},{"step":585,"reward":-0.8545553088188171,"core":-0.20265161991119385,"cost":-0.03138544410467148,"kl":0.11957871541380882,"entropy":0.4954100772738457},{"step":586,"reward":-0.3258574306964874,"core":0.0447239987552166,"cost":-0.023319974541664124,"kl":0.10950535908341408,"entropy":0.5186738520860672},{"step":587,"reward":-0.31736502051353455,"core":0.058735888451337814,"cost":-0.011480431072413921,"kl":0.1128349769860506,"entropy":0.5048224553465843},{"step":588,"reward":-1.1364067792892456,"core":-0.22824810445308685,"cost":-0.07185729593038559,"kl":0.14210161566734314,"entropy":0.553768202662468},{"step":589,"reward":-0.6206638216972351,"core":-0.09113183617591858,"cost":-0.005859662313014269,"kl":0.1193533856421709,"entropy":0.5146292150020599},{"step":590,"reward":-0.7032336592674255,"core":-0.13447971642017365,"cost":-0.008008955977857113,"kl":0.1449353452771902,"entropy":0.5014290362596512},{"step":591,"reward":-0.2164289504289627,"core":0.06323939561843872,"cost":-0.020602324977517128,"kl":0.0894035454839468,"entropy":0.43132372200489044},{"step":592,"reward":-1.7866758108139038,"core":-0.47844624519348145,"cost":-0.08672850579023361,"kl":0.11468333564698696,"entropy":0.4955109730362892},{"step":593,"reward":-0.4339614808559418,"core":-0.04197072982788086,"cost":-0.00248575652949512,"kl":0.11534724198281765,"entropy":0.5806992501020432},{"step":594,"reward":-2.5827198028564453,"core":-0.7580910325050354,"cost":-0.14703942835330963,"kl":0.07381376717239618,"entropy":0.4228743836283684},{"step":595,"reward":-1.922133445739746,"core":-0.5382266640663147,"cost":-0.16600672900676727,"kl":0.10910648107528687,"entropy":0.44841133058071136},{"step":596,"reward":-1.412819504737854,"core":-0.333730548620224,"cost":-0.09647411108016968,"kl":0.12516053207218647,"entropy":0.6149762719869614},{"step":597,"reward":-1.000409483909607,"core":-0.21039052307605743,"cost":-0.07177292555570602,"kl":0.10518520697951317,"entropy":0.5158037543296814},{"step":598,"reward":-1.3254643678665161,"core":-0.35242366790771484,"cost":-0.072507843375206,"kl":0.10938615165650845,"entropy":0.5385245680809021},{"step":599,"reward":-1.9729524850845337,"core":-0.5686948299407959,"cost":-0.1355978101491928,"kl":0.10161146707832813,"entropy":0.5040400624275208},{"step":600,"reward":-0.19848859310150146,"core":0.05449443683028221,"cost":-0.02574797160923481,"kl":0.17342353984713554,"entropy":0.5546907261013985}],"nonzero_advantage":[{"step":1,"nonzero":0.5416666666666666},{"step":3,"nonzero":0.2708333333333333},{"step":5,"nonzero":0.375},{"step":7,"nonzero":0.2916666666666667},{"step":9,"nonzero":0.1875},{"step":11,"nonzero":0.4583333333333333},{"step":13,"nonzero":0.10416666666666667},{"step":15,"nonzero":0.3333333333333333},{"step":17,"nonzero":0.5416666666666666},{"step":19,"nonzero":0.20833333333333334},{"step":21,"nonzero":0.4166666666666667},{"step":23,"nonzero":0.375},{"step":25,"nonzero":0.375},{"step":27,"nonzero":0.5},{"step":29,"nonzero":0.3541666666666667},{"step":31,"nonzero":0.3333333333333333},{"step":33,"nonzero":0.16666666666666666},{"step":35,"nonzero":0.14583333333333334},{"step":37,"nonzero":0.4166666666666667},{"step":39,"nonzero":0.5},{"step":41,"nonzero":0.3958333333333333},{"step":43,"nonzero":0.5416666666666666},{"step":45,"nonzero":0.4583333333333333},{"step":47,"nonzero":0.3125},{"step":49,"nonzero":0.14583333333333334},{"step":51,"nonzero":0.4791666666666667},{"step":53,"nonzero":0.5833333333333334},{"step":55,"nonzero":0.3541666666666667},{"step":57,"nonzero":0.5416666666666666},{"step":59,"nonzero":0.625},{"step":61,"nonzero":0.3958333333333333},{"step":63,"nonzero":0.5416666666666666},{"step":65,"nonzero":0.7916666666666666},{"step":67,"nonzero":0.5833333333333334},{"step":69,"nonzero":0.6458333333333334},{"step":71,"nonzero":0.3958333333333333},{"step":73,"nonzero":0.5416666666666666},{"step":75,"nonzero":0.5833333333333334},{"step":77,"nonzero":0.5416666666666666},{"step":79,"nonzero":0.5833333333333334},{"step":81,"nonzero":0.4375},{"step":83,"nonzero":0.6875},{"step":85,"nonzero":0.4166666666666667},{"step":87,"nonzero":0.3125},{"step":89,"nonzero":0.4583333333333333},{"step":91,"nonzero":0.5208333333333334},{"step":93,"nonzero":0.4791666666666667},{"step":95,"nonzero":0.3958333333333333},{"step":97,"nonzero":0.8541666666666666},{"step":99,"nonzero":0.7083333333333334},{"step":101,"nonzero":0.4375},{"step":103,"nonzero":0.4791666666666667},{"step":105,"nonzero":0.8333333333333334},{"step":107,"nonzero":0.7916666666666666},{"step":109,"nonzero":0.75},{"step":111,"nonzero":0.625},{"step":113,"nonzero":0.7083333333333334},{"step":115,"nonzero":0.22916666666666666},{"step":117,"nonzero":0.875},{"step":119,"nonzero":0.75},{"step":121,"nonzero":0.3958333333333333},{"step":123,"nonzero":0.6458333333333334},{"step":125,"nonzero":0.4375},{"step":127,"nonzero":0.6666666666666666},{"step":129,"nonzero":0.6666666666666666},{"step":131,"nonzero":0.5833333333333334},{"step":133,"nonzero":0.6458333333333334},{"step":135,"nonzero":0.75},{"step":137,"nonzero":0.6041666666666666},{"step":139,"nonzero":0.22916666666666666},{"step":141,"nonzero":0.375},{"step":143,"nonzero":0.4791666666666667},{"step":145,"nonzero":0.5416666666666666},{"step":147,"nonzero":0.625},{"step":149,"nonzero":0.3333333333333333},{"step":151,"nonzero":0.5},{"step":153,"nonzero":0.5},{"step":155,"nonzero":0.5625},{"step":157,"nonzero":0.4791666666666667},{"step":159,"nonzero":0.4583333333333333},{"step":161,"nonzero":0.5208333333333334},{"step":163,"nonzero":0.375},{"step":165,"nonzero":0.4791666666666667},{"step":167,"nonzero":0.7083333333333334},{"step":169,"nonzero":0.6666666666666666},{"step":171,"nonzero":0.4583333333333333},{"step":173,"nonzero":0.4791666666666667},{"step":175,"nonzero":0.4166666666666667},{"step":177,"nonzero":0.5625},{"step":179,"nonzero":0.6666666666666666},{"step":181,"nonzero":0.4583333333333333},{"step":183,"nonzero":0.4791666666666667},{"step":185,"nonzero":0.08333333333333333},{"step":187,"nonzero":0.4791666666666667},{"step":189,"nonzero":0.5208333333333334},{"step":191,"nonzero":0.5625},{"step":193,"nonzero":0.3333333333333333},{"step":195,"nonzero":0.5625},{"step":197,"nonzero":0.2708333333333333},{"step":199,"nonzero":0.8333333333333334},{"step":201,"nonzero":0.8125},{"step":203,"nonzero":0.3333333333333333},{"step":205,"nonzero":0.75},{"step":207,"nonzero":0.625},{"step":209,"nonzero":0.4166666666666667},{"step":211,"nonzero":0.5416666666666666},{"step":213,"nonzero":0.7083333333333334},{"step":215,"nonzero":0.22916666666666666},{"step":217,"nonzero":0.14583333333333334},{"step":219,"nonzero":0.6458333333333334},{"step":221,"nonzero":0.3333333333333333},{"step":223,"nonzero":0.5625},{"step":225,"nonzero":0.4791666666666667},{"step":227,"nonzero":0.4583333333333333},{"step":229,"nonzero":0.5833333333333334},{"step":231,"nonzero":0.875},{"step":233,"nonzero":0.5208333333333334},{"step":235,"nonzero":0.3541666666666667},{"step":237,"nonzero":0.6458333333333334},{"step":239,"nonzero":0.6666666666666666},{"step":241,"nonzero":0.2708333333333333},{"step":243,"nonzero":0.6041666666666666},{"step":245,"nonzero":0.6666666666666666},{"step":247,"nonzero":0.6875},{"step":249,"nonzero":0.625},{"step":251,"nonzero":0.6666666666666666},{"step":253,"nonzero":0.5833333333333334},{"step":255,"nonzero":0.7083333333333334},{"step":257,"nonzero":0.625},{"step":259,"nonzero":0.6458333333333334},{"step":261,"nonzero":0.4375},{"step":263,"nonzero":0.3958333333333333},{"step":265,"nonzero":0.7291666666666666},{"step":267,"nonzero":0.4791666666666667},{"step":269,"nonzero":0.14583333333333334},{"step":271,"nonzero":0.6666666666666666},{"step":273,"nonzero":0.7291666666666666},{"step":275,"nonzero":0.375},{"step":277,"nonzero":0.4375},{"step":279,"nonzero":0.6875},{"step":281,"nonzero":0.6041666666666666},{"step":283,"nonzero":0.7291666666666666},{"step":285,"nonzero":0.5833333333333334},{"step":287,"nonzero":0.5833333333333334},{"step":289,"nonzero":0.6041666666666666},{"step":291,"nonzero":0.625},{"step":293,"nonzero":0.6875},{"step":295,"nonzero":0.6041666666666666},{"step":297,"nonzero":0.625},{"step":299,"nonzero":0.7083333333333334},{"step":301,"nonzero":0.5416666666666666},{"step":303,"nonzero":0.5},{"step":305,"nonzero":0.5833333333333334},{"step":307,"nonzero":0.2916666666666667},{"step":309,"nonzero":0.5833333333333334},{"step":311,"nonzero":0.4583333333333333},{"step":313,"nonzero":0.5625},{"step":315,"nonzero":0.5},{"step":317,"nonzero":0.7916666666666666},{"step":319,"nonzero":0.7291666666666666},{"step":321,"nonzero":0.1875},{"step":323,"nonzero":0.7916666666666666},{"step":325,"nonzero":0.6458333333333334},{"step":327,"nonzero":0.5625},{"step":329,"nonzero":0.6458333333333334},{"step":331,"nonzero":0.3125},{"step":333,"nonzero":0.5416666666666666},{"step":335,"nonzero":0.4791666666666667},{"step":337,"nonzero":0.5416666666666666},{"step":339,"nonzero":0.375},{"step":341,"nonzero":0.4583333333333333},{"step":343,"nonzero":0.6041666666666666},{"step":345,"nonzero":0.4791666666666667},{"step":347,"nonzero":0.5416666666666666},{"step":349,"nonzero":0.5625},{"step":351,"nonzero":0.3541666666666667},{"step":353,"nonzero":0.7708333333333334},{"step":355,"nonzero":0.7291666666666666},{"step":357,"nonzero":0.5208333333333334},{"step":359,"nonzero":0.6666666666666666},{"step":361,"nonzero":0.4375},{"step":363,"nonzero":0.625},{"step":365,"nonzero":0.5625},{"step":367,"nonzero":0.6458333333333334},{"step":369,"nonzero":0.6041666666666666},{"step":371,"nonzero":0.5},{"step":373,"nonzero":0.9166666666666666},{"step":375,"nonzero":0.7708333333333334},{"step":377,"nonzero":0.625},{"step":379,"nonzero":0.5},{"step":381,"nonzero":0.4791666666666667},{"step":383,"nonzero":0.9583333333333334},{"step":385,"nonzero":0.7916666666666666},{"step":387,"nonzero":0.5833333333333334},{"step":389,"nonzero":0.7291666666666666},{"step":391,"nonzero":0.4791666666666667},{"step":393,"nonzero":0.7708333333333334},{"step":395,"nonzero":0.6875},{"step":397,"nonzero":0.7916666666666666},{"step":399,"nonzero":0.5833333333333334},{"step":401,"nonzero":0.75},{"step":403,"nonzero":0.4375},{"step":405,"nonzero":0.6458333333333334},{"step":407,"nonzero":0.6666666666666666},{"step":409,"nonzero":0.6041666666666666},{"step":411,"nonzero":0.7083333333333334},{"step":413,"nonzero":0.625},{"step":415,"nonzero":0.5833333333333334},{"step":417,"nonzero":0.8541666666666666},{"step":419,"nonzero":0.7083333333333334},{"step":421,"nonzero":0.4583333333333333},{"step":423,"nonzero":0.4166666666666667},{"step":425,"nonzero":0.7708333333333334},{"step":427,"nonzero":0.7916666666666666},{"step":429,"nonzero":0.5416666666666666},{"step":431,"nonzero":0.4583333333333333},{"step":433,"nonzero":0.6458333333333334},{"step":435,"nonzero":0.8541666666666666},{"step":437,"nonzero":0.8125},{"step":439,"nonzero":0.3958333333333333},{"step":441,"nonzero":0.3125},{"step":443,"nonzero":0.4791666666666667},{"step":445,"nonzero":0.3125},{"step":447,"nonzero":0.6458333333333334},{"step":449,"nonzero":0.5625},{"step":451,"nonzero":0.4583333333333333},{"step":453,"nonzero":0.5833333333333334},{"step":455,"nonzero":0.625},{"step":457,"nonzero":0.7083333333333334},{"step":459,"nonzero":0.6041666666666666},{"step":461,"nonzero":0.6458333333333334},{"step":463,"nonzero":0.6875},{"step":465,"nonzero":0.75},{"step":467,"nonzero":0.08333333333333333},{"step":469,"nonzero":0.4375},{"step":471,"nonzero":0.3333333333333333},{"step":473,"nonzero":0.8333333333333334},{"step":475,"nonzero":0.9375},{"step":477,"nonzero":0.4583333333333333},{"step":479,"nonzero":0.6458333333333334},{"step":481,"nonzero":0.7916666666666666},{"step":483,"nonzero":0.5416666666666666},{"step":485,"nonzero":0.4791666666666667},{"step":487,"nonzero":0.10416666666666667},{"step":489,"nonzero":0.7708333333333334},{"step":491,"nonzero":0.7708333333333334},{"step":493,"nonzero":0.875},{"step":495,"nonzero":0.8541666666666666},{"step":497,"nonzero":0.7708333333333334},{"step":499,"nonzero":0.6041666666666666},{"step":501,"nonzero":0.5},{"step":503,"nonzero":0.875},{"step":505,"nonzero":0.7291666666666666},{"step":507,"nonzero":0.75},{"step":509,"nonzero":0.7083333333333334},{"step":511,"nonzero":0.7708333333333334},{"step":513,"nonzero":0.6458333333333334},{"step":515,"nonzero":0.6875},{"step":517,"nonzero":0.6666666666666666},{"step":519,"nonzero":0.7916666666666666},{"step":521,"nonzero":0.4375},{"step":523,"nonzero":0.4166666666666667},{"step":525,"nonzero":0.7291666666666666},{"step":527,"nonzero":0.75},{"step":529,"nonzero":0.3333333333333333},{"step":531,"nonzero":0.7916666666666666},{"step":533,"nonzero":0.6041666666666666},{"step":535,"nonzero":0.5833333333333334},{"step":537,"nonzero":0.8333333333333334},{"step":539,"nonzero":0.6875},{"step":541,"nonzero":0.4791666666666667},{"step":543,"nonzero":0.6666666666666666},{"step":545,"nonzero":0.4375},{"step":547,"nonzero":0.6041666666666666},{"step":549,"nonzero":0.5208333333333334},{"step":551,"nonzero":0.9166666666666666},{"step":553,"nonzero":0.5208333333333334},{"step":555,"nonzero":0.6458333333333334},{"step":557,"nonzero":0.8541666666666666},{"step":559,"nonzero":0.4166666666666667},{"step":561,"nonzero":0.6875},{"step":563,"nonzero":0.6458333333333334},{"step":565,"nonzero":0.5833333333333334},{"step":567,"nonzero":0.6666666666666666},{"step":569,"nonzero":0.7083333333333334},{"step":571,"nonzero":0.7708333333333334},{"step":573,"nonzero":0.75},{"step":575,"nonzero":0.5208333333333334},{"step":577,"nonzero":0.5625},{"step":579,"nonzero":0.5833333333333334},{"step":581,"nonzero":0.875},{"step":583,"nonzero":0.5208333333333334},{"step":585,"nonzero":0.5416666666666666},{"step":587,"nonzero":0.4583333333333333},{"step":589,"nonzero":0.9791666666666666},{"step":591,"nonzero":0.6666666666666666},{"step":593,"nonzero":0.7916666666666666},{"step":595,"nonzero":0.3333333333333333},{"step":597,"nonzero":0.7916666666666666},{"step":599,"nonzero":0.7291666666666666},{"step":601,"nonzero":0.4791666666666667},{"step":603,"nonzero":0.8958333333333334},{"step":605,"nonzero":0.4583333333333333},{"step":607,"nonzero":0.5833333333333334},{"step":609,"nonzero":0.8541666666666666},{"step":611,"nonzero":0.5},{"step":613,"nonzero":0.6041666666666666},{"step":615,"nonzero":0.8125},{"step":617,"nonzero":0.625},{"step":619,"nonzero":0.625},{"step":621,"nonzero":0.8958333333333334},{"step":623,"nonzero":0.6458333333333334},{"step":625,"nonzero":0.0},{"step":627,"nonzero":0.6041666666666666},{"step":629,"nonzero":0.3125},{"step":631,"nonzero":0.3958333333333333},{"step":633,"nonzero":0.75},{"step":635,"nonzero":0.7916666666666666},{"step":637,"nonzero":0.8125},{"step":639,"nonzero":0.4583333333333333},{"step":641,"nonzero":0.3958333333333333},{"step":643,"nonzero":0.6666666666666666},{"step":645,"nonzero":0.6458333333333334},{"step":647,"nonzero":0.9583333333333334},{"step":649,"nonzero":0.7916666666666666},{"step":651,"nonzero":0.6458333333333334},{"step":653,"nonzero":0.8333333333333334},{"step":655,"nonzero":0.625},{"step":657,"nonzero":0.9375},{"step":659,"nonzero":0.5208333333333334},{"step":661,"nonzero":0.7291666666666666},{"step":663,"nonzero":0.5},{"step":665,"nonzero":0.9166666666666666},{"step":667,"nonzero":0.625},{"step":669,"nonzero":0.8125},{"step":671,"nonzero":0.5625},{"step":673,"nonzero":0.5833333333333334},{"step":675,"nonzero":0.5625},{"step":677,"nonzero":0.6875},{"step":679,"nonzero":0.6875},{"step":681,"nonzero":0.6458333333333334},{"step":683,"nonzero":0.875},{"step":685,"nonzero":0.875},{"step":687,"nonzero":0.4583333333333333},{"step":689,"nonzero":0.8125},{"step":691,"nonzero":0.5625},{"step":693,"nonzero":0.9166666666666666},{"step":695,"nonzero":0.4375},{"step":697,"nonzero":0.8958333333333334},{"step":699,"nonzero":0.8333333333333334},{"step":701,"nonzero":0.9583333333333334},{"step":703,"nonzero":0.6875},{"step":705,"nonzero":0.5},{"step":707,"nonzero":0.6041666666666666},{"step":709,"nonzero":0.7083333333333334},{"step":711,"nonzero":0.9166666666666666},{"step":713,"nonzero":0.8125},{"step":715,"nonzero":0.4583333333333333},{"step":717,"nonzero":0.8333333333333334},{"step":719,"nonzero":0.625},{"step":721,"nonzero":0.4791666666666667},{"step":723,"nonzero":0.8125},{"step":725,"nonzero":0.6875},{"step":727,"nonzero":0.4166666666666667},{"step":729,"nonzero":0.5416666666666666},{"step":731,"nonzero":0.5833333333333334},{"step":733,"nonzero":0.7916666666666666},{"step":735,"nonzero":0.6458333333333334},{"step":737,"nonzero":0.875},{"step":739,"nonzero":0.7291666666666666},{"step":741,"nonzero":0.6666666666666666},{"step":743,"nonzero":0.7708333333333334},{"step":745,"nonzero":0.3125},{"step":747,"nonzero":0.6458333333333334},{"step":749,"nonzero":0.5416666666666666},{"step":751,"nonzero":0.8333333333333334},{"step":753,"nonzero":0.5416666666666666},{"step":755,"nonzero":0.625},{"step":757,"nonzero":0.8333333333333334},{"step":759,"nonzero":0.6875},{"step":761,"nonzero":0.8541666666666666},{"step":763,"nonzero":1.0},{"step":765,"nonzero":0.6666666666666666},{"step":767,"nonzero":0.9375},{"step":769,"nonzero":0.4791666666666667},{"step":771,"nonzero":0.7708333333333334},{"step":773,"nonzero":0.7083333333333334},{"step":775,"nonzero":0.875},{"step":777,"nonzero":0.7916666666666666},{"step":779,"nonzero":0.6458333333333334},{"step":781,"nonzero":0.75},{"step":783,"nonzero":0.375},{"step":785,"nonzero":0.8958333333333334},{"step":787,"nonzero":0.625},{"step":789,"nonzero":0.625},{"step":791,"nonzero":0.6458333333333334},{"step":793,"nonzero":0.5416666666666666},{"step":795,"nonzero":0.6041666666666666},{"step":797,"nonzero":0.6666666666666666},{"step":799,"nonzero":0.7916666666666666},{"step":801,"nonzero":0.6458333333333334},{"step":803,"nonzero":0.7083333333333334},{"step":805,"nonzero":0.8541666666666666},{"step":807,"nonzero":0.2708333333333333},{"step":809,"nonzero":0.9375},{"step":811,"nonzero":0.875},{"step":813,"nonzero":0.75},{"step":815,"nonzero":0.6041666666666666},{"step":817,"nonzero":0.6458333333333334},{"step":819,"nonzero":0.7083333333333334},{"step":821,"nonzero":0.6458333333333334},{"step":823,"nonzero":0.75},{"step":825,"nonzero":0.9583333333333334},{"step":827,"nonzero":0.7083333333333334},{"step":829,"nonzero":0.4583333333333333},{"step":831,"nonzero":0.875},{"step":833,"nonzero":0.7916666666666666},{"step":835,"nonzero":0.7708333333333334},{"step":837,"nonzero":0.8125},{"step":839,"nonzero":0.7083333333333334},{"step":841,"nonzero":0.5833333333333334},{"step":843,"nonzero":0.7708333333333334},{"step":845,"nonzero":0.8958333333333334},{"step":847,"nonzero":0.625},{"step":849,"nonzero":0.5833333333333334},{"step":851,"nonzero":0.8333333333333334},{"step":853,"nonzero":0.875},{"step":855,"nonzero":0.7708333333333334},{"step":857,"nonzero":0.8958333333333334},{"step":859,"nonzero":0.8541666666666666},{"step":861,"nonzero":0.5625},{"step":863,"nonzero":0.6875},{"step":865,"nonzero":0.8333333333333334},{"step":867,"nonzero":0.1875},{"step":869,"nonzero":0.8125},{"step":871,"nonzero":0.5833333333333334},{"step":873,"nonzero":0.8541666666666666},{"step":875,"nonzero":0.7916666666666666},{"step":877,"nonzero":0.875},{"step":879,"nonzero":0.9583333333333334},{"step":881,"nonzero":0.7291666666666666},{"step":883,"nonzero":0.4583333333333333},{"step":885,"nonzero":0.8125},{"step":887,"nonzero":0.7083333333333334},{"step":889,"nonzero":0.75},{"step":891,"nonzero":0.5416666666666666},{"step":893,"nonzero":0.20833333333333334},{"step":895,"nonzero":0.7708333333333334},{"step":897,"nonzero":0.5208333333333334},{"step":899,"nonzero":0.7291666666666666},{"step":901,"nonzero":0.8541666666666666},{"step":903,"nonzero":0.875},{"step":905,"nonzero":0.8125},{"step":907,"nonzero":0.7708333333333334},{"step":909,"nonzero":0.4791666666666667},{"step":911,"nonzero":0.3125},{"step":913,"nonzero":0.7291666666666666},{"step":915,"nonzero":0.9583333333333334},{"step":917,"nonzero":0.7083333333333334},{"step":919,"nonzero":0.4791666666666667},{"step":921,"nonzero":0.6458333333333334},{"step":923,"nonzero":0.6041666666666666},{"step":925,"nonzero":0.6041666666666666},{"step":927,"nonzero":0.7083333333333334},{"step":929,"nonzero":0.8125},{"step":931,"nonzero":0.8125},{"step":933,"nonzero":0.625},{"step":935,"nonzero":0.6041666666666666},{"step":937,"nonzero":0.9166666666666666},{"step":939,"nonzero":0.7916666666666666},{"step":941,"nonzero":0.7916666666666666},{"step":943,"nonzero":1.0},{"step":945,"nonzero":0.6041666666666666},{"step":947,"nonzero":0.6666666666666666},{"step":949,"nonzero":0.7708333333333334},{"step":951,"nonzero":0.7916666666666666},{"step":953,"nonzero":0.8333333333333334},{"step":955,"nonzero":0.4375},{"step":957,"nonzero":0.9583333333333334},{"step":959,"nonzero":0.6875},{"step":961,"nonzero":0.625},{"step":963,"nonzero":0.6458333333333334},{"step":965,"nonzero":0.6458333333333334},{"step":967,"nonzero":0.8541666666666666},{"step":969,"nonzero":0.8125},{"step":971,"nonzero":0.7708333333333334},{"step":973,"nonzero":0.9375},{"step":975,"nonzero":0.7291666666666666},{"step":977,"nonzero":0.625},{"step":979,"nonzero":0.8125},{"step":981,"nonzero":0.6666666666666666},{"step":983,"nonzero":0.6666666666666666},{"step":985,"nonzero":0.4583333333333333},{"step":987,"nonzero":0.9791666666666666},{"step":989,"nonzero":0.7083333333333334},{"step":991,"nonzero":0.7916666666666666},{"step":993,"nonzero":0.7916666666666666},{"step":995,"nonzero":0.6458333333333334},{"step":997,"nonzero":0.6041666666666666},{"step":999,"nonzero":0.6041666666666666},{"step":1001,"nonzero":0.75},{"step":1003,"nonzero":0.7916666666666666},{"step":1005,"nonzero":0.5},{"step":1007,"nonzero":0.8541666666666666},{"step":1009,"nonzero":0.4375},{"step":1011,"nonzero":0.75},{"step":1013,"nonzero":0.7708333333333334},{"step":1015,"nonzero":0.7708333333333334},{"step":1017,"nonzero":0.7916666666666666},{"step":1019,"nonzero":0.8958333333333334},{"step":1021,"nonzero":0.7916666666666666},{"step":1023,"nonzero":0.8958333333333334},{"step":1025,"nonzero":0.9375},{"step":1027,"nonzero":0.5833333333333334},{"step":1029,"nonzero":0.875},{"step":1031,"nonzero":0.8125},{"step":1033,"nonzero":0.6666666666666666},{"step":1035,"nonzero":0.9583333333333334},{"step":1037,"nonzero":0.7083333333333334},{"step":1039,"nonzero":0.5},{"step":1041,"nonzero":0.8125},{"step":1043,"nonzero":0.5208333333333334},{"step":1045,"nonzero":0.8541666666666666},{"step":1047,"nonzero":0.8541666666666666},{"step":1049,"nonzero":0.8958333333333334},{"step":1051,"nonzero":0.8958333333333334},{"step":1053,"nonzero":0.8333333333333334},{"step":1055,"nonzero":0.7083333333333334},{"step":1057,"nonzero":0.375},{"step":1059,"nonzero":0.7291666666666666},{"step":1061,"nonzero":0.8541666666666666},{"step":1063,"nonzero":0.9375},{"step":1065,"nonzero":0.875},{"step":1067,"nonzero":0.7708333333333334},{"step":1069,"nonzero":0.8125},{"step":1071,"nonzero":0.7708333333333334},{"step":1073,"nonzero":0.8958333333333334},{"step":1075,"nonzero":1.0},{"step":1077,"nonzero":0.4166666666666667},{"step":1079,"nonzero":0.9166666666666666},{"step":1081,"nonzero":0.7291666666666666},{"step":1083,"nonzero":0.8958333333333334},{"step":1085,"nonzero":0.7083333333333334},{"step":1087,"nonzero":0.75},{"step":1089,"nonzero":0.8333333333333334},{"step":1091,"nonzero":0.75},{"step":1093,"nonzero":1.0},{"step":1095,"nonzero":0.7083333333333334},{"step":1097,"nonzero":0.7708333333333334},{"step":1099,"nonzero":0.8333333333333334},{"step":1101,"nonzero":0.7916666666666666},{"step":1103,"nonzero":0.7708333333333334},{"step":1105,"nonzero":0.9166666666666666},{"step":1107,"nonzero":0.875},{"step":1109,"nonzero":0.6458333333333334},{"step":1111,"nonzero":0.7291666666666666},{"step":1113,"nonzero":0.7708333333333334},{"step":1115,"nonzero":0.8958333333333334},{"step":1117,"nonzero":0.7916666666666666},{"step":1119,"nonzero":0.1875},{"step":1121,"nonzero":0.8541666666666666},{"step":1123,"nonzero":0.8541666666666666},{"step":1125,"nonzero":0.7291666666666666},{"step":1127,"nonzero":0.9166666666666666},{"step":1129,"nonzero":0.7291666666666666},{"step":1131,"nonzero":0.9375},{"step":1133,"nonzero":1.0},{"step":1135,"nonzero":0.7291666666666666},{"step":1137,"nonzero":0.6875},{"step":1139,"nonzero":0.6666666666666666},{"step":1141,"nonzero":0.8958333333333334},{"step":1143,"nonzero":0.9166666666666666},{"step":1145,"nonzero":0.6666666666666666},{"step":1147,"nonzero":0.6875},{"step":1149,"nonzero":0.6666666666666666},{"step":1151,"nonzero":0.7708333333333334},{"step":1153,"nonzero":0.7083333333333334},{"step":1155,"nonzero":0.9583333333333334},{"step":1157,"nonzero":0.4791666666666667},{"step":1159,"nonzero":0.9166666666666666},{"step":1161,"nonzero":0.5416666666666666},{"step":1163,"nonzero":0.8125},{"step":1165,"nonzero":0.8125},{"step":1167,"nonzero":0.7291666666666666},{"step":1169,"nonzero":0.8125},{"step":1171,"nonzero":0.7291666666666666},{"step":1173,"nonzero":0.7083333333333334},{"step":1175,"nonzero":0.875},{"step":1177,"nonzero":0.8958333333333334},{"step":1179,"nonzero":0.8958333333333334},{"step":1181,"nonzero":0.8958333333333334},{"step":1183,"nonzero":0.5416666666666666},{"step":1185,"nonzero":0.875},{"step":1187,"nonzero":0.5833333333333334},{"step":1189,"nonzero":0.4583333333333333},{"step":1191,"nonzero":0.6875},{"step":1193,"nonzero":0.9166666666666666},{"step":1195,"nonzero":0.6666666666666666},{"step":1197,"nonzero":0.5625},{"step":1199,"nonzero":0.8125}]},{"id":"cispo-repor-lora-500","label":"CISPO + REPO-R \u00b7 LoRA \u00b7 500 prompts","run_ids":["1790891672","1790920657"],"loss":"CISPO","adapter":"LoRA","prompts":500,"repo_r":true,"holdout_by_checkpoint":[{"step":0,"holdout":-1.3664,"train_core":-0.9836,"valid_rate":0.9},{"step":100,"holdout":-0.4798,"train_core":-0.3529,"valid_rate":0.94},{"step":200,"holdout":0.0562,"train_core":0.0072,"valid_rate":0.94},{"step":300,"holdout":0.4056,"train_core":0.1832,"valid_rate":0.96},{"step":400,"holdout":0.8274,"train_core":0.4531,"valid_rate":0.92},{"step":500,"holdout":1.3222,"train_core":0.6938,"valid_rate":0.94},{"step":600,"holdout":1.3275,"train_core":0.7282,"valid_rate":1.0}],"selected_step":600,"selected_holdout":1.3275,"train_steps":[{"step":1,"reward":-2.4800984859466553,"core":-0.7827860713005066,"cost":-0.09962290525436401,"kl":0.0,"entropy":0.510869562625885},{"step":3,"reward":-3.5488293170928955,"core":-1.0702965259552002,"cost":-0.2634730339050293,"kl":0.0008120036509353667,"entropy":0.493293896317482},{"step":5,"reward":-3.1384308338165283,"core":-0.9482024312019348,"cost":-0.1886027604341507,"kl":0.0010438511526444927,"entropy":0.5534127056598663},{"step":7,"reward":-3.0718936920166016,"core":-0.9437708258628845,"cost":-0.16760022938251495,"kl":0.00241644965717569,"entropy":0.5269011333584785},{"step":9,"reward":-4.53913688659668,"core":-1.3435837030410767,"cost":-0.4506637156009674,"kl":0.005227072921115905,"entropy":0.557785302400589},{"step":11,"reward":-1.9578577280044556,"core":-0.5951321125030518,"cost":-0.12048957496881485,"kl":0.004965615691617131,"entropy":0.5049529671669006},{"step":13,"reward":-4.503388404846191,"core":-1.3267731666564941,"cost":-0.47321251034736633,"kl":0.00759335036855191,"entropy":0.5072611644864082},{"step":15,"reward":-2.7714779376983643,"core":-0.8005001544952393,"cost":-0.19795282185077667,"kl":0.01671960623934865,"entropy":0.5753631293773651},{"step":17,"reward":-1.7605009078979492,"core":-0.5086545348167419,"cost":-0.06781485676765442,"kl":0.023998029064387083,"entropy":0.5107320323586464},{"step":19,"reward":-1.8909387588500977,"core":-0.5569833517074585,"cost":-0.12302345037460327,"kl":0.04154130630195141,"entropy":0.5485504865646362},{"step":21,"reward":-2.235137939453125,"core":-0.6733599305152893,"cost":-0.08227641135454178,"kl":0.03085100743919611,"entropy":0.50377057492733},{"step":23,"reward":-2.1444287300109863,"core":-0.575002133846283,"cost":-0.20041199028491974,"kl":0.05054823961108923,"entropy":0.4712950438261032},{"step":25,"reward":-1.9801816940307617,"core":-0.5893372297286987,"cost":-0.05629129707813263,"kl":0.06283653061836958,"entropy":0.4789009988307953},{"step":27,"reward":-2.1311471462249756,"core":-0.6014610528945923,"cost":-0.15142269432544708,"kl":0.05753443855792284,"entropy":0.5129551738500595},{"step":29,"reward":-2.123197317123413,"core":-0.5905430912971497,"cost":-0.10652458667755127,"kl":0.09007616341114044,"entropy":0.5219669193029404},{"step":31,"reward":-2.5302388668060303,"core":-0.696448802947998,"cost":-0.1697356253862381,"kl":0.12887570820748806,"entropy":0.5854045003652573},{"step":33,"reward":-3.3708794116973877,"core":-1.0069761276245117,"cost":-0.2742716073989868,"kl":0.09573433361947536,"entropy":0.46014436334371567},{"step":35,"reward":-1.5439929962158203,"core":-0.4045928418636322,"cost":-0.043600842356681824,"kl":0.20139333978295326,"entropy":0.6313909292221069},{"step":37,"reward":-1.4089349508285522,"core":-0.40138426423072815,"cost":-0.010339883156120777,"kl":0.13695957884192467,"entropy":0.5264226943254471},{"step":39,"reward":-0.8719952702522278,"core":-0.16879235208034515,"cost":-0.05174097418785095,"kl":0.11936070956289768,"entropy":0.556674063205719},{"step":41,"reward":-2.173069953918457,"core":-0.6117315292358398,"cost":-0.16036473214626312,"kl":0.11015030927956104,"entropy":0.5513717532157898},{"step":43,"reward":-2.1589620113372803,"core":-0.6446157693862915,"cost":-0.06861349940299988,"kl":0.1606837660074234,"entropy":0.5358458831906319},{"step":45,"reward":-2.215240716934204,"core":-0.6437242031097412,"cost":-0.09183716773986816,"kl":0.12821219488978386,"entropy":0.49277982860803604},{"step":47,"reward":-1.3869585990905762,"core":-0.3816898763179779,"cost":-0.03728442266583443,"kl":0.11119243502616882,"entropy":0.47393976151943207},{"step":49,"reward":-2.382918119430542,"core":-0.7118222117424011,"cost":-0.10727540403604507,"kl":0.13018057495355606,"entropy":0.3944734036922455},{"step":51,"reward":-0.9184797406196594,"core":-0.19549721479415894,"cost":-0.02790725976228714,"kl":0.12077762745320797,"entropy":0.4651038274168968},{"step":53,"reward":-0.5368418097496033,"core":-0.054265499114990234,"cost":-0.03124135732650757,"kl":0.12983671389520168,"entropy":0.43106255680322647},{"step":55,"reward":-2.151812791824341,"core":-0.6397774815559387,"cost":-0.10316013544797897,"kl":0.16716193966567516,"entropy":0.4388536140322685},{"step":57,"reward":-0.751481294631958,"core":-0.20300066471099854,"cost":-0.03462276980280876,"kl":0.12414931692183018,"entropy":0.547082781791687},{"step":59,"reward":-1.1412979364395142,"core":-0.2220471352338791,"cost":0.0,"kl":0.12877612933516502,"entropy":0.429437592625618},{"step":61,"reward":-2.538945436477661,"core":-0.7722672820091248,"cost":-0.057715389877557755,"kl":0.11405384540557861,"entropy":0.4934469982981682},{"step":63,"reward":-1.4432722330093384,"core":-0.4615447521209717,"cost":-0.0017654799157753587,"kl":0.10079734399914742,"entropy":0.5956020504236221},{"step":65,"reward":-0.8569910526275635,"core":-0.19012944400310516,"cost":-0.02069372497498989,"kl":0.12073780037462711,"entropy":0.6556705087423325},{"step":67,"reward":-1.484519124031067,"core":-0.4083329439163208,"cost":-0.0004779427545145154,"kl":0.11089520715177059,"entropy":0.6645316183567047},{"step":69,"reward":-1.2115622758865356,"core":-0.3413606584072113,"cost":-0.02913874201476574,"kl":0.1248286571353674,"entropy":0.6494316160678864},{"step":71,"reward":-2.732174873352051,"core":-0.8138343691825867,"cost":-0.15599501132965088,"kl":0.0970850009471178,"entropy":0.5338392108678818},{"step":73,"reward":-2.121309518814087,"core":-0.6053666472434998,"cost":-0.14666682481765747,"kl":0.10887745581567287,"entropy":0.6000661700963974},{"step":75,"reward":-1.2529760599136353,"core":-0.38217854499816895,"cost":-0.007229769602417946,"kl":0.09411346539855003,"entropy":0.7401961237192154},{"step":77,"reward":-1.3433436155319214,"core":-0.35386261343955994,"cost":-0.010763020254671574,"kl":0.06782046053558588,"entropy":0.5660388767719269},{"step":79,"reward":-0.934196949005127,"core":-0.2300872802734375,"cost":-0.022778434678912163,"kl":0.09334428794682026,"entropy":0.6228888481855392},{"step":81,"reward":-2.7245099544525146,"core":-0.820999801158905,"cost":-0.13264143466949463,"kl":0.08416762016713619,"entropy":0.5588776171207428},{"step":83,"reward":-1.6320533752441406,"core":-0.4735032618045807,"cost":-0.10328177362680435,"kl":0.08237952925264835,"entropy":0.5460623204708099},{"step":85,"reward":-1.7533187866210938,"core":-0.47946664690971375,"cost":-0.10867685079574585,"kl":0.09415933303534985,"entropy":0.5908830463886261},{"step":87,"reward":-1.9059828519821167,"core":-0.581261396408081,"cost":-0.02934839390218258,"kl":0.08477222733199596,"entropy":0.6229986548423767},{"step":89,"reward":-1.1968797445297241,"core":-0.327303022146225,"cost":-0.03500719368457794,"kl":0.11398885026574135,"entropy":0.6574126482009888},{"step":91,"reward":-2.101516008377075,"core":-0.6574745774269104,"cost":-0.025512749329209328,"kl":0.10147776082158089,"entropy":0.6002525985240936},{"step":93,"reward":-1.8267383575439453,"core":-0.5370877385139465,"cost":-0.05404432490468025,"kl":0.096506891772151,"entropy":0.5560720637440681},{"step":95,"reward":-1.0735270977020264,"core":-0.2643216848373413,"cost":-0.041075218468904495,"kl":0.08992979116737843,"entropy":0.5612439960241318},{"step":97,"reward":-1.2645666599273682,"core":-0.33163538575172424,"cost":-0.07048171758651733,"kl":0.08801590465009212,"entropy":0.5508332997560501},{"step":99,"reward":-0.8274891972541809,"core":-0.14408771693706512,"cost":-0.08808356523513794,"kl":0.09658909030258656,"entropy":0.5954773277044296},{"step":101,"reward":-2.1344916820526123,"core":-0.632814884185791,"cost":-0.04911172389984131,"kl":0.08800840564072132,"entropy":0.5592847168445587},{"step":103,"reward":-1.3306113481521606,"core":-0.36620032787323,"cost":-0.06843209266662598,"kl":0.06848220434039831,"entropy":0.5145852789282799},{"step":105,"reward":-1.292330265045166,"core":-0.35778066515922546,"cost":-0.0478304959833622,"kl":0.07351408712565899,"entropy":0.4904506206512451},{"step":107,"reward":-1.9730974435806274,"core":-0.5767650008201599,"cost":-0.09719081968069077,"kl":0.0709304939955473,"entropy":0.581298753619194},{"step":109,"reward":-1.3501371145248413,"core":-0.4032789468765259,"cost":-0.02827993966639042,"kl":0.10434240102767944,"entropy":0.5799446851015091},{"step":111,"reward":-1.5157684087753296,"core":-0.4501473009586334,"cost":-0.08555164933204651,"kl":0.08515912294387817,"entropy":0.46837998926639557},{"step":113,"reward":-1.7231616973876953,"core":-0.5291450023651123,"cost":-0.029278233647346497,"kl":0.10694031603634357,"entropy":0.5289254263043404},{"step":115,"reward":-1.488197684288025,"core":-0.4433099329471588,"cost":-0.020215729251503944,"kl":0.08819893933832645,"entropy":0.478645384311676},{"step":117,"reward":-1.1627577543258667,"core":-0.29450058937072754,"cost":-0.06396086513996124,"kl":0.143237991258502,"entropy":0.45647193491458893},{"step":119,"reward":-1.7799779176712036,"core":-0.5330409407615662,"cost":-0.022091126069426537,"kl":0.11624750308692455,"entropy":0.4392082020640373},{"step":121,"reward":-1.1203179359436035,"core":-0.3291831314563751,"cost":-0.01788211055099964,"kl":0.07799918204545975,"entropy":0.3756719157099724},{"step":123,"reward":-0.652330219745636,"core":-0.13117150962352753,"cost":-0.0629754289984703,"kl":0.16080080904066563,"entropy":0.4124230146408081},{"step":125,"reward":-3.1461241245269775,"core":-0.9448473453521729,"cost":-0.19717907905578613,"kl":0.09366516955196857,"entropy":0.38842634856700897},{"step":127,"reward":-0.6804463863372803,"core":-0.1228574737906456,"cost":0.0,"kl":0.10621425695717335,"entropy":0.42613616585731506},{"step":129,"reward":-1.1203888654708862,"core":-0.26402148604393005,"cost":-0.03179478272795677,"kl":0.12242666073143482,"entropy":0.5030472353100777},{"step":131,"reward":-0.6766569018363953,"core":-0.08117839694023132,"cost":-0.01786228083074093,"kl":0.11273890547454357,"entropy":0.47875382006168365},{"step":133,"reward":-1.777504563331604,"core":-0.5311025977134705,"cost":-0.05028539523482323,"kl":0.14754786901175976,"entropy":0.5067029222846031},{"step":135,"reward":-1.4157620668411255,"core":-0.4099547863006592,"cost":-0.04370461031794548,"kl":0.11796137876808643,"entropy":0.5535534024238586},{"step":137,"reward":-0.36670342087745667,"core":-0.028423331677913666,"cost":-0.003782127285376191,"kl":0.09510370157659054,"entropy":0.4846339821815491},{"step":139,"reward":-2.3328664302825928,"core":-0.6992557644844055,"cost":-0.07796189934015274,"kl":0.17612490989267826,"entropy":0.514997273683548},{"step":141,"reward":-2.047776460647583,"core":-0.6133395433425903,"cost":-0.07314443588256836,"kl":0.11109906435012817,"entropy":0.5564313530921936},{"step":143,"reward":-0.3655236065387726,"core":-0.056271765381097794,"cost":0.0,"kl":0.10972840897738934,"entropy":0.46588384360074997},{"step":145,"reward":-1.2033213376998901,"core":-0.33281221985816956,"cost":0.0,"kl":0.08554253540933132,"entropy":0.48469530791044235},{"step":147,"reward":-1.6326899528503418,"core":-0.45493635535240173,"cost":-0.06294775009155273,"kl":0.08417806960642338,"entropy":0.5764971077442169},{"step":149,"reward":-2.125345468521118,"core":-0.622455358505249,"cost":-0.1550348997116089,"kl":0.09160518832504749,"entropy":0.44868700951337814},{"step":151,"reward":-1.3315902948379517,"core":-0.2978533208370209,"cost":-0.09141891449689865,"kl":0.11695109494030476,"entropy":0.4457639679312706},{"step":153,"reward":-1.3465720415115356,"core":-0.3843308389186859,"cost":-0.07287073880434036,"kl":0.1485990583896637,"entropy":0.47793103009462357},{"step":155,"reward":-1.7812104225158691,"core":-0.49603357911109924,"cost":-0.08790584653615952,"kl":0.1127612330019474,"entropy":0.5480482876300812},{"step":157,"reward":-1.8522869348526,"core":-0.5179188847541809,"cost":-0.1125916838645935,"kl":0.1296842359006405,"entropy":0.538073867559433},{"step":159,"reward":-1.6489216089248657,"core":-0.39250412583351135,"cost":-0.13419675827026367,"kl":0.14725694060325623,"entropy":0.5058868229389191},{"step":161,"reward":-1.1702680587768555,"core":-0.27279746532440186,"cost":-0.05504761263728142,"kl":0.12816132977604866,"entropy":0.48706502467393875},{"step":163,"reward":-1.887026309967041,"core":-0.509989857673645,"cost":-0.11354786157608032,"kl":0.13980590924620628,"entropy":0.5597225278615952},{"step":165,"reward":-1.5194343328475952,"core":-0.4310978353023529,"cost":-0.03013426996767521,"kl":0.15664932131767273,"entropy":0.5659267902374268},{"step":167,"reward":-0.8838810920715332,"core":-0.20541507005691528,"cost":-0.035914476960897446,"kl":0.1381109170615673,"entropy":0.5796419233083725},{"step":169,"reward":-0.6187387108802795,"core":0.008128941059112549,"cost":-0.0213943999260664,"kl":0.11352209560573101,"entropy":0.5843797326087952},{"step":171,"reward":-1.0783729553222656,"core":-0.2828289270401001,"cost":-0.011358626186847687,"kl":0.16340532153844833,"entropy":0.6288695484399796},{"step":173,"reward":-1.2695837020874023,"core":-0.3544520437717438,"cost":-0.04385035112500191,"kl":0.10372200608253479,"entropy":0.5955919772386551},{"step":175,"reward":-1.0669978857040405,"core":-0.2827175557613373,"cost":-0.04490415379405022,"kl":0.22441593185067177,"entropy":0.6052450239658356},{"step":177,"reward":-0.8478579521179199,"core":-0.20215146243572235,"cost":-0.006153803784400225,"kl":0.12934337370097637,"entropy":0.5696764886379242},{"step":179,"reward":-1.3596607446670532,"core":-0.3726007044315338,"cost":-0.02227764017879963,"kl":0.16459956392645836,"entropy":0.629888579249382},{"step":181,"reward":-0.9285130500793457,"core":-0.1644337922334671,"cost":-0.046186063438653946,"kl":0.14696580171585083,"entropy":0.527144655585289},{"step":183,"reward":-0.9739010334014893,"core":-0.21698980033397675,"cost":-0.0017322910716757178,"kl":0.11698947660624981,"entropy":0.4880805015563965},{"step":185,"reward":-1.0385868549346924,"core":-0.21719305217266083,"cost":-0.061608877032995224,"kl":0.13444728404283524,"entropy":0.5032499507069588},{"step":187,"reward":-1.5846613645553589,"core":-0.41437044739723206,"cost":-0.052505653351545334,"kl":0.11936609260737896,"entropy":0.5115436613559723},{"step":189,"reward":-0.4009458124637604,"core":-0.03324027359485626,"cost":-0.02457086555659771,"kl":0.13156196102499962,"entropy":0.5163770169019699},{"step":191,"reward":-1.0173958539962769,"core":-0.244746133685112,"cost":-0.027020573616027832,"kl":0.1325779426842928,"entropy":0.48062705248594284},{"step":193,"reward":-1.4790006875991821,"core":-0.4121061861515045,"cost":-0.0812501609325409,"kl":0.22004476934671402,"entropy":0.5973400622606277},{"step":195,"reward":-0.11516424268484116,"core":0.10571924597024918,"cost":-0.011283159255981445,"kl":0.21342028677463531,"entropy":0.5012779459357262},{"step":197,"reward":-0.8793805241584778,"core":-0.1978318691253662,"cost":-0.015886660665273666,"kl":0.19509097933769226,"entropy":0.45050429552793503},{"step":199,"reward":-0.8235559463500977,"core":-0.17156052589416504,"cost":-1.6898478634175262e-06,"kl":0.13529284857213497,"entropy":0.5212548300623894},{"step":201,"reward":0.6136035919189453,"core":0.4374115467071533,"cost":0.0,"kl":0.1924213543534279,"entropy":0.5055622234940529},{"step":203,"reward":-1.3257356882095337,"core":-0.35539746284484863,"cost":-0.04707059636712074,"kl":0.15703991055488586,"entropy":0.46640971302986145},{"step":205,"reward":-0.7216518521308899,"core":-0.11869990825653076,"cost":0.0,"kl":0.26360565796494484,"entropy":0.6238556355237961},{"step":207,"reward":-0.03608637675642967,"core":0.14408303797245026,"cost":-0.008862607181072235,"kl":0.22583868354558945,"entropy":0.5893662422895432},{"step":209,"reward":-1.0723987817764282,"core":-0.2541685402393341,"cost":-0.025294432416558266,"kl":0.1584884412586689,"entropy":0.4661809131503105},{"step":211,"reward":-0.18824994564056396,"core":0.036363083869218826,"cost":-0.04341330751776695,"kl":0.27174199372529984,"entropy":0.4746876433491707},{"step":213,"reward":-0.743889331817627,"core":-0.15445227921009064,"cost":0.0,"kl":0.15928149595856667,"entropy":0.5068609565496445},{"step":215,"reward":-0.6319935321807861,"core":-0.12894009053707123,"cost":0.0,"kl":0.21419956162571907,"entropy":0.5209194645285606},{"step":217,"reward":-0.5394815802574158,"core":-0.08534321933984756,"cost":-0.004982056561857462,"kl":0.2496972307562828,"entropy":0.5524256899952888},{"step":219,"reward":-0.4217790365219116,"core":0.07622400671243668,"cost":-0.07976171374320984,"kl":0.195928156375885,"entropy":0.5940190851688385},{"step":221,"reward":0.062029823660850525,"core":0.16748957335948944,"cost":0.0,"kl":0.2434137836098671,"entropy":0.5013580471277237},{"step":223,"reward":-0.7609000205993652,"core":-0.11793148517608643,"cost":-0.018365422263741493,"kl":0.19577887281775475,"entropy":0.5590034127235413},{"step":225,"reward":-1.2493053674697876,"core":-0.36055031418800354,"cost":-0.03764183074235916,"kl":0.23246512562036514,"entropy":0.53188556432724},{"step":227,"reward":-1.386194109916687,"core":-0.3655358850955963,"cost":-0.0680435374379158,"kl":0.3309290036559105,"entropy":0.39460649341344833},{"step":229,"reward":-0.6137967109680176,"core":-0.09990481287240982,"cost":-0.0220881849527359,"kl":0.150022828951478,"entropy":0.4903779476881027},{"step":231,"reward":1.1959308385849,"core":0.7186649441719055,"cost":0.0,"kl":0.20170579478144646,"entropy":0.459584578871727},{"step":233,"reward":0.43447601795196533,"core":0.256931871175766,"cost":0.0,"kl":0.19185160845518112,"entropy":0.48985805362463},{"step":235,"reward":-0.4109090566635132,"core":-0.04966684803366661,"cost":-0.015483030118048191,"kl":0.2129271999001503,"entropy":0.5581221729516983},{"step":237,"reward":0.031123897060751915,"core":0.1912509649991989,"cost":0.0,"kl":0.17470965161919594,"entropy":0.527438297867775},{"step":239,"reward":-0.5204995274543762,"core":-0.044052090495824814,"cost":-0.06013767793774605,"kl":0.1915595792233944,"entropy":0.5835366994142532},{"step":241,"reward":-1.2368502616882324,"core":-0.34307053685188293,"cost":-0.015493891201913357,"kl":0.14474189840257168,"entropy":0.458910197019577},{"step":243,"reward":-0.24509449303150177,"core":0.12337437272071838,"cost":-0.0019695693626999855,"kl":0.20039570331573486,"entropy":0.5783252418041229},{"step":245,"reward":-0.6286489367485046,"core":-0.1005837544798851,"cost":-0.0010685924207791686,"kl":0.15223411843180656,"entropy":0.5740880966186523},{"step":247,"reward":-1.0241419076919556,"core":-0.2624345123767853,"cost":-0.011762864887714386,"kl":0.14534766785800457,"entropy":0.4980396181344986},{"step":249,"reward":-0.4540437161922455,"core":0.04579510912299156,"cost":-0.05457020923495293,"kl":0.18016371503472328,"entropy":0.5669971853494644},{"step":251,"reward":0.1892353743314743,"core":0.1484895795583725,"cost":0.0,"kl":0.14123956859111786,"entropy":0.6212934106588364},{"step":253,"reward":-0.9592066407203674,"core":-0.14812566339969635,"cost":-0.0838019922375679,"kl":0.15050837770104408,"entropy":0.43761707097291946},{"step":255,"reward":-0.39830446243286133,"core":-0.016196122393012047,"cost":0.0,"kl":0.1320667862892151,"entropy":0.5599944740533829},{"step":257,"reward":-0.9281284213066101,"core":-0.1836327165365219,"cost":-0.0886983871459961,"kl":0.21453126519918442,"entropy":0.5736325830221176},{"step":259,"reward":-0.745911180973053,"core":-0.1674743890762329,"cost":-0.012011532671749592,"kl":0.11263114027678967,"entropy":0.5145115852355957},{"step":261,"reward":-1.5617977380752563,"core":-0.4440902769565582,"cost":-0.07817226648330688,"kl":0.1843564212322235,"entropy":0.5613106489181519},{"step":263,"reward":-1.0638540983200073,"core":-0.260127454996109,"cost":-0.038550443947315216,"kl":0.16306889429688454,"entropy":0.5392099171876907},{"step":265,"reward":-0.027811190113425255,"core":0.0828140377998352,"cost":-0.02958282083272934,"kl":0.15347613766789436,"entropy":0.4888388440012932},{"step":267,"reward":-1.4370969533920288,"core":-0.3958521783351898,"cost":-0.012165953405201435,"kl":0.13029879331588745,"entropy":0.43363987654447556},{"step":269,"reward":-0.8817901611328125,"core":-0.14655311405658722,"cost":-0.0015486072516068816,"kl":0.1801394782960415,"entropy":0.5175367370247841},{"step":271,"reward":-0.31211748719215393,"core":0.11366903781890869,"cost":-0.06485512852668762,"kl":0.18927612155675888,"entropy":0.46030984818935394},{"step":273,"reward":0.5848777890205383,"core":0.4021264612674713,"cost":0.0,"kl":0.21121835708618164,"entropy":0.4635762497782707},{"step":275,"reward":-0.909853458404541,"core":-0.21381385624408722,"cost":-0.040141332894563675,"kl":0.18107642605900764,"entropy":0.4251624047756195},{"step":277,"reward":-0.1996980756521225,"core":0.04048864170908928,"cost":-0.018239015713334084,"kl":0.18807169049978256,"entropy":0.46909835934638977},{"step":279,"reward":0.03408711031079292,"core":0.13682086765766144,"cost":-0.0033916558604687452,"kl":0.20325764268636703,"entropy":0.4749487563967705},{"step":281,"reward":-1.2088207006454468,"core":-0.3290535509586334,"cost":-0.008295135572552681,"kl":0.2007567510008812,"entropy":0.5377405881881714},{"step":283,"reward":-0.0446741096675396,"core":0.14769689738750458,"cost":-0.03442952781915665,"kl":0.22184891626238823,"entropy":0.3997400924563408},{"step":285,"reward":0.23303437232971191,"core":0.16614167392253876,"cost":0.0,"kl":0.1945023573935032,"entropy":0.43194733560085297},{"step":287,"reward":0.937598705291748,"core":0.5015913844108582,"cost":0.0,"kl":0.21888817101716995,"entropy":0.43477581441402435},{"step":289,"reward":-0.6087689399719238,"core":-0.11870137602090836,"cost":-0.013723775744438171,"kl":0.19435133039951324,"entropy":0.45466797798871994},{"step":291,"reward":0.376910537481308,"core":0.27013885974884033,"cost":-0.02887178771197796,"kl":0.2072168067097664,"entropy":0.5089828670024872},{"step":293,"reward":-0.04453156515955925,"core":0.090970478951931,"cost":-0.012074763886630535,"kl":0.24622224271297455,"entropy":0.4590006545186043},{"step":295,"reward":0.61869215965271,"core":0.3133186101913452,"cost":-0.006257184315472841,"kl":0.22575250267982483,"entropy":0.4355880096554756},{"step":297,"reward":0.07041727751493454,"core":0.13079629838466644,"cost":0.0,"kl":0.22812220081686974,"entropy":0.575475350022316},{"step":299,"reward":1.017878770828247,"core":0.46970561146736145,"cost":-0.00866132602095604,"kl":0.2147008180618286,"entropy":0.4589272513985634},{"step":301,"reward":0.3724256455898285,"core":0.2818913757801056,"cost":-0.014592002145946026,"kl":0.24844739586114883,"entropy":0.5591190755367279},{"step":303,"reward":-0.14568279683589935,"core":0.12149875611066818,"cost":-0.051211703568696976,"kl":0.23248126730322838,"entropy":0.5456725209951401},{"step":305,"reward":-0.6432631611824036,"core":-0.09222164750099182,"cost":-0.051275309175252914,"kl":0.2045505940914154,"entropy":0.48296797275543213},{"step":307,"reward":0.17990398406982422,"core":0.21594922244548798,"cost":-0.007311741355806589,"kl":0.300493061542511,"entropy":0.602050319314003},{"step":309,"reward":0.4117087423801422,"core":0.2547120153903961,"cost":0.0,"kl":0.12891077063977718,"entropy":0.49224182963371277},{"step":311,"reward":-0.9905714988708496,"core":-0.23425476253032684,"cost":-0.02535148710012436,"kl":0.15546482801437378,"entropy":0.5486430376768112},{"step":313,"reward":-0.6242946982383728,"core":-0.09179425239562988,"cost":-0.020577723160386086,"kl":0.20997538045048714,"entropy":0.5291296392679214},{"step":315,"reward":-0.043050963431596756,"core":0.14259837567806244,"cost":-0.019982075318694115,"kl":0.20662323012948036,"entropy":0.5808582603931427},{"step":317,"reward":0.31697896122932434,"core":0.30460605025291443,"cost":0.0,"kl":0.25057507306337357,"entropy":0.5643417537212372},{"step":319,"reward":-0.3048255741596222,"core":0.08807311207056046,"cost":0.0,"kl":0.16299154981970787,"entropy":0.5144863948225975},{"step":321,"reward":-0.031145473942160606,"core":0.15788845717906952,"cost":-0.002378406934440136,"kl":0.3380899801850319,"entropy":0.586032047867775},{"step":323,"reward":0.7382991909980774,"core":0.44051501154899597,"cost":0.0,"kl":0.23607050627470016,"entropy":0.5578959733247757},{"step":325,"reward":0.911372184753418,"core":0.5367209911346436,"cost":0.0,"kl":0.29061442241072655,"entropy":0.6004626303911209},{"step":327,"reward":-0.21136005222797394,"core":0.04309532046318054,"cost":0.0,"kl":0.2640717439353466,"entropy":0.5682990849018097},{"step":329,"reward":0.9354187846183777,"core":0.6180158853530884,"cost":0.0,"kl":0.25503215566277504,"entropy":0.5521897971630096},{"step":331,"reward":-0.5638380646705627,"core":-0.05772816017270088,"cost":-0.007132098078727722,"kl":0.27082914113998413,"entropy":0.5935508608818054},{"step":333,"reward":0.19592368602752686,"core":0.23500381410121918,"cost":-0.010976952500641346,"kl":0.2668848969042301,"entropy":0.5356012284755707},{"step":335,"reward":0.32364484667778015,"core":0.23739777505397797,"cost":0.0,"kl":0.3626183047890663,"entropy":0.5609504580497742},{"step":337,"reward":-0.3569765090942383,"core":0.012549151666462421,"cost":0.0,"kl":0.22866616025567055,"entropy":0.5995488613843918},{"step":339,"reward":0.17580850422382355,"core":0.20450247824192047,"cost":0.0,"kl":0.326026976108551,"entropy":0.5835603028535843},{"step":341,"reward":-0.3026207983493805,"core":0.019601143896579742,"cost":0.0,"kl":0.28810185194015503,"entropy":0.6213263273239136},{"step":343,"reward":0.9674808979034424,"core":0.5218861699104309,"cost":0.0,"kl":0.24685163423419,"entropy":0.5380288511514664},{"step":345,"reward":-1.0449281930923462,"core":-0.21444709599018097,"cost":-0.07852480560541153,"kl":0.2645077630877495,"entropy":0.586762398481369},{"step":347,"reward":0.18854010105133057,"core":0.1886610984802246,"cost":-0.009533170610666275,"kl":0.25425221771001816,"entropy":0.4912080690264702},{"step":349,"reward":-0.008752226829528809,"core":0.13693663477897644,"cost":-0.007651956751942635,"kl":0.27524228394031525,"entropy":0.5443075299263},{"step":351,"reward":0.3933768570423126,"core":0.31865236163139343,"cost":0.0,"kl":0.2704950086772442,"entropy":0.5501673519611359},{"step":353,"reward":0.9515964388847351,"core":0.46927908062934875,"cost":0.0,"kl":0.2671874538064003,"entropy":0.6194235384464264},{"step":355,"reward":0.4959333837032318,"core":0.33170536160469055,"cost":0.0,"kl":0.2826100289821625,"entropy":0.5772670954465866},{"step":357,"reward":0.7878106236457825,"core":0.39142611622810364,"cost":0.0,"kl":0.29536184668540955,"entropy":0.5985508412122726},{"step":359,"reward":1.2992995977401733,"core":0.6313732266426086,"cost":0.0,"kl":0.3450533300638199,"entropy":0.5118290185928345},{"step":361,"reward":0.7890819907188416,"core":0.4350118637084961,"cost":-0.009201574139297009,"kl":0.34402409195899963,"entropy":0.5742141902446747},{"step":363,"reward":0.7443597316741943,"core":0.5156164765357971,"cost":-0.02534354478120804,"kl":0.27992916107177734,"entropy":0.5838293433189392},{"step":365,"reward":0.3004573881626129,"core":0.2652449309825897,"cost":0.0,"kl":0.3167703151702881,"entropy":0.5673753768205643},{"step":367,"reward":0.6536170840263367,"core":0.43512168526649475,"cost":-0.01086991187185049,"kl":0.513423003256321,"entropy":0.5937155783176422},{"step":369,"reward":0.38935866951942444,"core":0.2765052318572998,"cost":0.0,"kl":0.3398204669356346,"entropy":0.6178755015134811},{"step":371,"reward":-0.4036359488964081,"core":-0.023950742557644844,"cost":-0.015298164449632168,"kl":0.32155459001660347,"entropy":0.545686885714531},{"step":373,"reward":0.8147340416908264,"core":0.4267942011356354,"cost":0.0,"kl":0.23977812752127647,"entropy":0.5541045963764191},{"step":375,"reward":0.25034281611442566,"core":0.2992270290851593,"cost":-0.030146127566695213,"kl":0.39856529980897903,"entropy":0.5920996516942978},{"step":377,"reward":0.2886790931224823,"core":0.27944836020469666,"cost":0.0,"kl":0.31164637207984924,"entropy":0.6198618412017822},{"step":379,"reward":-0.34203171730041504,"core":-0.05962154269218445,"cost":-0.008120276965200901,"kl":0.2352117821574211,"entropy":0.6183144897222519},{"step":381,"reward":-0.7669739127159119,"core":-0.16023486852645874,"cost":-0.05208001658320427,"kl":0.26499275490641594,"entropy":0.6177134960889816},{"step":383,"reward":0.37694430351257324,"core":0.31996020674705505,"cost":-0.030887285247445107,"kl":0.19044948741793633,"entropy":0.6268354058265686},{"step":385,"reward":0.05333958938717842,"core":0.23657143115997314,"cost":-0.03223436698317528,"kl":0.3498826026916504,"entropy":0.6119342595338821},{"step":387,"reward":-0.3296672999858856,"core":0.004848802927881479,"cost":-0.05202260613441467,"kl":0.2715051807463169,"entropy":0.6119587272405624},{"step":389,"reward":0.4201394021511078,"core":0.3423561155796051,"cost":0.0,"kl":0.2514321655035019,"entropy":0.5870322585105896},{"step":391,"reward":-0.21094898879528046,"core":0.03425595164299011,"cost":-0.0013660731492564082,"kl":0.2406451590359211,"entropy":0.5930562168359756},{"step":393,"reward":0.4662017524242401,"core":0.3685833513736725,"cost":0.0,"kl":0.20982657000422478,"entropy":0.6048166155815125},{"step":395,"reward":0.017133891582489014,"core":0.12141909450292587,"cost":-0.003974239341914654,"kl":0.252067182213068,"entropy":0.5651335418224335},{"step":397,"reward":1.257154941558838,"core":0.601955235004425,"cost":-0.003199914237484336,"kl":0.2373216263949871,"entropy":0.5852569937705994},{"step":399,"reward":0.41893434524536133,"core":0.2915152609348297,"cost":-0.010694441385567188,"kl":0.19666551426053047,"entropy":0.5808866322040558},{"step":401,"reward":0.2912902534008026,"core":0.27171269059181213,"cost":-0.0035829211119562387,"kl":0.32046205550432205,"entropy":0.5787037909030914},{"step":403,"reward":0.31445255875587463,"core":0.21349124610424042,"cost":-0.009014495648443699,"kl":0.27804455161094666,"entropy":0.6264316588640213},{"step":405,"reward":-0.0305728018283844,"core":0.1360238939523697,"cost":-0.010844950564205647,"kl":0.2651587128639221,"entropy":0.661929652094841},{"step":407,"reward":0.04135968163609505,"core":0.17149358987808228,"cost":-0.018356237560510635,"kl":0.29135937988758087,"entropy":0.6832526922225952},{"step":409,"reward":-0.1619885414838791,"core":0.09788530319929123,"cost":-0.02198638580739498,"kl":0.3001508004963398,"entropy":0.61374731361866},{"step":411,"reward":-0.4420686960220337,"core":-0.00912782084196806,"cost":-0.024158179759979248,"kl":0.2780687175691128,"entropy":0.5808645486831665},{"step":413,"reward":-0.284807413816452,"core":0.07540243864059448,"cost":-0.0284158606082201,"kl":0.2919830158352852,"entropy":0.646606370806694},{"step":415,"reward":-0.4441174566745758,"core":-0.033710185438394547,"cost":-0.004026010632514954,"kl":0.29672735929489136,"entropy":0.6223157197237015},{"step":417,"reward":0.31996193528175354,"core":0.22840212285518646,"cost":-0.0014043805422261357,"kl":0.2196575067937374,"entropy":0.6231921315193176},{"step":419,"reward":0.8322939276695251,"core":0.4956648051738739,"cost":0.0,"kl":0.25413432717323303,"entropy":0.6094013005495071},{"step":421,"reward":0.14213524758815765,"core":0.18064208328723907,"cost":-0.007036704104393721,"kl":0.3034173771739006,"entropy":0.6206006705760956},{"step":423,"reward":0.4415973126888275,"core":0.255441814661026,"cost":0.0,"kl":0.2620536535978317,"entropy":0.6193304061889648},{"step":425,"reward":0.4521050453186035,"core":0.247110053896904,"cost":-0.031158871948719025,"kl":0.31391722708940506,"entropy":0.5953061580657959},{"step":427,"reward":0.5946819186210632,"core":0.36516761779785156,"cost":0.0,"kl":0.26328034698963165,"entropy":0.565454289317131},{"step":429,"reward":0.27954718470573425,"core":0.20037980377674103,"cost":-0.007097019348293543,"kl":0.29652972519397736,"entropy":0.6080521643161774},{"step":431,"reward":0.6775744557380676,"core":0.38942813873291016,"cost":0.0,"kl":0.2925311103463173,"entropy":0.5179975926876068},{"step":433,"reward":0.8521630167961121,"core":0.4303438663482666,"cost":-0.003017408074811101,"kl":0.25456708297133446,"entropy":0.4659547731280327},{"step":435,"reward":0.31936192512512207,"core":0.3005305826663971,"cost":-0.029540566727519035,"kl":0.33850663900375366,"entropy":0.5789670646190643},{"step":437,"reward":1.0871233940124512,"core":0.5559491515159607,"cost":0.0,"kl":0.3091464415192604,"entropy":0.5310062766075134},{"step":439,"reward":0.21259288489818573,"core":0.18884700536727905,"cost":0.0,"kl":0.2845250964164734,"entropy":0.5586799383163452},{"step":441,"reward":0.23990027606487274,"core":0.22462260723114014,"cost":-0.022322511300444603,"kl":0.24034639075398445,"entropy":0.5314311534166336},{"step":443,"reward":0.3039594888687134,"core":0.2424771636724472,"cost":-0.05831824615597725,"kl":0.3731018900871277,"entropy":0.5802012979984283},{"step":445,"reward":0.6415926814079285,"core":0.3999192416667938,"cost":-0.016660034656524658,"kl":0.28778909891843796,"entropy":0.5069449543952942},{"step":447,"reward":0.1886436492204666,"core":0.20621682703495026,"cost":-0.04117254540324211,"kl":0.30766206234693527,"entropy":0.5018594861030579},{"step":449,"reward":0.5455278754234314,"core":0.39791250228881836,"cost":0.0,"kl":0.2895582392811775,"entropy":0.5926579684019089},{"step":451,"reward":0.8812001347541809,"core":0.5212567448616028,"cost":0.0,"kl":0.26833898946642876,"entropy":0.49783702939748764},{"step":453,"reward":0.2259802669286728,"core":0.18375901877880096,"cost":-0.022275030612945557,"kl":0.33270014077425003,"entropy":0.4753987044095993},{"step":455,"reward":0.28054723143577576,"core":0.33138278126716614,"cost":0.0,"kl":0.3040589764714241,"entropy":0.5509755462408066},{"step":457,"reward":1.353357195854187,"core":0.7234813570976257,"cost":0.0,"kl":0.2820982486009598,"entropy":0.5872505158185959},{"step":459,"reward":0.49310433864593506,"core":0.2857995927333832,"cost":-0.004355074837803841,"kl":0.29932715743780136,"entropy":0.4956704080104828},{"step":461,"reward":0.9686064720153809,"core":0.44217464327812195,"cost":-0.01382816955447197,"kl":0.48538190871477127,"entropy":0.4736044481396675},{"step":463,"reward":0.5259366035461426,"core":0.3844209909439087,"cost":0.0,"kl":0.30460838228464127,"entropy":0.5570484548807144},{"step":465,"reward":0.8981084823608398,"core":0.4503897726535797,"cost":-0.003580794669687748,"kl":0.305870920419693,"entropy":0.513285256922245},{"step":467,"reward":0.7483372688293457,"core":0.38526955246925354,"cost":0.0,"kl":0.33966246992349625,"entropy":0.5477927625179291},{"step":469,"reward":0.8690745234489441,"core":0.5061299800872803,"cost":-0.005262468010187149,"kl":0.2831275463104248,"entropy":0.5644214004278183},{"step":471,"reward":0.9647350311279297,"core":0.5405941605567932,"cost":0.0,"kl":0.35154661536216736,"entropy":0.46173034608364105},{"step":473,"reward":0.7426285147666931,"core":0.48921966552734375,"cost":0.0,"kl":0.29273249208927155,"entropy":0.599802240729332},{"step":475,"reward":0.9415035247802734,"core":0.5145798325538635,"cost":0.0,"kl":0.3450321927666664,"entropy":0.5290343910455704},{"step":477,"reward":0.11950608342885971,"core":0.18700002133846283,"cost":-0.0190793015062809,"kl":0.32111288607120514,"entropy":0.4386117607355118},{"step":479,"reward":0.4351801872253418,"core":0.3166564702987671,"cost":-0.015572641044855118,"kl":0.2744150087237358,"entropy":0.48704206943511963},{"step":481,"reward":1.4103494882583618,"core":0.7639291882514954,"cost":0.0,"kl":0.30959295108914375,"entropy":0.44264352321624756},{"step":483,"reward":1.0999468564987183,"core":0.5465587973594666,"cost":-0.030875956639647484,"kl":0.2945784814655781,"entropy":0.47925645112991333},{"step":485,"reward":1.148277759552002,"core":0.5265340805053711,"cost":-0.019859077408909798,"kl":0.32882019877433777,"entropy":0.5041847229003906},{"step":487,"reward":0.5962424874305725,"core":0.4410914480686188,"cost":-0.03595757484436035,"kl":0.29761914163827896,"entropy":0.4899395704269409},{"step":489,"reward":0.38172459602355957,"core":0.340329647064209,"cost":-0.0035998784005641937,"kl":0.33269933611154556,"entropy":0.52071712911129},{"step":491,"reward":-0.06986141949892044,"core":0.1147834062576294,"cost":-0.023957429453730583,"kl":0.21268903091549873,"entropy":0.4633854627609253},{"step":493,"reward":0.5999760627746582,"core":0.42401638627052307,"cost":0.0,"kl":0.294204019010067,"entropy":0.4904073625802994},{"step":495,"reward":0.8997035026550293,"core":0.47696611285209656,"cost":0.0,"kl":0.37636110931634903,"entropy":0.5082402005791664},{"step":497,"reward":0.921654999256134,"core":0.4840461313724518,"cost":0.0,"kl":0.32702913880348206,"entropy":0.4695401266217232},{"step":499,"reward":0.6903591752052307,"core":0.433573842048645,"cost":-0.0033512236550450325,"kl":0.3138698637485504,"entropy":0.5188441127538681},{"step":501,"reward":0.5705631971359253,"core":0.3264922797679901,"cost":-0.009979214519262314,"kl":0.2939600646495819,"entropy":0.5562377274036407},{"step":503,"reward":0.0444219596683979,"core":0.15609808266162872,"cost":-0.002881354419514537,"kl":0.25322067737579346,"entropy":0.4433075562119484},{"step":505,"reward":1.0467075109481812,"core":0.5386735796928406,"cost":0.0,"kl":0.34842541813850403,"entropy":0.5337105542421341},{"step":507,"reward":0.3156956136226654,"core":0.2870398163795471,"cost":-0.03922010958194733,"kl":0.37562885880470276,"entropy":0.5274716317653656},{"step":509,"reward":0.26377731561660767,"core":0.2749570906162262,"cost":0.0,"kl":0.3239951804280281,"entropy":0.5570691972970963},{"step":511,"reward":0.4626670777797699,"core":0.25302547216415405,"cost":-0.00577356293797493,"kl":0.42293260991573334,"entropy":0.5382233560085297},{"step":513,"reward":0.7170714735984802,"core":0.41954469680786133,"cost":-0.008517920970916748,"kl":0.28901220858097076,"entropy":0.553154319524765},{"step":515,"reward":1.0543874502182007,"core":0.5502078533172607,"cost":-0.0025121141225099564,"kl":0.24614408984780312,"entropy":0.491704985499382},{"step":517,"reward":0.6624557375907898,"core":0.3928596079349518,"cost":0.0,"kl":0.26375697553157806,"entropy":0.4609694257378578},{"step":519,"reward":0.2581663429737091,"core":0.25883013010025024,"cost":0.0,"kl":0.37120188027620316,"entropy":0.5334804803133011},{"step":521,"reward":1.1307145357131958,"core":0.6199820041656494,"cost":0.0,"kl":0.34198836982250214,"entropy":0.5034190714359283},{"step":523,"reward":0.8465218544006348,"core":0.4987199604511261,"cost":-0.0045206318609416485,"kl":0.2934170439839363,"entropy":0.516002707183361},{"step":525,"reward":0.1635046899318695,"core":0.19946245849132538,"cost":-0.013459901325404644,"kl":0.31560543924570084,"entropy":0.5233869850635529},{"step":527,"reward":1.056167483329773,"core":0.5284070372581482,"cost":0.0,"kl":0.2620886154472828,"entropy":0.5066010802984238},{"step":529,"reward":0.22597210109233856,"core":0.2437705546617508,"cost":0.0,"kl":0.30491866916418076,"entropy":0.5290439575910568},{"step":531,"reward":0.46684756875038147,"core":0.3107242286205292,"cost":-0.0027957253623753786,"kl":0.41047409921884537,"entropy":0.6376134306192398},{"step":533,"reward":1.2567354440689087,"core":0.6219778060913086,"cost":0.0,"kl":0.34920455515384674,"entropy":0.42589134722948074},{"step":535,"reward":0.7808544039726257,"core":0.3792104721069336,"cost":-0.0060688406229019165,"kl":0.3054903894662857,"entropy":0.5121859535574913},{"step":537,"reward":1.298169493675232,"core":0.673248291015625,"cost":-0.004464134108275175,"kl":0.3303489610552788,"entropy":0.4737781882286072},{"step":539,"reward":0.3014083802700043,"core":0.2590864598751068,"cost":-0.009478170424699783,"kl":0.27350109443068504,"entropy":0.45429224520921707},{"step":541,"reward":1.3894578218460083,"core":0.6999797821044922,"cost":0.0,"kl":0.31668761372566223,"entropy":0.5078694149851799},{"step":543,"reward":1.0615419149398804,"core":0.49996528029441833,"cost":-0.014124265871942043,"kl":0.3433217257261276,"entropy":0.4797910824418068},{"step":545,"reward":1.0252180099487305,"core":0.4672739803791046,"cost":-0.003884297562763095,"kl":0.3210265263915062,"entropy":0.5192843526601791},{"step":547,"reward":1.4217700958251953,"core":0.636181116104126,"cost":0.0,"kl":0.3500117063522339,"entropy":0.5499943643808365},{"step":549,"reward":1.2000573873519897,"core":0.5193764567375183,"cost":0.0,"kl":0.2713436745107174,"entropy":0.47412578761577606},{"step":551,"reward":0.6168307662010193,"core":0.40530070662498474,"cost":-0.00031604382093064487,"kl":0.4200827181339264,"entropy":0.5564372092485428},{"step":553,"reward":0.19694776833057404,"core":0.2833287715911865,"cost":-0.023041436448693275,"kl":0.329261165112257,"entropy":0.5124558284878731},{"step":555,"reward":0.57329261302948,"core":0.3536059856414795,"cost":-0.001367418677546084,"kl":0.29179465025663376,"entropy":0.4834766313433647},{"step":557,"reward":0.840362548828125,"core":0.5134958624839783,"cost":0.0,"kl":0.3471021205186844,"entropy":0.537771075963974},{"step":559,"reward":1.0265146493911743,"core":0.5791831612586975,"cost":0.0,"kl":0.3111248016357422,"entropy":0.5330992788076401},{"step":561,"reward":-0.18618559837341309,"core":0.074834443628788,"cost":-0.0161135196685791,"kl":0.28939542174339294,"entropy":0.479464627802372},{"step":563,"reward":0.930652379989624,"core":0.4947871267795563,"cost":-0.0005642910837195814,"kl":0.3485849052667618,"entropy":0.4662451669573784},{"step":565,"reward":0.2833062708377838,"core":0.26659098267555237,"cost":-0.021150751039385796,"kl":0.33223509043455124,"entropy":0.5666604787111282},{"step":567,"reward":1.303445816040039,"core":0.70428866147995,"cost":-0.003787769703194499,"kl":0.3549632430076599,"entropy":0.5083301290869713},{"step":569,"reward":0.33180490136146545,"core":0.3307662904262543,"cost":-0.012725054286420345,"kl":0.24673213437199593,"entropy":0.47621093690395355},{"step":571,"reward":0.6443237662315369,"core":0.4527616500854492,"cost":-0.011360798962414265,"kl":0.41246988624334335,"entropy":0.4961036667227745},{"step":573,"reward":1.1725226640701294,"core":0.570435106754303,"cost":-0.0037225086707621813,"kl":0.2731010727584362,"entropy":0.5194188207387924},{"step":575,"reward":0.9832114577293396,"core":0.54311603307724,"cost":-0.0035379526671022177,"kl":0.30871153622865677,"entropy":0.5252924263477325},{"step":577,"reward":0.2536717653274536,"core":0.24613185226917267,"cost":0.0,"kl":0.3449707105755806,"entropy":0.5292720198631287},{"step":579,"reward":1.0810900926589966,"core":0.6634809374809265,"cost":-0.016758013516664505,"kl":0.300712700933218,"entropy":0.4958314374089241},{"step":581,"reward":0.6697303652763367,"core":0.4310946464538574,"cost":-0.010316611267626286,"kl":0.36836883425712585,"entropy":0.5086673498153687},{"step":583,"reward":0.9905633926391602,"core":0.6421994566917419,"cost":-0.01087102945894003,"kl":0.3354504033923149,"entropy":0.48084093630313873},{"step":585,"reward":0.8392133712768555,"core":0.513742983341217,"cost":-0.0031171326991170645,"kl":0.38461438566446304,"entropy":0.4899895265698433},{"step":587,"reward":0.5756908059120178,"core":0.4356468915939331,"cost":-0.02011292241513729,"kl":0.3478536605834961,"entropy":0.5890531092882156},{"step":589,"reward":0.7186970114707947,"core":0.42595359683036804,"cost":-0.012100148014724255,"kl":0.39635858684778214,"entropy":0.53627510368824},{"step":591,"reward":1.2964932918548584,"core":0.5869501233100891,"cost":0.0,"kl":0.3249842971563339,"entropy":0.5256503820419312},{"step":593,"reward":0.9853283762931824,"core":0.5570579171180725,"cost":0.0,"kl":0.2851564921438694,"entropy":0.48797110468149185},{"step":595,"reward":0.43336907029151917,"core":0.3003931939601898,"cost":0.0,"kl":0.31700148433446884,"entropy":0.5636470168828964},{"step":597,"reward":0.8956702351570129,"core":0.4998648464679718,"cost":-0.010431190021336079,"kl":0.2906138077378273,"entropy":0.47726713865995407},{"step":599,"reward":0.6603398323059082,"core":0.38913699984550476,"cost":-0.0067468248307704926,"kl":0.3525463044643402,"entropy":0.4908304065465927}],"nonzero_advantage":[{"step":1,"nonzero":0.4166666666666667},{"step":3,"nonzero":0.125},{"step":5,"nonzero":0.3541666666666667},{"step":7,"nonzero":0.375},{"step":9,"nonzero":0.0},{"step":11,"nonzero":0.4583333333333333},{"step":13,"nonzero":0.20833333333333334},{"step":15,"nonzero":0.5},{"step":17,"nonzero":0.5833333333333334},{"step":19,"nonzero":0.3541666666666667},{"step":21,"nonzero":0.4791666666666667},{"step":23,"nonzero":0.5416666666666666},{"step":25,"nonzero":0.5},{"step":27,"nonzero":0.5625},{"step":29,"nonzero":0.5208333333333334},{"step":31,"nonzero":0.5625},{"step":33,"nonzero":0.20833333333333334},{"step":35,"nonzero":0.5833333333333334},{"step":37,"nonzero":0.4375},{"step":39,"nonzero":0.7083333333333334},{"step":41,"nonzero":0.3125},{"step":43,"nonzero":0.5625},{"step":45,"nonzero":0.6458333333333334},{"step":47,"nonzero":0.625},{"step":49,"nonzero":0.4166666666666667},{"step":51,"nonzero":0.5},{"step":53,"nonzero":0.8333333333333334},{"step":55,"nonzero":0.6041666666666666},{"step":57,"nonzero":0.8125},{"step":59,"nonzero":0.75},{"step":61,"nonzero":0.3958333333333333},{"step":63,"nonzero":0.7291666666666666},{"step":65,"nonzero":0.6666666666666666},{"step":67,"nonzero":0.8125},{"step":69,"nonzero":0.5416666666666666},{"step":71,"nonzero":0.25},{"step":73,"nonzero":0.4166666666666667},{"step":75,"nonzero":0.625},{"step":77,"nonzero":0.6666666666666666},{"step":79,"nonzero":0.7916666666666666},{"step":81,"nonzero":0.4791666666666667},{"step":83,"nonzero":0.5416666666666666},{"step":85,"nonzero":0.5625},{"step":87,"nonzero":0.5625},{"step":89,"nonzero":0.6666666666666666},{"step":91,"nonzero":0.4791666666666667},{"step":93,"nonzero":0.4583333333333333},{"step":95,"nonzero":0.5416666666666666},{"step":97,"nonzero":0.7291666666666666},{"step":99,"nonzero":0.875},{"step":101,"nonzero":0.22916666666666666},{"step":103,"nonzero":0.7916666666666666},{"step":105,"nonzero":0.6458333333333334},{"step":107,"nonzero":0.6666666666666666},{"step":109,"nonzero":0.7708333333333334},{"step":111,"nonzero":0.6458333333333334},{"step":113,"nonzero":0.4166666666666667},{"step":115,"nonzero":0.5416666666666666},{"step":117,"nonzero":0.8125},{"step":119,"nonzero":0.6458333333333334},{"step":121,"nonzero":0.7916666666666666},{"step":123,"nonzero":0.875},{"step":125,"nonzero":0.25},{"step":127,"nonzero":0.7291666666666666},{"step":129,"nonzero":0.8541666666666666},{"step":131,"nonzero":0.7291666666666666},{"step":133,"nonzero":0.7083333333333334},{"step":135,"nonzero":0.75},{"step":137,"nonzero":0.875},{"step":139,"nonzero":0.3541666666666667},{"step":141,"nonzero":0.5208333333333334},{"step":143,"nonzero":0.9375},{"step":145,"nonzero":0.5208333333333334},{"step":147,"nonzero":0.5},{"step":149,"nonzero":0.5},{"step":151,"nonzero":0.8125},{"step":153,"nonzero":0.6875},{"step":155,"nonzero":0.625},{"step":157,"nonzero":0.7291666666666666},{"step":159,"nonzero":0.625},{"step":161,"nonzero":0.75},{"step":163,"nonzero":0.6875},{"step":165,"nonzero":0.5208333333333334},{"step":167,"nonzero":0.875},{"step":169,"nonzero":0.8333333333333334},{"step":171,"nonzero":0.8333333333333334},{"step":173,"nonzero":0.6041666666666666},{"step":175,"nonzero":0.8125},{"step":177,"nonzero":0.6666666666666666},{"step":179,"nonzero":0.5625},{"step":181,"nonzero":0.7083333333333334},{"step":183,"nonzero":0.75},{"step":185,"nonzero":0.6458333333333334},{"step":187,"nonzero":0.5833333333333334},{"step":189,"nonzero":0.9166666666666666},{"step":191,"nonzero":0.8333333333333334},{"step":193,"nonzero":0.7708333333333334},{"step":195,"nonzero":0.9375},{"step":197,"nonzero":0.8333333333333334},{"step":199,"nonzero":0.9583333333333334},{"step":201,"nonzero":0.9791666666666666},{"step":203,"nonzero":0.625},{"step":205,"nonzero":0.875},{"step":207,"nonzero":0.9791666666666666},{"step":209,"nonzero":0.6666666666666666},{"step":211,"nonzero":0.9166666666666666},{"step":213,"nonzero":0.8541666666666666},{"step":215,"nonzero":1.0},{"step":217,"nonzero":0.9166666666666666},{"step":219,"nonzero":0.875},{"step":221,"nonzero":0.8958333333333334},{"step":223,"nonzero":0.7708333333333334},{"step":225,"nonzero":0.6041666666666666},{"step":227,"nonzero":0.5416666666666666},{"step":229,"nonzero":0.8958333333333334},{"step":231,"nonzero":1.0},{"step":233,"nonzero":1.0},{"step":235,"nonzero":0.9166666666666666},{"step":237,"nonzero":0.8333333333333334},{"step":239,"nonzero":0.8958333333333334},{"step":241,"nonzero":0.625},{"step":243,"nonzero":0.8541666666666666},{"step":245,"nonzero":0.6875},{"step":247,"nonzero":0.625},{"step":249,"nonzero":0.8958333333333334},{"step":251,"nonzero":0.9375},{"step":253,"nonzero":0.8333333333333334},{"step":255,"nonzero":0.8333333333333334},{"step":257,"nonzero":0.7708333333333334},{"step":259,"nonzero":0.6041666666666666},{"step":261,"nonzero":0.8125},{"step":263,"nonzero":0.6875},{"step":265,"nonzero":0.9166666666666666},{"step":267,"nonzero":0.8125},{"step":269,"nonzero":0.8333333333333334},{"step":271,"nonzero":0.8958333333333334},{"step":273,"nonzero":0.9375},{"step":275,"nonzero":0.5833333333333334},{"step":277,"nonzero":0.8541666666666666},{"step":279,"nonzero":0.9583333333333334},{"step":281,"nonzero":0.5416666666666666},{"step":283,"nonzero":0.9583333333333334},{"step":285,"nonzero":0.875},{"step":287,"nonzero":1.0},{"step":289,"nonzero":0.6666666666666666},{"step":291,"nonzero":0.9583333333333334},{"step":293,"nonzero":0.8333333333333334},{"step":295,"nonzero":0.8541666666666666},{"step":297,"nonzero":0.8333333333333334},{"step":299,"nonzero":1.0},{"step":301,"nonzero":0.9583333333333334},{"step":303,"nonzero":0.9166666666666666},{"step":305,"nonzero":0.6458333333333334},{"step":307,"nonzero":0.8541666666666666},{"step":309,"nonzero":0.9583333333333334},{"step":311,"nonzero":0.6041666666666666},{"step":313,"nonzero":0.9583333333333334},{"step":315,"nonzero":0.8125},{"step":317,"nonzero":0.9375},{"step":319,"nonzero":0.8541666666666666},{"step":321,"nonzero":0.9166666666666666},{"step":323,"nonzero":1.0},{"step":325,"nonzero":1.0},{"step":327,"nonzero":0.875},{"step":329,"nonzero":1.0},{"step":331,"nonzero":0.8125},{"step":333,"nonzero":0.9791666666666666},{"step":335,"nonzero":0.9166666666666666},{"step":337,"nonzero":0.8958333333333334},{"step":339,"nonzero":0.9791666666666666},{"step":341,"nonzero":0.9791666666666666},{"step":343,"nonzero":1.0},{"step":345,"nonzero":0.7708333333333334},{"step":347,"nonzero":0.9375},{"step":349,"nonzero":0.9583333333333334},{"step":351,"nonzero":0.9791666666666666},{"step":353,"nonzero":0.9583333333333334},{"step":355,"nonzero":0.9583333333333334},{"step":357,"nonzero":1.0},{"step":359,"nonzero":1.0},{"step":361,"nonzero":1.0},{"step":363,"nonzero":0.9791666666666666},{"step":365,"nonzero":0.9375},{"step":367,"nonzero":0.9791666666666666},{"step":369,"nonzero":1.0},{"step":371,"nonzero":0.8958333333333334},{"step":373,"nonzero":0.9791666666666666},{"step":375,"nonzero":0.9791666666666666},{"step":377,"nonzero":0.875},{"step":379,"nonzero":0.8541666666666666},{"step":381,"nonzero":0.875},{"step":383,"nonzero":0.9166666666666666},{"step":385,"nonzero":0.9583333333333334},{"step":387,"nonzero":0.9583333333333334},{"step":389,"nonzero":1.0},{"step":391,"nonzero":0.8125},{"step":393,"nonzero":1.0},{"step":395,"nonzero":0.8958333333333334},{"step":397,"nonzero":1.0},{"step":399,"nonzero":0.8958333333333334},{"step":801,"nonzero":0.875},{"step":803,"nonzero":0.8541666666666666},{"step":805,"nonzero":1.0},{"step":807,"nonzero":0.9375},{"step":809,"nonzero":0.875},{"step":811,"nonzero":0.8541666666666666},{"step":813,"nonzero":0.9166666666666666},{"step":815,"nonzero":0.875},{"step":817,"nonzero":0.9583333333333334},{"step":819,"nonzero":1.0},{"step":821,"nonzero":1.0},{"step":823,"nonzero":0.9375},{"step":825,"nonzero":0.9583333333333334},{"step":827,"nonzero":0.9791666666666666},{"step":829,"nonzero":1.0},{"step":831,"nonzero":0.9791666666666666},{"step":833,"nonzero":1.0},{"step":835,"nonzero":0.9166666666666666},{"step":837,"nonzero":1.0},{"step":839,"nonzero":0.9375},{"step":841,"nonzero":0.9375},{"step":843,"nonzero":0.875},{"step":845,"nonzero":0.9791666666666666},{"step":847,"nonzero":0.9791666666666666},{"step":849,"nonzero":0.9583333333333334},{"step":851,"nonzero":1.0},{"step":853,"nonzero":0.9583333333333334},{"step":855,"nonzero":0.9166666666666666},{"step":857,"nonzero":1.0},{"step":859,"nonzero":0.9583333333333334},{"step":861,"nonzero":1.0},{"step":863,"nonzero":1.0},{"step":865,"nonzero":1.0},{"step":867,"nonzero":1.0},{"step":869,"nonzero":0.9791666666666666},{"step":871,"nonzero":1.0},{"step":873,"nonzero":0.9583333333333334},{"step":875,"nonzero":1.0},{"step":877,"nonzero":0.9583333333333334},{"step":879,"nonzero":0.9166666666666666},{"step":881,"nonzero":1.0},{"step":883,"nonzero":1.0},{"step":885,"nonzero":1.0},{"step":887,"nonzero":1.0},{"step":889,"nonzero":0.9166666666666666},{"step":891,"nonzero":0.7291666666666666},{"step":893,"nonzero":1.0},{"step":895,"nonzero":0.9791666666666666},{"step":897,"nonzero":1.0},{"step":899,"nonzero":0.9791666666666666},{"step":901,"nonzero":0.9791666666666666},{"step":903,"nonzero":0.9375},{"step":905,"nonzero":1.0},{"step":907,"nonzero":0.9375},{"step":909,"nonzero":1.0},{"step":911,"nonzero":0.9166666666666666},{"step":913,"nonzero":0.9791666666666666},{"step":915,"nonzero":1.0},{"step":917,"nonzero":0.9791666666666666},{"step":919,"nonzero":0.9583333333333334},{"step":921,"nonzero":1.0},{"step":923,"nonzero":0.9791666666666666},{"step":925,"nonzero":0.9166666666666666},{"step":927,"nonzero":1.0},{"step":929,"nonzero":0.9375},{"step":931,"nonzero":0.9791666666666666},{"step":933,"nonzero":1.0},{"step":935,"nonzero":1.0},{"step":937,"nonzero":0.9791666666666666},{"step":939,"nonzero":0.6875},{"step":941,"nonzero":1.0},{"step":943,"nonzero":1.0},{"step":945,"nonzero":1.0},{"step":947,"nonzero":1.0},{"step":949,"nonzero":0.9791666666666666},{"step":951,"nonzero":0.9583333333333334},{"step":953,"nonzero":0.9583333333333334},{"step":955,"nonzero":1.0},{"step":957,"nonzero":0.9791666666666666},{"step":959,"nonzero":1.0},{"step":961,"nonzero":0.8541666666666666},{"step":963,"nonzero":1.0},{"step":965,"nonzero":0.875},{"step":967,"nonzero":1.0},{"step":969,"nonzero":0.7916666666666666},{"step":971,"nonzero":0.9583333333333334},{"step":973,"nonzero":1.0},{"step":975,"nonzero":1.0},{"step":977,"nonzero":1.0},{"step":979,"nonzero":1.0},{"step":981,"nonzero":0.9583333333333334},{"step":983,"nonzero":1.0},{"step":985,"nonzero":0.9791666666666666},{"step":987,"nonzero":1.0},{"step":989,"nonzero":1.0},{"step":991,"nonzero":1.0},{"step":993,"nonzero":1.0},{"step":995,"nonzero":0.875},{"step":997,"nonzero":1.0},{"step":999,"nonzero":1.0}]},{"id":"cispo-repor-lora-226","label":"CISPO + REPO-R \u00b7 LoRA \u00b7 226 prompts","run_ids":["1790958861"],"loss":"CISPO","adapter":"LoRA","prompts":226,"repo_r":true,"holdout_by_checkpoint":[{"step":0,"holdout":-1.3664,"train_core":-0.9836,"valid_rate":0.9},{"step":100,"holdout":-0.4693,"train_core":-0.3638,"valid_rate":0.94},{"step":200,"holdout":-0.1312,"train_core":-0.1526,"valid_rate":0.98},{"step":300,"holdout":0.8458,"train_core":0.2932,"valid_rate":0.96},{"step":400,"holdout":0.5059,"train_core":0.2334,"valid_rate":0.98},{"step":500,"holdout":1.7998,"train_core":0.8237,"valid_rate":0.98},{"step":600,"holdout":1.5811,"train_core":0.695,"valid_rate":0.96}],"selected_step":500,"selected_holdout":1.7998,"train_steps":[{"step":1,"reward":-2.2132349014282227,"core":-0.679081916809082,"cost":-0.09526743739843369,"kl":0.0,"entropy":0.5601700246334076},{"step":3,"reward":-2.651991605758667,"core":-0.8303236365318298,"cost":-0.1075056791305542,"kl":0.000877986938576214,"entropy":0.5284291952848434},{"step":5,"reward":-2.2209160327911377,"core":-0.685722827911377,"cost":-0.09401684999465942,"kl":0.002676972304470837,"entropy":0.5284935981035233},{"step":7,"reward":-1.082573413848877,"core":-0.2722651958465576,"cost":-0.07293131947517395,"kl":0.00576048472430557,"entropy":0.5059025809168816},{"step":9,"reward":-1.7322367429733276,"core":-0.5255213379859924,"cost":-0.09776618331670761,"kl":0.010216954629868269,"entropy":0.5406343936920166},{"step":11,"reward":-3.150214433670044,"core":-0.9241538047790527,"cost":-0.2538471221923828,"kl":0.008821151917800307,"entropy":0.4951176047325134},{"step":13,"reward":-2.100555658340454,"core":-0.6221248507499695,"cost":-0.08934862166643143,"kl":0.012151250150054693,"entropy":0.48960085213184357},{"step":15,"reward":-2.47225284576416,"core":-0.7393338084220886,"cost":-0.07809395343065262,"kl":0.014063709415495396,"entropy":0.43144576996564865},{"step":17,"reward":-1.494632363319397,"core":-0.4190390408039093,"cost":0.0,"kl":0.027266420889645815,"entropy":0.4812484160065651},{"step":19,"reward":-1.9282325506210327,"core":-0.6025228500366211,"cost":-0.022059163078665733,"kl":0.02908573253080249,"entropy":0.5276857241988182},{"step":21,"reward":-1.8202985525131226,"core":-0.4985164701938629,"cost":-0.08771783113479614,"kl":0.032192240469157696,"entropy":0.4797334372997284},{"step":23,"reward":-2.0989696979522705,"core":-0.60176682472229,"cost":-0.12481886148452759,"kl":0.036623493768274784,"entropy":0.4563339799642563},{"step":25,"reward":-2.2971408367156982,"core":-0.6628527045249939,"cost":-0.14028529822826385,"kl":0.037962920032441616,"entropy":0.44477681815624237},{"step":27,"reward":-2.3704373836517334,"core":-0.6596313118934631,"cost":-0.1576550453901291,"kl":0.048338381573557854,"entropy":0.5294971764087677},{"step":29,"reward":-0.8685585856437683,"core":-0.1947847604751587,"cost":0.0,"kl":0.05345088290050626,"entropy":0.5353463217616081},{"step":31,"reward":-1.111743450164795,"core":-0.2987564504146576,"cost":-0.03371471166610718,"kl":0.06647293828427792,"entropy":0.5878045409917831},{"step":33,"reward":-1.5128990411758423,"core":-0.41838538646698,"cost":-0.07124578207731247,"kl":0.081394555978477,"entropy":0.6086690723896027},{"step":35,"reward":-1.5181589126586914,"core":-0.41905727982521057,"cost":-0.06590593606233597,"kl":0.09229682385921478,"entropy":0.5771851092576981},{"step":37,"reward":-0.9355841279029846,"core":-0.21830956637859344,"cost":-0.07993439584970474,"kl":0.10188081860542297,"entropy":0.5972316116094589},{"step":39,"reward":-1.258222222328186,"core":-0.2998909056186676,"cost":-0.08383353799581528,"kl":0.20858146902173758,"entropy":0.4532816857099533},{"step":41,"reward":-1.665511965751648,"core":-0.49702873826026917,"cost":-0.03252260014414787,"kl":0.09210363402962685,"entropy":0.5984776616096497},{"step":43,"reward":-1.9306468963623047,"core":-0.5188463926315308,"cost":-0.06965558230876923,"kl":0.11597582511603832,"entropy":0.5969132483005524},{"step":45,"reward":-1.2490631341934204,"core":-0.31608498096466064,"cost":0.0,"kl":0.08723714202642441,"entropy":0.6251467913389206},{"step":47,"reward":-1.9569053649902344,"core":-0.5193578600883484,"cost":-0.1024060919880867,"kl":0.05984451062977314,"entropy":0.6342487782239914},{"step":49,"reward":-1.1758025884628296,"core":-0.3037512004375458,"cost":-0.02667079120874405,"kl":0.06922662630677223,"entropy":0.6292558163404465},{"step":51,"reward":-0.7593404650688171,"core":-0.09618548303842545,"cost":-0.014819636009633541,"kl":0.05342007055878639,"entropy":0.6122621148824692},{"step":53,"reward":-1.372861385345459,"core":-0.37529659271240234,"cost":-0.01841829903423786,"kl":0.04598941095173359,"entropy":0.6350187510251999},{"step":55,"reward":-1.440101981163025,"core":-0.39555513858795166,"cost":-0.0618637390434742,"kl":0.03998041432350874,"entropy":0.49868883937597275},{"step":57,"reward":-0.44853153824806213,"core":-0.010057453997433186,"cost":-0.008587838150560856,"kl":0.04390475805848837,"entropy":0.4957694262266159},{"step":59,"reward":-0.7408573627471924,"core":-0.08991595357656479,"cost":-0.011585407890379429,"kl":0.04950092639774084,"entropy":0.5553905665874481},{"step":61,"reward":-1.0484967231750488,"core":-0.3149763345718384,"cost":0.0,"kl":0.05062880273908377,"entropy":0.5508272796869278},{"step":63,"reward":-1.0004346370697021,"core":-0.2153133898973465,"cost":-0.0023290133103728294,"kl":0.05121444817632437,"entropy":0.5394125580787659},{"step":65,"reward":-1.2086296081542969,"core":-0.351114422082901,"cost":-8.402872481383383e-05,"kl":0.03282015956938267,"entropy":0.42035383731126785},{"step":67,"reward":-1.6276136636734009,"core":-0.49476000666618347,"cost":-0.08604922145605087,"kl":0.05375559628009796,"entropy":0.5028656497597694},{"step":69,"reward":-1.2806516885757446,"core":-0.32787859439849854,"cost":-0.01896098628640175,"kl":0.04933717381209135,"entropy":0.41799894720315933},{"step":71,"reward":-0.23132307827472687,"core":0.017817163839936256,"cost":-0.011358045041561127,"kl":0.05554546508938074,"entropy":0.4961027204990387},{"step":73,"reward":-1.2297393083572388,"core":-0.3778698146343231,"cost":-0.00024704282986931503,"kl":0.051631985232234,"entropy":0.4068265035748482},{"step":75,"reward":-0.4291762411594391,"core":-0.07018844038248062,"cost":0.0,"kl":0.058536434546113014,"entropy":0.45932281017303467},{"step":77,"reward":-0.3767496645450592,"core":-0.007466351147741079,"cost":-0.021504269912838936,"kl":0.09717222303152084,"entropy":0.5124116986989975},{"step":79,"reward":-1.0133084058761597,"core":-0.22197186946868896,"cost":-0.07155201584100723,"kl":0.07682080194354057,"entropy":0.4529610797762871},{"step":81,"reward":-0.9647064208984375,"core":-0.22686457633972168,"cost":-0.002175165107473731,"kl":0.08170433156192303,"entropy":0.5302670896053314},{"step":83,"reward":-0.3706502616405487,"core":0.008807790465652943,"cost":-0.017610855400562286,"kl":0.09026754088699818,"entropy":0.4168596789240837},{"step":85,"reward":-0.4325229823589325,"core":-0.05511520802974701,"cost":-0.007672406267374754,"kl":0.12773464433848858,"entropy":0.515613965690136},{"step":87,"reward":-0.8729957938194275,"core":-0.18909569084644318,"cost":-0.0011243565240874887,"kl":0.10913279838860035,"entropy":0.5262210816144943},{"step":89,"reward":-0.5399126410484314,"core":-0.11948877573013306,"cost":-0.042988140136003494,"kl":0.14593675918877125,"entropy":0.5005698576569557},{"step":91,"reward":-0.13257859647274017,"core":0.0727773904800415,"cost":-0.0038839152548462152,"kl":0.13579623401165009,"entropy":0.535770520567894},{"step":93,"reward":-0.343373566865921,"core":-0.004570330027490854,"cost":0.0,"kl":0.1327002290636301,"entropy":0.5119699388742447},{"step":95,"reward":-1.0471681356430054,"core":-0.2137368768453598,"cost":-0.0008642568136565387,"kl":0.1157450433820486,"entropy":0.498918816447258},{"step":97,"reward":-0.22075574100017548,"core":0.01069567073136568,"cost":0.0,"kl":0.20193348824977875,"entropy":0.6181138008832932},{"step":99,"reward":0.2878182828426361,"core":0.25823691487312317,"cost":0.0,"kl":0.24176449701189995,"entropy":0.6251750290393829},{"step":101,"reward":-1.1126905679702759,"core":-0.22871096432209015,"cost":-0.04734617471694946,"kl":0.2727530784904957,"entropy":0.61153344810009},{"step":103,"reward":-0.7207353711128235,"core":-0.0877605676651001,"cost":-0.010951611213386059,"kl":0.21365348994731903,"entropy":0.5756984353065491},{"step":105,"reward":-0.8360604643821716,"core":-0.16001681983470917,"cost":-0.011365472339093685,"kl":0.15263349376618862,"entropy":0.5862282514572144},{"step":107,"reward":0.09972977638244629,"core":0.29079750180244446,"cost":-0.015494300983846188,"kl":0.17790672183036804,"entropy":0.6082173436880112},{"step":109,"reward":-0.6320386528968811,"core":-0.08817049115896225,"cost":-0.028673427179455757,"kl":0.23267997801303864,"entropy":0.6525848954916},{"step":111,"reward":-0.6274225115776062,"core":-0.1200820580124855,"cost":-0.007646000478416681,"kl":0.14160111919045448,"entropy":0.5699350237846375},{"step":113,"reward":-0.896766185760498,"core":-0.17086993157863617,"cost":-0.034372735768556595,"kl":0.15655018016695976,"entropy":0.6386580020189285},{"step":115,"reward":-0.2933558225631714,"core":0.0394664891064167,"cost":-0.004269960802048445,"kl":0.1492944173514843,"entropy":0.6057516634464264},{"step":117,"reward":-0.5155442357063293,"core":-0.036913756281137466,"cost":-0.008010356687009335,"kl":0.17298727110028267,"entropy":0.630939394235611},{"step":119,"reward":-0.48908376693725586,"core":-0.03579341992735863,"cost":-0.0017907466972246766,"kl":0.177604328840971,"entropy":0.5811883062124252},{"step":121,"reward":0.3729040324687958,"core":0.29970285296440125,"cost":-0.009103789925575256,"kl":0.1580950990319252,"entropy":0.5446539968252182},{"step":123,"reward":-0.7635752558708191,"core":-0.08538109064102173,"cost":-0.020444029942154884,"kl":0.13001962192356586,"entropy":0.5192401856184006},{"step":125,"reward":-1.0808032751083374,"core":-0.23969262838363647,"cost":-0.026528678834438324,"kl":0.15190806426107883,"entropy":0.584165170788765},{"step":127,"reward":-1.2804604768753052,"core":-0.3440885543823242,"cost":-0.03506237640976906,"kl":0.1704561412334442,"entropy":0.615480899810791},{"step":129,"reward":-1.041361927986145,"core":-0.236747145652771,"cost":-0.010116849094629288,"kl":0.1319569107145071,"entropy":0.5646819323301315},{"step":131,"reward":-0.2874823808670044,"core":-0.02364584058523178,"cost":0.0,"kl":0.11713072471320629,"entropy":0.5573883503675461},{"step":133,"reward":-0.2315165400505066,"core":0.040857989341020584,"cost":0.0,"kl":0.16210172325372696,"entropy":0.6027518510818481},{"step":135,"reward":-0.3770667314529419,"core":-0.07843692600727081,"cost":-0.012927952222526073,"kl":0.16893135756254196,"entropy":0.5177633315324783},{"step":137,"reward":0.07543974369764328,"core":0.18190550804138184,"cost":0.0,"kl":0.1286751590669155,"entropy":0.5378055945038795},{"step":139,"reward":-0.7401013374328613,"core":-0.1300404816865921,"cost":-0.00338190421462059,"kl":0.15368730574846268,"entropy":0.5397985205054283},{"step":141,"reward":-0.563236653804779,"core":-0.05075843632221222,"cost":0.0,"kl":0.11723039671778679,"entropy":0.5240459516644478},{"step":143,"reward":-0.5703895092010498,"core":-0.034511346369981766,"cost":-0.013255558907985687,"kl":0.2030448019504547,"entropy":0.5818204879760742},{"step":145,"reward":0.014817667193710804,"core":0.15983428061008453,"cost":-0.018312782049179077,"kl":0.1327928975224495,"entropy":0.39023999869823456},{"step":147,"reward":-0.3594932556152344,"core":0.036689046770334244,"cost":0.0,"kl":0.1599838174879551,"entropy":0.4753744453191757},{"step":149,"reward":-0.015816211700439453,"core":0.11857045441865921,"cost":-0.0027880140114575624,"kl":0.18566845171153545,"entropy":0.519988052546978},{"step":151,"reward":-0.8875608444213867,"core":-0.1616683155298233,"cost":-0.034135665744543076,"kl":0.20966482907533646,"entropy":0.5961852073669434},{"step":153,"reward":-0.745802104473114,"core":-0.11581011861562729,"cost":-0.046045467257499695,"kl":0.16366425529122353,"entropy":0.4744064658880234},{"step":155,"reward":-1.8679962158203125,"core":-0.5294145345687866,"cost":-0.13365177810192108,"kl":0.18131396546959877,"entropy":0.5834118574857712},{"step":157,"reward":0.9047828316688538,"core":0.44103190302848816,"cost":-0.00030592139228247106,"kl":0.14582019858062267,"entropy":0.5216106921434402},{"step":159,"reward":-0.7717280387878418,"core":-0.11411857604980469,"cost":-0.010122179053723812,"kl":0.19972678273916245,"entropy":0.5814648866653442},{"step":161,"reward":0.008022497408092022,"core":0.18773655593395233,"cost":-0.03198627755045891,"kl":0.16685566678643227,"entropy":0.5412092581391335},{"step":163,"reward":-0.20890109241008759,"core":0.13618023693561554,"cost":0.0,"kl":0.17999640107154846,"entropy":0.5611605197191238},{"step":165,"reward":-1.1536073684692383,"core":-0.31380245089530945,"cost":-0.020023921504616737,"kl":0.14525590278208256,"entropy":0.5487293377518654},{"step":167,"reward":-0.07774296402931213,"core":0.22201363742351532,"cost":-0.013926784507930279,"kl":0.1499507687985897,"entropy":0.5779197812080383},{"step":169,"reward":0.21911269426345825,"core":0.23224084079265594,"cost":0.0,"kl":0.1711418479681015,"entropy":0.6784240305423737},{"step":171,"reward":0.11723405867815018,"core":0.22495697438716888,"cost":-0.03094467520713806,"kl":0.14937415346503258,"entropy":0.5941091775894165},{"step":173,"reward":-0.43277251720428467,"core":-0.022376323118805885,"cost":0.0,"kl":0.14877190068364143,"entropy":0.638477236032486},{"step":175,"reward":0.23324960470199585,"core":0.31135663390159607,"cost":-0.012430943548679352,"kl":0.16374222934246063,"entropy":0.6242499798536301},{"step":177,"reward":-0.6358127593994141,"core":-0.11077988147735596,"cost":-0.013645052909851074,"kl":0.1533352229744196,"entropy":0.5373736768960953},{"step":179,"reward":0.38193702697753906,"core":0.3304641544818878,"cost":-0.014566448517143726,"kl":0.15046751871705055,"entropy":0.5346425473690033},{"step":181,"reward":-0.23263873159885406,"core":0.050084877759218216,"cost":-0.033881884068250656,"kl":0.14578574523329735,"entropy":0.5511398315429688},{"step":183,"reward":-0.20689420402050018,"core":0.053474556654691696,"cost":-0.010062192566692829,"kl":0.21429333835840225,"entropy":0.5900607854127884},{"step":185,"reward":0.07455674558877945,"core":0.15693894028663635,"cost":-0.040334656834602356,"kl":0.32261304184794426,"entropy":0.5643311142921448},{"step":187,"reward":0.0798698291182518,"core":0.16745011508464813,"cost":0.0,"kl":0.20467738807201385,"entropy":0.6377539336681366},{"step":189,"reward":-0.2598048448562622,"core":0.007804118096828461,"cost":0.0,"kl":0.15786205604672432,"entropy":0.5530246496200562},{"step":191,"reward":-0.6697422862052917,"core":-0.05770805478096008,"cost":-0.07071572542190552,"kl":0.1872054971754551,"entropy":0.5235223174095154},{"step":193,"reward":-0.12084636837244034,"core":0.1494220346212387,"cost":-0.03191519156098366,"kl":0.1641787439584732,"entropy":0.5827356278896332},{"step":195,"reward":-0.8818106651306152,"core":-0.17746025323867798,"cost":-0.04655729606747627,"kl":0.1400948353111744,"entropy":0.5499733090400696},{"step":197,"reward":-0.4729839861392975,"core":-0.0319502018392086,"cost":-0.03534404933452606,"kl":0.17974135093390942,"entropy":0.5658315569162369},{"step":199,"reward":-0.643725574016571,"core":-0.08540601283311844,"cost":-0.03939161077141762,"kl":0.2002442218363285,"entropy":0.6076435148715973},{"step":201,"reward":0.31553196907043457,"core":0.3021102845668793,"cost":-0.02760319411754608,"kl":0.16270200163125992,"entropy":0.5822337716817856},{"step":203,"reward":0.3349718153476715,"core":0.29322266578674316,"cost":-0.006098540499806404,"kl":0.16099446266889572,"entropy":0.5092594102025032},{"step":205,"reward":0.6711669564247131,"core":0.4432869255542755,"cost":-0.0024477532133460045,"kl":0.17636215314269066,"entropy":0.5749356374144554},{"step":207,"reward":0.4579874277114868,"core":0.3364435136318207,"cost":0.0,"kl":0.37096865475177765,"entropy":0.5322209000587463},{"step":209,"reward":-0.2644093930721283,"core":0.07000089436769485,"cost":-0.014325686730444431,"kl":0.18041279911994934,"entropy":0.5858588665723801},{"step":211,"reward":0.09876084327697754,"core":0.21176277101039886,"cost":-0.03584536164999008,"kl":0.2865205146372318,"entropy":0.5323510393500328},{"step":213,"reward":-0.1291763335466385,"core":0.06337972730398178,"cost":0.0,"kl":0.21097415685653687,"entropy":0.5898418724536896},{"step":215,"reward":0.5930097103118896,"core":0.46480491757392883,"cost":-0.011741656810045242,"kl":0.20476581901311874,"entropy":0.5694116801023483},{"step":217,"reward":0.42253056168556213,"core":0.21310825645923615,"cost":0.0,"kl":0.18523263558745384,"entropy":0.42149700969457626},{"step":219,"reward":0.8435260653495789,"core":0.4293994903564453,"cost":0.0,"kl":0.25604458898305893,"entropy":0.5106409713625908},{"step":221,"reward":0.1702703833580017,"core":0.21463613212108612,"cost":-0.05956432223320007,"kl":0.25482144951820374,"entropy":0.5080586299300194},{"step":223,"reward":0.14587010443210602,"core":0.15623950958251953,"cost":0.0,"kl":0.18304075673222542,"entropy":0.5674522072076797},{"step":225,"reward":0.1730457991361618,"core":0.3544941246509552,"cost":-0.051045045256614685,"kl":0.23113088309764862,"entropy":0.5325855910778046},{"step":227,"reward":-0.6930844187736511,"core":-0.14187279343605042,"cost":0.0,"kl":0.2478559985756874,"entropy":0.587635800242424},{"step":229,"reward":0.026880979537963867,"core":0.1718316525220871,"cost":-0.06317678093910217,"kl":0.30854491144418716,"entropy":0.5034252032637596},{"step":231,"reward":-0.018188314512372017,"core":0.15658286213874817,"cost":-0.003016583388671279,"kl":0.28762567415833473,"entropy":0.5815366804599762},{"step":233,"reward":0.6676791310310364,"core":0.4743565618991852,"cost":-0.0035010147839784622,"kl":0.27533090114593506,"entropy":0.6044594347476959},{"step":235,"reward":0.11914724111557007,"core":0.2263495922088623,"cost":0.0,"kl":0.18242793530225754,"entropy":0.6055769324302673},{"step":237,"reward":0.982181966304779,"core":0.529179573059082,"cost":-0.0004855812294408679,"kl":0.2940586470067501,"entropy":0.6472502946853638},{"step":239,"reward":0.10279376059770584,"core":0.21599926054477692,"cost":0.0,"kl":0.30676424503326416,"entropy":0.5555723756551743},{"step":241,"reward":0.10347182303667068,"core":0.15209829807281494,"cost":-0.08369338512420654,"kl":0.2824360281229019,"entropy":0.5676486939191818},{"step":243,"reward":0.5903971195220947,"core":0.3861675262451172,"cost":0.0,"kl":0.26014574617147446,"entropy":0.5664304047822952},{"step":245,"reward":0.45599398016929626,"core":0.40743574500083923,"cost":0.0,"kl":0.28181250020861626,"entropy":0.5283467918634415},{"step":247,"reward":0.03619004413485527,"core":0.16703760623931885,"cost":-0.012764501385390759,"kl":0.25581158325076103,"entropy":0.5459482297301292},{"step":249,"reward":0.3012365996837616,"core":0.2761387825012207,"cost":0.0,"kl":0.32037753611803055,"entropy":0.5673457086086273},{"step":251,"reward":0.5138333439826965,"core":0.4780248701572418,"cost":-0.030811183154582977,"kl":0.3434208109974861,"entropy":0.5625409334897995},{"step":253,"reward":0.601513683795929,"core":0.38161250948905945,"cost":-0.0019632449839264154,"kl":0.3158736526966095,"entropy":0.5190648883581161},{"step":255,"reward":1.1771355867385864,"core":0.6538322567939758,"cost":-0.0034403891768306494,"kl":0.26049551740288734,"entropy":0.4994467943906784},{"step":257,"reward":0.5943998098373413,"core":0.4151158332824707,"cost":-0.007150033488869667,"kl":0.2719713784754276,"entropy":0.5259546041488647},{"step":259,"reward":0.7252482771873474,"core":0.5514935851097107,"cost":0.0,"kl":0.2912783771753311,"entropy":0.5068199411034584},{"step":261,"reward":0.33295193314552307,"core":0.30554890632629395,"cost":-0.00800428818911314,"kl":0.2760167866945267,"entropy":0.533225804567337},{"step":263,"reward":0.0338062047958374,"core":0.15678225457668304,"cost":0.0,"kl":0.2962687686085701,"entropy":0.5174476876854897},{"step":265,"reward":-0.2757231891155243,"core":0.01636497676372528,"cost":0.0,"kl":0.2675178237259388,"entropy":0.5368423312902451},{"step":267,"reward":0.17248980700969696,"core":0.25155478715896606,"cost":0.0,"kl":0.25371890887618065,"entropy":0.5151832103729248},{"step":269,"reward":0.6609408855438232,"core":0.45682117342948914,"cost":-0.022930145263671875,"kl":0.32757560908794403,"entropy":0.5153941363096237},{"step":271,"reward":0.06065742298960686,"core":0.16349446773529053,"cost":0.0,"kl":0.3384737968444824,"entropy":0.527038112282753},{"step":273,"reward":1.2024409770965576,"core":0.7058047652244568,"cost":0.0,"kl":0.25902988016605377,"entropy":0.5156683921813965},{"step":275,"reward":0.32747822999954224,"core":0.23420919477939606,"cost":0.0,"kl":0.29419751465320587,"entropy":0.5896673202514648},{"step":277,"reward":0.7091159820556641,"core":0.48417234420776367,"cost":0.0,"kl":0.3728322461247444,"entropy":0.5433204174041748},{"step":279,"reward":0.46712586283683777,"core":0.33331021666526794,"cost":-0.012858513742685318,"kl":0.2698163017630577,"entropy":0.5503263771533966},{"step":281,"reward":0.1262006312608719,"core":0.23633445799350739,"cost":0.0,"kl":0.30445531755685806,"entropy":0.5542689114809036},{"step":283,"reward":0.5618101358413696,"core":0.4135787785053253,"cost":0.0,"kl":0.22032862156629562,"entropy":0.5115035101771355},{"step":285,"reward":0.5760106444358826,"core":0.3979225158691406,"cost":0.0,"kl":0.2707880027592182,"entropy":0.537921279668808},{"step":287,"reward":0.9134693145751953,"core":0.5976476073265076,"cost":0.0,"kl":0.2702101767063141,"entropy":0.5564342737197876},{"step":289,"reward":0.4101901054382324,"core":0.3603390157222748,"cost":0.0,"kl":0.263979934155941,"entropy":0.613148108124733},{"step":291,"reward":0.3675696551799774,"core":0.276022344827652,"cost":-0.007564526051282883,"kl":0.2090890258550644,"entropy":0.5411119759082794},{"step":293,"reward":0.8685879111289978,"core":0.5691397786140442,"cost":0.0,"kl":0.28793909400701523,"entropy":0.5808069258928299},{"step":295,"reward":0.6787655353546143,"core":0.41562214493751526,"cost":-0.02926371991634369,"kl":0.20535285398364067,"entropy":0.5736171454191208},{"step":297,"reward":0.5154147744178772,"core":0.3068673014640808,"cost":-0.018694931641221046,"kl":0.327840905636549,"entropy":0.5238810628652573},{"step":299,"reward":0.4446147382259369,"core":0.32113298773765564,"cost":0.0,"kl":0.27998754009604454,"entropy":0.5881013721227646},{"step":301,"reward":-0.21984344720840454,"core":0.07311442494392395,"cost":-0.010345007292926311,"kl":0.25297892838716507,"entropy":0.5259525328874588},{"step":303,"reward":0.6700829863548279,"core":0.40394875407218933,"cost":0.0,"kl":0.2702598161995411,"entropy":0.58168825507164},{"step":305,"reward":0.5624446868896484,"core":0.41617751121520996,"cost":-0.0029587631579488516,"kl":0.18680549785494804,"entropy":0.6005099415779114},{"step":307,"reward":0.7075405120849609,"core":0.4125352203845978,"cost":-0.002296139718964696,"kl":0.26088444143533707,"entropy":0.5708949863910675},{"step":309,"reward":0.8785898685455322,"core":0.4526127278804779,"cost":0.0,"kl":0.1646236926317215,"entropy":0.49538570642471313},{"step":311,"reward":0.21757350862026215,"core":0.28038325905799866,"cost":0.0,"kl":0.207937840372324,"entropy":0.5838397145271301},{"step":313,"reward":0.23517046868801117,"core":0.2961859405040741,"cost":-0.006987929809838533,"kl":0.18577676638960838,"entropy":0.6172129809856415},{"step":315,"reward":1.4065345525741577,"core":0.8295633792877197,"cost":0.0,"kl":0.2210814207792282,"entropy":0.527135893702507},{"step":317,"reward":0.8004762530326843,"core":0.482133150100708,"cost":0.0,"kl":0.2542266808450222,"entropy":0.6031005084514618},{"step":319,"reward":1.200169324874878,"core":0.6991567611694336,"cost":0.0,"kl":0.2125820703804493,"entropy":0.47683360427618027},{"step":321,"reward":0.4925421178340912,"core":0.3987671434879303,"cost":-0.009558567777276039,"kl":0.20227834954857826,"entropy":0.5356839671730995},{"step":323,"reward":0.5578925013542175,"core":0.4009224474430084,"cost":0.0,"kl":0.24955973774194717,"entropy":0.4970272406935692},{"step":325,"reward":0.4913523197174072,"core":0.3361891210079193,"cost":0.0,"kl":0.2772491052746773,"entropy":0.6317533999681473},{"step":327,"reward":0.005257656332105398,"core":0.1645270586013794,"cost":-0.017547359690070152,"kl":0.22399026528000832,"entropy":0.5970292538404465},{"step":329,"reward":-0.45982876420021057,"core":0.030109921470284462,"cost":0.0,"kl":0.22719934955239296,"entropy":0.6619179993867874},{"step":331,"reward":-0.2989598214626312,"core":0.031019777059555054,"cost":0.0,"kl":0.2446439117193222,"entropy":0.6096166223287582},{"step":333,"reward":-0.1467542052268982,"core":0.08920076489448547,"cost":-0.003844495164230466,"kl":0.19431012496352196,"entropy":0.5900186747312546},{"step":335,"reward":0.6420760154724121,"core":0.38723698258399963,"cost":-0.049105871468782425,"kl":0.2142699509859085,"entropy":0.6036180704832077},{"step":337,"reward":0.666038990020752,"core":0.3250443637371063,"cost":-0.0020390504505485296,"kl":0.269842516630888,"entropy":0.6107093542814255},{"step":339,"reward":0.22514621913433075,"core":0.336847186088562,"cost":-0.04449935629963875,"kl":0.2905483916401863,"entropy":0.6339275985956192},{"step":341,"reward":0.37865567207336426,"core":0.3042537271976471,"cost":-0.0003151318524032831,"kl":0.21590107306838036,"entropy":0.551063060760498},{"step":343,"reward":0.7822684645652771,"core":0.4706987142562866,"cost":0.0,"kl":0.2952801510691643,"entropy":0.5232356935739517},{"step":345,"reward":-0.07808814197778702,"core":0.11460600048303604,"cost":0.0,"kl":0.2570851296186447,"entropy":0.6028427332639694},{"step":347,"reward":1.197932243347168,"core":0.5784818530082703,"cost":-0.0013479989720508456,"kl":0.228873148560524,"entropy":0.43969473242759705},{"step":349,"reward":0.24236313998699188,"core":0.24813981354236603,"cost":-0.009751892648637295,"kl":0.19405434280633926,"entropy":0.5549872368574142},{"step":351,"reward":-0.3729664385318756,"core":0.01619189791381359,"cost":-0.017079411074519157,"kl":0.25680486112833023,"entropy":0.5867912918329239},{"step":353,"reward":0.6669611930847168,"core":0.4475545883178711,"cost":-0.030846016481518745,"kl":0.2600296549499035,"entropy":0.5423224419355392},{"step":355,"reward":-0.15121518075466156,"core":0.14708204567432404,"cost":-0.017932942137122154,"kl":0.2449476420879364,"entropy":0.5729205459356308},{"step":357,"reward":1.2524505853652954,"core":0.7628667950630188,"cost":0.0,"kl":0.18277579545974731,"entropy":0.5302053615450859},{"step":359,"reward":0.7920017242431641,"core":0.5326070189476013,"cost":0.0,"kl":0.3587324321269989,"entropy":0.49192607402801514},{"step":361,"reward":0.20770235359668732,"core":0.30898216366767883,"cost":-0.04056030884385109,"kl":0.24505644664168358,"entropy":0.5153012424707413},{"step":363,"reward":-0.04944467544555664,"core":0.16324211657047272,"cost":-0.030905934050679207,"kl":0.23802043497562408,"entropy":0.6110781133174896},{"step":365,"reward":-0.31593775749206543,"core":0.07948725670576096,"cost":0.0,"kl":0.24360066652297974,"entropy":0.5056907385587692},{"step":367,"reward":0.8419855237007141,"core":0.5439683794975281,"cost":0.0,"kl":0.21543779224157333,"entropy":0.4515073224902153},{"step":369,"reward":-0.15078504383563995,"core":0.12058625370264053,"cost":-0.01724305935204029,"kl":0.3167257755994797,"entropy":0.5425917506217957},{"step":371,"reward":0.5674086213111877,"core":0.3758237361907959,"cost":0.0,"kl":0.22121582552790642,"entropy":0.4521337077021599},{"step":373,"reward":0.29350998997688293,"core":0.3574574887752533,"cost":0.0,"kl":0.21910613030195236,"entropy":0.5396137982606888},{"step":375,"reward":1.0720843076705933,"core":0.593881368637085,"cost":0.0,"kl":0.19886229187250137,"entropy":0.5511983782052994},{"step":377,"reward":0.6565826535224915,"core":0.34921208024024963,"cost":0.0,"kl":0.21315200254321098,"entropy":0.48550812900066376},{"step":379,"reward":-0.22700238227844238,"core":0.08044124394655228,"cost":-0.0018493466777727008,"kl":0.17763419821858406,"entropy":0.5117730274796486},{"step":381,"reward":-0.7394043803215027,"core":-0.04740242287516594,"cost":-0.07183463126420975,"kl":0.21947981044650078,"entropy":0.5510755926370621},{"step":383,"reward":0.6350552439689636,"core":0.44576844573020935,"cost":0.0,"kl":0.29432401433587074,"entropy":0.48964788764715195},{"step":385,"reward":0.6679603457450867,"core":0.4764518737792969,"cost":-0.0202545877546072,"kl":0.20633643493056297,"entropy":0.5983201414346695},{"step":387,"reward":0.5183864235877991,"core":0.38976600766181946,"cost":-0.009267736226320267,"kl":0.18156332150101662,"entropy":0.5361372530460358},{"step":389,"reward":0.4588141441345215,"core":0.4146065413951874,"cost":-0.030924828723073006,"kl":0.18759816139936447,"entropy":0.5568763539195061},{"step":391,"reward":0.6136431694030762,"core":0.4690634310245514,"cost":-0.010167471133172512,"kl":0.2037830427289009,"entropy":0.48185601085424423},{"step":393,"reward":0.5037370324134827,"core":0.3039053976535797,"cost":-0.013561424799263477,"kl":0.31824741512537,"entropy":0.5434115082025528},{"step":395,"reward":0.4684489667415619,"core":0.3418770730495453,"cost":0.0,"kl":0.25987301021814346,"entropy":0.5733502954244614},{"step":397,"reward":0.37922239303588867,"core":0.2629453241825104,"cost":0.0,"kl":0.33137623965740204,"entropy":0.5079429000616074},{"step":399,"reward":0.40337374806404114,"core":0.31750261783599854,"cost":-0.003782127285376191,"kl":0.21322324872016907,"entropy":0.5018052160739899},{"step":401,"reward":0.6913831233978271,"core":0.4210502803325653,"cost":0.0,"kl":0.24797778949141502,"entropy":0.5555500388145447},{"step":403,"reward":0.754105269908905,"core":0.46993863582611084,"cost":0.0,"kl":0.19331690669059753,"entropy":0.5238730385899544},{"step":405,"reward":0.5732414126396179,"core":0.37466928362846375,"cost":-0.00013368506915867329,"kl":0.28648386895656586,"entropy":0.5225890055298805},{"step":407,"reward":1.2525391578674316,"core":0.5870835781097412,"cost":0.0,"kl":0.24108342826366425,"entropy":0.574554979801178},{"step":409,"reward":0.155109241604805,"core":0.2508426606655121,"cost":-0.06712927669286728,"kl":0.28070634603500366,"entropy":0.5106920227408409},{"step":411,"reward":0.3422412872314453,"core":0.3356908857822418,"cost":-0.036562636494636536,"kl":0.2466820441186428,"entropy":0.5671113133430481},{"step":413,"reward":0.13385869562625885,"core":0.25184354186058044,"cost":-0.008169078268110752,"kl":0.2205098159611225,"entropy":0.5807741284370422},{"step":415,"reward":0.3782869577407837,"core":0.35465261340141296,"cost":-0.017597466707229614,"kl":0.2518395334482193,"entropy":0.5739404410123825},{"step":417,"reward":0.2046395093202591,"core":0.2353896051645279,"cost":0.0,"kl":0.2726030647754669,"entropy":0.5437920019030571},{"step":419,"reward":0.3965022563934326,"core":0.3444523811340332,"cost":-0.016020985320210457,"kl":0.25019747391343117,"entropy":0.5533275976777077},{"step":421,"reward":0.6172583699226379,"core":0.41980651021003723,"cost":-0.014099232852458954,"kl":0.2440262921154499,"entropy":0.5366984456777573},{"step":423,"reward":-0.022166088223457336,"core":0.13343115150928497,"cost":0.0,"kl":0.235329058021307,"entropy":0.5873418748378754},{"step":425,"reward":0.9115619659423828,"core":0.5239011645317078,"cost":0.0,"kl":0.29545164853334427,"entropy":0.5471419841051102},{"step":427,"reward":0.32148441672325134,"core":0.29979994893074036,"cost":-0.03565412387251854,"kl":0.42306315898895264,"entropy":0.5775783658027649},{"step":429,"reward":0.7847976684570312,"core":0.5764120221138,"cost":0.0,"kl":0.17928943037986755,"entropy":0.5548174530267715},{"step":431,"reward":0.576384425163269,"core":0.3846796452999115,"cost":-0.02598298154771328,"kl":0.334916889667511,"entropy":0.5273719429969788},{"step":433,"reward":0.2117820829153061,"core":0.2710421085357666,"cost":0.0,"kl":0.1950751468539238,"entropy":0.5861028432846069},{"step":435,"reward":0.4590221345424652,"core":0.381420761346817,"cost":-0.04512688145041466,"kl":0.2055995836853981,"entropy":0.5675455629825592},{"step":437,"reward":0.7806269526481628,"core":0.5406906604766846,"cost":0.0,"kl":0.29106054455041885,"entropy":0.528440922498703},{"step":439,"reward":0.5757730007171631,"core":0.37567901611328125,"cost":0.0,"kl":0.19247863441705704,"entropy":0.598885178565979},{"step":441,"reward":0.72871994972229,"core":0.4742799699306488,"cost":0.0,"kl":0.2721191979944706,"entropy":0.5727026537060738},{"step":443,"reward":-0.11713867634534836,"core":0.1440836638212204,"cost":0.0,"kl":0.2467510774731636,"entropy":0.575989380478859},{"step":445,"reward":0.8061018586158752,"core":0.5253039002418518,"cost":0.0,"kl":0.23178375884890556,"entropy":0.5117111280560493},{"step":447,"reward":0.41720032691955566,"core":0.334881067276001,"cost":0.0,"kl":0.2388050425797701,"entropy":0.5057590827345848},{"step":449,"reward":0.8810203075408936,"core":0.59156733751297,"cost":-0.004372799769043922,"kl":0.2665192149579525,"entropy":0.5211716890335083},{"step":451,"reward":0.23136277496814728,"core":0.3189904987812042,"cost":-0.01315869390964508,"kl":0.2461807318031788,"entropy":0.6176482886075974},{"step":453,"reward":0.5252587795257568,"core":0.3207680583000183,"cost":0.0,"kl":0.22847681492567062,"entropy":0.4623083174228668},{"step":455,"reward":0.8144609332084656,"core":0.5542801022529602,"cost":0.0,"kl":0.25823459029197693,"entropy":0.5843357294797897},{"step":457,"reward":0.9216163754463196,"core":0.6198205351829529,"cost":0.0,"kl":0.16129177808761597,"entropy":0.5336208939552307},{"step":459,"reward":0.8332862257957458,"core":0.5215473771095276,"cost":0.0,"kl":0.3005652502179146,"entropy":0.504863329231739},{"step":461,"reward":0.2665257453918457,"core":0.2661248743534088,"cost":-0.016879556700587273,"kl":0.2877589985728264,"entropy":0.6174903512001038},{"step":463,"reward":0.6610097289085388,"core":0.4532977044582367,"cost":0.0,"kl":0.2906203158199787,"entropy":0.5366761535406113},{"step":465,"reward":0.17414458096027374,"core":0.23150180280208588,"cost":-0.01415338646620512,"kl":0.2511695772409439,"entropy":0.5913380086421967},{"step":467,"reward":0.4862648546695709,"core":0.3351771831512451,"cost":0.0,"kl":0.30822814255952835,"entropy":0.550058588385582},{"step":469,"reward":1.3409438133239746,"core":0.6597594618797302,"cost":0.0,"kl":0.26987506076693535,"entropy":0.5510857701301575},{"step":471,"reward":0.4636053740978241,"core":0.319540798664093,"cost":0.0,"kl":0.2633652649819851,"entropy":0.5491151958703995},{"step":473,"reward":0.5291589498519897,"core":0.35995590686798096,"cost":-0.011920347809791565,"kl":0.2861531116068363,"entropy":0.5480358004570007},{"step":475,"reward":1.1159356832504272,"core":0.5432566404342651,"cost":-0.007055232767015696,"kl":0.3182317838072777,"entropy":0.5372194647789001},{"step":477,"reward":0.544910728931427,"core":0.38970518112182617,"cost":0.0,"kl":0.38788656145334244,"entropy":0.5631779432296753},{"step":479,"reward":1.141594409942627,"core":0.7199923396110535,"cost":0.0,"kl":0.2840418554842472,"entropy":0.5581151396036148},{"step":481,"reward":0.7852954864501953,"core":0.4997609555721283,"cost":0.0,"kl":0.2800350710749626,"entropy":0.5528707951307297},{"step":483,"reward":0.9661481976509094,"core":0.5845404863357544,"cost":0.0,"kl":0.27745221555233,"entropy":0.5484039634466171},{"step":485,"reward":0.677013635635376,"core":0.5379976630210876,"cost":0.0,"kl":0.26469736546278,"entropy":0.546140268445015},{"step":487,"reward":1.0895229578018188,"core":0.5326330661773682,"cost":0.0,"kl":0.3749806582927704,"entropy":0.6226987838745117},{"step":489,"reward":1.1204224824905396,"core":0.6223947405815125,"cost":0.0,"kl":0.3344854637980461,"entropy":0.5386370345950127},{"step":491,"reward":1.0568314790725708,"core":0.668725311756134,"cost":0.0,"kl":0.34150705486536026,"entropy":0.571413442492485},{"step":493,"reward":0.41006168723106384,"core":0.3003130853176117,"cost":-0.01705656200647354,"kl":0.34531236439943314,"entropy":0.5034100711345673},{"step":495,"reward":1.1474093198776245,"core":0.6268738508224487,"cost":-0.004966370295733213,"kl":0.3519170209765434,"entropy":0.5841361433267593},{"step":497,"reward":1.1975661516189575,"core":0.6400914788246155,"cost":0.0,"kl":0.3314337879419327,"entropy":0.523021474480629},{"step":499,"reward":1.1867423057556152,"core":0.6645742654800415,"cost":0.0,"kl":0.3481346368789673,"entropy":0.5849275439977646},{"step":501,"reward":1.0072458982467651,"core":0.6081418395042419,"cost":0.0,"kl":0.3504241928458214,"entropy":0.5718592554330826},{"step":503,"reward":0.886202871799469,"core":0.6166654229164124,"cost":-0.010395440272986889,"kl":0.26089827343821526,"entropy":0.5157693773508072},{"step":505,"reward":1.0091110467910767,"core":0.6298577189445496,"cost":-0.03401611000299454,"kl":0.2310253530740738,"entropy":0.5073865577578545},{"step":507,"reward":1.0259088277816772,"core":0.5401968955993652,"cost":-0.013256861828267574,"kl":0.32305822521448135,"entropy":0.5202466398477554},{"step":509,"reward":0.9597101211547852,"core":0.5716995596885681,"cost":0.0,"kl":0.3369663804769516,"entropy":0.5604574680328369},{"step":511,"reward":1.2003977298736572,"core":0.5827309489250183,"cost":0.0,"kl":0.33705103397369385,"entropy":0.5463020354509354},{"step":513,"reward":0.9828858375549316,"core":0.5751627683639526,"cost":-0.01925189048051834,"kl":0.2686684876680374,"entropy":0.4780203625559807},{"step":515,"reward":0.47928810119628906,"core":0.3530787229537964,"cost":0.0,"kl":0.3086608275771141,"entropy":0.5761120021343231},{"step":517,"reward":0.8716025352478027,"core":0.5343984961509705,"cost":-0.003146059811115265,"kl":0.3094078078866005,"entropy":0.4916349723935127},{"step":519,"reward":1.0101838111877441,"core":0.6250792145729065,"cost":-0.013500337488949299,"kl":0.28317247703671455,"entropy":0.5688176155090332},{"step":521,"reward":1.0196809768676758,"core":0.6037176251411438,"cost":-0.012223680503666401,"kl":0.27828652039170265,"entropy":0.5405161306262016},{"step":523,"reward":1.3789480924606323,"core":0.8065906167030334,"cost":0.0,"kl":0.32577909529209137,"entropy":0.5560568571090698},{"step":525,"reward":1.1433073282241821,"core":0.6492049098014832,"cost":0.0,"kl":0.3358641639351845,"entropy":0.53647530823946},{"step":527,"reward":1.2934964895248413,"core":0.8178810477256775,"cost":0.0,"kl":0.1947260946035385,"entropy":0.5402382910251617},{"step":529,"reward":1.5331573486328125,"core":0.795635461807251,"cost":0.0,"kl":0.3564757853746414,"entropy":0.4336242079734802},{"step":531,"reward":0.9201191067695618,"core":0.5915940403938293,"cost":0.0,"kl":0.39398549497127533,"entropy":0.5321591272950172},{"step":533,"reward":1.0251110792160034,"core":0.5254497528076172,"cost":0.0,"kl":0.3739275708794594,"entropy":0.5277553126215935},{"step":535,"reward":1.5372940301895142,"core":0.8601128458976746,"cost":0.0,"kl":0.282128244638443,"entropy":0.4457874447107315},{"step":537,"reward":1.4560279846191406,"core":0.7782034873962402,"cost":0.0,"kl":0.3784065544605255,"entropy":0.5211841017007828},{"step":539,"reward":0.918557345867157,"core":0.5743778944015503,"cost":0.0,"kl":0.37123682349920273,"entropy":0.5632769167423248},{"step":541,"reward":1.1891191005706787,"core":0.590174674987793,"cost":0.0,"kl":0.3404095396399498,"entropy":0.5541228652000427},{"step":543,"reward":1.2604666948318481,"core":0.7365545630455017,"cost":0.0,"kl":0.26767127215862274,"entropy":0.49931715428829193},{"step":545,"reward":1.187368392944336,"core":0.6903743743896484,"cost":0.0,"kl":0.37155307084321976,"entropy":0.5346287935972214},{"step":547,"reward":0.6916952133178711,"core":0.5205439925193787,"cost":0.0,"kl":0.30647333711385727,"entropy":0.524225577712059},{"step":549,"reward":1.5639585256576538,"core":0.7923533320426941,"cost":0.0,"kl":0.367901936173439,"entropy":0.4949047341942787},{"step":551,"reward":0.6504274010658264,"core":0.457289457321167,"cost":0.0,"kl":0.4542786106467247,"entropy":0.553803101181984},{"step":553,"reward":1.4671096801757812,"core":0.8457741141319275,"cost":0.0,"kl":0.3407251238822937,"entropy":0.508706197142601},{"step":555,"reward":1.507973074913025,"core":0.8990767598152161,"cost":0.0,"kl":0.2659200690686703,"entropy":0.5001865178346634},{"step":557,"reward":0.9122889041900635,"core":0.536982536315918,"cost":-0.0019143667304888368,"kl":0.31543679535388947,"entropy":0.5077360942959785},{"step":559,"reward":0.6447528004646301,"core":0.4989047050476074,"cost":0.0,"kl":0.38553307205438614,"entropy":0.5601785629987717},{"step":561,"reward":0.79339998960495,"core":0.496938019990921,"cost":0.0,"kl":0.4918610751628876,"entropy":0.5915090143680573},{"step":563,"reward":0.3229524791240692,"core":0.35187646746635437,"cost":0.0,"kl":0.3583175465464592,"entropy":0.55368272960186},{"step":565,"reward":0.659317135810852,"core":0.4723373353481293,"cost":-0.007720716763287783,"kl":0.24831503629684448,"entropy":0.5321042090654373},{"step":567,"reward":0.8315918445587158,"core":0.46201834082603455,"cost":-0.017368895933032036,"kl":0.36742809414863586,"entropy":0.5832142233848572},{"step":569,"reward":1.3280210494995117,"core":0.7215352058410645,"cost":0.0,"kl":0.2605246715247631,"entropy":0.5153355076909065},{"step":571,"reward":0.8715431094169617,"core":0.601751983165741,"cost":0.0,"kl":0.3750206306576729,"entropy":0.5360155999660492},{"step":573,"reward":1.222738265991211,"core":0.5335498452186584,"cost":0.0,"kl":0.3595639541745186,"entropy":0.5364615321159363},{"step":575,"reward":0.8416958451271057,"core":0.5532345175743103,"cost":0.0,"kl":0.3162037841975689,"entropy":0.5783839225769043},{"step":577,"reward":0.8189361095428467,"core":0.4546474516391754,"cost":0.0,"kl":0.30869830027222633,"entropy":0.5708921998739243},{"step":579,"reward":0.7405531406402588,"core":0.42175018787384033,"cost":0.0,"kl":0.3064114972949028,"entropy":0.5913536697626114},{"step":581,"reward":0.07626499980688095,"core":0.2526450455188751,"cost":-0.009019769728183746,"kl":0.3306024372577667,"entropy":0.49841319769620895},{"step":583,"reward":0.42454567551612854,"core":0.4223880469799042,"cost":-0.007788743823766708,"kl":0.3538488820195198,"entropy":0.6174066811800003},{"step":585,"reward":1.2867969274520874,"core":0.7414560317993164,"cost":0.0,"kl":0.24176780879497528,"entropy":0.508063517510891},{"step":587,"reward":1.2174930572509766,"core":0.6134608387947083,"cost":0.0,"kl":0.30416662991046906,"entropy":0.47647521644830704},{"step":589,"reward":0.7935133576393127,"core":0.5330770611763,"cost":0.0,"kl":0.26613669097423553,"entropy":0.5496535301208496},{"step":591,"reward":0.8371065258979797,"core":0.5994277596473694,"cost":-0.04371875524520874,"kl":0.3626948222517967,"entropy":0.5513704419136047},{"step":593,"reward":1.1669148206710815,"core":0.7197847366333008,"cost":0.0,"kl":0.2577858492732048,"entropy":0.5017066150903702},{"step":595,"reward":0.9865476489067078,"core":0.6364677548408508,"cost":0.0,"kl":0.34433913975954056,"entropy":0.5756095498800278},{"step":597,"reward":0.23759599030017853,"core":0.26560118794441223,"cost":-0.014873389154672623,"kl":0.3019797131419182,"entropy":0.5966010093688965},{"step":599,"reward":0.8966676592826843,"core":0.6080208420753479,"cost":-0.01343587040901184,"kl":0.29834963753819466,"entropy":0.5612747073173523}],"nonzero_advantage":[{"step":1,"nonzero":0.20833333333333334},{"step":3,"nonzero":0.4375},{"step":5,"nonzero":0.20833333333333334},{"step":7,"nonzero":0.7708333333333334},{"step":9,"nonzero":0.5625},{"step":11,"nonzero":0.4375},{"step":13,"nonzero":0.5208333333333334},{"step":15,"nonzero":0.6666666666666666},{"step":17,"nonzero":0.7708333333333334},{"step":19,"nonzero":0.4791666666666667},{"step":21,"nonzero":0.2916666666666667},{"step":23,"nonzero":0.5833333333333334},{"step":25,"nonzero":0.6041666666666666},{"step":27,"nonzero":0.5208333333333334},{"step":29,"nonzero":0.875},{"step":31,"nonzero":0.7083333333333334},{"step":33,"nonzero":0.625},{"step":35,"nonzero":0.6041666666666666},{"step":37,"nonzero":0.625},{"step":39,"nonzero":0.75},{"step":41,"nonzero":0.4583333333333333},{"step":43,"nonzero":0.875},{"step":45,"nonzero":0.6875},{"step":47,"nonzero":0.75},{"step":49,"nonzero":0.7291666666666666},{"step":51,"nonzero":0.9375},{"step":53,"nonzero":0.8125},{"step":55,"nonzero":0.5416666666666666},{"step":57,"nonzero":0.8333333333333334},{"step":59,"nonzero":0.875},{"step":61,"nonzero":0.7708333333333334},{"step":63,"nonzero":0.6875},{"step":65,"nonzero":0.5416666666666666},{"step":67,"nonzero":0.6458333333333334},{"step":69,"nonzero":0.6666666666666666},{"step":71,"nonzero":0.8958333333333334},{"step":73,"nonzero":0.7291666666666666},{"step":75,"nonzero":0.8541666666666666},{"step":77,"nonzero":0.9375},{"step":79,"nonzero":0.9375},{"step":81,"nonzero":0.8125},{"step":83,"nonzero":0.7291666666666666},{"step":85,"nonzero":0.9166666666666666},{"step":87,"nonzero":0.875},{"step":89,"nonzero":0.9583333333333334},{"step":91,"nonzero":0.9166666666666666},{"step":93,"nonzero":0.8541666666666666},{"step":95,"nonzero":0.6458333333333334},{"step":97,"nonzero":0.8958333333333334},{"step":99,"nonzero":0.9375},{"step":101,"nonzero":0.8541666666666666},{"step":103,"nonzero":0.875},{"step":105,"nonzero":0.8333333333333334},{"step":107,"nonzero":0.9791666666666666},{"step":109,"nonzero":0.875},{"step":111,"nonzero":0.8541666666666666},{"step":113,"nonzero":0.7708333333333334},{"step":115,"nonzero":0.8958333333333334},{"step":117,"nonzero":0.8958333333333334},{"step":119,"nonzero":0.7708333333333334},{"step":121,"nonzero":0.9583333333333334},{"step":123,"nonzero":0.8958333333333334},{"step":125,"nonzero":0.875},{"step":127,"nonzero":0.7708333333333334},{"step":129,"nonzero":0.6666666666666666},{"step":131,"nonzero":0.8333333333333334},{"step":133,"nonzero":0.9375},{"step":135,"nonzero":0.875},{"step":137,"nonzero":0.9166666666666666},{"step":139,"nonzero":0.6875},{"step":141,"nonzero":0.9375},{"step":143,"nonzero":0.875},{"step":145,"nonzero":0.9583333333333334},{"step":147,"nonzero":0.6875},{"step":149,"nonzero":0.9375},{"step":151,"nonzero":0.7083333333333334},{"step":153,"nonzero":0.7708333333333334},{"step":155,"nonzero":0.5416666666666666},{"step":157,"nonzero":0.875},{"step":159,"nonzero":0.875},{"step":161,"nonzero":0.7708333333333334},{"step":163,"nonzero":0.9583333333333334},{"step":165,"nonzero":0.8333333333333334},{"step":167,"nonzero":0.8958333333333334},{"step":169,"nonzero":0.9375},{"step":171,"nonzero":0.8541666666666666},{"step":173,"nonzero":0.9791666666666666},{"step":175,"nonzero":0.9375},{"step":177,"nonzero":0.7083333333333334},{"step":179,"nonzero":0.9375},{"step":181,"nonzero":0.9583333333333334},{"step":183,"nonzero":0.8958333333333334},{"step":185,"nonzero":0.9583333333333334},{"step":187,"nonzero":0.8958333333333334},{"step":189,"nonzero":0.8333333333333334},{"step":191,"nonzero":0.875},{"step":193,"nonzero":0.9375},{"step":195,"nonzero":0.7083333333333334},{"step":197,"nonzero":0.8541666666666666},{"step":199,"nonzero":0.9375},{"step":201,"nonzero":0.8125},{"step":203,"nonzero":0.9791666666666666},{"step":205,"nonzero":0.9791666666666666},{"step":207,"nonzero":0.9791666666666666},{"step":209,"nonzero":0.7291666666666666},{"step":211,"nonzero":0.9166666666666666},{"step":213,"nonzero":0.9166666666666666},{"step":215,"nonzero":0.8958333333333334},{"step":217,"nonzero":0.9375},{"step":219,"nonzero":0.9791666666666666},{"step":221,"nonzero":0.875},{"step":223,"nonzero":0.9583333333333334},{"step":225,"nonzero":0.9583333333333334},{"step":227,"nonzero":0.6875},{"step":229,"nonzero":0.9166666666666666},{"step":231,"nonzero":0.9166666666666666},{"step":233,"nonzero":0.9375},{"step":235,"nonzero":0.9583333333333334},{"step":237,"nonzero":1.0},{"step":239,"nonzero":0.9791666666666666},{"step":241,"nonzero":0.9583333333333334},{"step":243,"nonzero":1.0},{"step":245,"nonzero":0.9791666666666666},{"step":247,"nonzero":0.9791666666666666},{"step":249,"nonzero":0.9166666666666666},{"step":251,"nonzero":0.9166666666666666},{"step":253,"nonzero":0.9791666666666666},{"step":255,"nonzero":0.9791666666666666},{"step":257,"nonzero":0.9166666666666666},{"step":259,"nonzero":0.9791666666666666},{"step":261,"nonzero":0.9375},{"step":263,"nonzero":0.9375},{"step":265,"nonzero":0.9583333333333334},{"step":267,"nonzero":0.9791666666666666},{"step":269,"nonzero":1.0},{"step":271,"nonzero":0.8958333333333334},{"step":273,"nonzero":1.0},{"step":275,"nonzero":1.0},{"step":277,"nonzero":1.0},{"step":279,"nonzero":0.9791666666666666},{"step":281,"nonzero":0.9791666666666666},{"step":283,"nonzero":1.0},{"step":285,"nonzero":1.0},{"step":287,"nonzero":1.0},{"step":289,"nonzero":0.9791666666666666},{"step":291,"nonzero":0.9583333333333334},{"step":293,"nonzero":0.9166666666666666},{"step":295,"nonzero":0.9583333333333334},{"step":297,"nonzero":0.9583333333333334},{"step":299,"nonzero":0.8958333333333334},{"step":301,"nonzero":0.8541666666666666},{"step":303,"nonzero":1.0},{"step":305,"nonzero":0.9791666666666666},{"step":307,"nonzero":0.9791666666666666},{"step":309,"nonzero":0.9791666666666666},{"step":311,"nonzero":0.8958333333333334},{"step":313,"nonzero":0.875},{"step":315,"nonzero":1.0},{"step":317,"nonzero":0.9583333333333334},{"step":319,"nonzero":1.0},{"step":321,"nonzero":0.9375},{"step":323,"nonzero":0.9791666666666666},{"step":325,"nonzero":0.9791666666666666},{"step":327,"nonzero":0.9166666666666666},{"step":329,"nonzero":0.8125},{"step":331,"nonzero":0.7708333333333334},{"step":333,"nonzero":0.9166666666666666},{"step":335,"nonzero":0.9791666666666666},{"step":337,"nonzero":1.0},{"step":339,"nonzero":0.9583333333333334},{"step":341,"nonzero":0.875},{"step":343,"nonzero":1.0},{"step":345,"nonzero":0.9583333333333334},{"step":347,"nonzero":1.0},{"step":349,"nonzero":0.9375},{"step":351,"nonzero":0.9375},{"step":353,"nonzero":0.9583333333333334},{"step":355,"nonzero":0.9166666666666666},{"step":357,"nonzero":1.0},{"step":359,"nonzero":0.9583333333333334},{"step":361,"nonzero":0.9166666666666666},{"step":363,"nonzero":0.9375},{"step":365,"nonzero":0.9166666666666666},{"step":367,"nonzero":1.0},{"step":369,"nonzero":0.9791666666666666},{"step":371,"nonzero":0.9166666666666666},{"step":373,"nonzero":0.9375},{"step":375,"nonzero":1.0},{"step":377,"nonzero":0.9791666666666666},{"step":379,"nonzero":0.75},{"step":381,"nonzero":0.7291666666666666},{"step":383,"nonzero":0.9791666666666666},{"step":385,"nonzero":0.9791666666666666},{"step":387,"nonzero":0.9375},{"step":389,"nonzero":0.9583333333333334},{"step":391,"nonzero":0.9375},{"step":393,"nonzero":0.9375},{"step":395,"nonzero":0.9791666666666666},{"step":397,"nonzero":0.9583333333333334},{"step":399,"nonzero":0.9375},{"step":401,"nonzero":0.9583333333333334},{"step":403,"nonzero":0.9791666666666666},{"step":405,"nonzero":1.0},{"step":407,"nonzero":1.0},{"step":409,"nonzero":0.9166666666666666},{"step":411,"nonzero":1.0},{"step":413,"nonzero":0.9791666666666666},{"step":415,"nonzero":0.9791666666666666},{"step":417,"nonzero":0.9791666666666666},{"step":419,"nonzero":0.9791666666666666},{"step":421,"nonzero":0.9791666666666666},{"step":423,"nonzero":0.9375},{"step":425,"nonzero":1.0},{"step":427,"nonzero":0.9375},{"step":429,"nonzero":0.9791666666666666},{"step":431,"nonzero":0.9791666666666666},{"step":433,"nonzero":0.9166666666666666},{"step":435,"nonzero":0.9583333333333334},{"step":437,"nonzero":1.0},{"step":439,"nonzero":0.9583333333333334},{"step":441,"nonzero":0.9791666666666666},{"step":443,"nonzero":0.9375},{"step":445,"nonzero":1.0},{"step":447,"nonzero":0.9583333333333334},{"step":449,"nonzero":1.0},{"step":451,"nonzero":0.9375},{"step":453,"nonzero":0.9583333333333334},{"step":455,"nonzero":0.9791666666666666},{"step":457,"nonzero":1.0},{"step":459,"nonzero":0.9791666666666666},{"step":461,"nonzero":0.9791666666666666},{"step":463,"nonzero":1.0},{"step":465,"nonzero":0.9791666666666666},{"step":467,"nonzero":0.9375},{"step":469,"nonzero":1.0},{"step":471,"nonzero":0.8541666666666666},{"step":473,"nonzero":0.9583333333333334},{"step":475,"nonzero":1.0},{"step":477,"nonzero":0.9791666666666666},{"step":479,"nonzero":1.0},{"step":481,"nonzero":1.0},{"step":483,"nonzero":1.0},{"step":485,"nonzero":0.9791666666666666},{"step":487,"nonzero":1.0},{"step":489,"nonzero":1.0},{"step":491,"nonzero":1.0},{"step":493,"nonzero":0.9583333333333334},{"step":495,"nonzero":1.0},{"step":497,"nonzero":1.0},{"step":499,"nonzero":1.0},{"step":501,"nonzero":1.0},{"step":503,"nonzero":1.0},{"step":505,"nonzero":0.9583333333333334},{"step":507,"nonzero":0.9583333333333334},{"step":509,"nonzero":1.0},{"step":511,"nonzero":1.0},{"step":513,"nonzero":0.9375},{"step":515,"nonzero":0.9583333333333334},{"step":517,"nonzero":1.0},{"step":519,"nonzero":1.0},{"step":521,"nonzero":1.0},{"step":523,"nonzero":1.0},{"step":525,"nonzero":1.0},{"step":527,"nonzero":1.0},{"step":529,"nonzero":1.0},{"step":531,"nonzero":1.0},{"step":533,"nonzero":1.0},{"step":535,"nonzero":1.0},{"step":537,"nonzero":1.0},{"step":539,"nonzero":1.0},{"step":541,"nonzero":1.0},{"step":543,"nonzero":1.0},{"step":545,"nonzero":0.9791666666666666},{"step":547,"nonzero":1.0},{"step":549,"nonzero":1.0},{"step":551,"nonzero":0.9583333333333334},{"step":553,"nonzero":1.0},{"step":555,"nonzero":1.0},{"step":557,"nonzero":0.9375},{"step":559,"nonzero":0.9583333333333334},{"step":561,"nonzero":0.9583333333333334},{"step":563,"nonzero":0.9791666666666666},{"step":565,"nonzero":0.875},{"step":567,"nonzero":1.0},{"step":569,"nonzero":1.0},{"step":571,"nonzero":0.9583333333333334},{"step":573,"nonzero":1.0},{"step":575,"nonzero":1.0},{"step":577,"nonzero":0.9791666666666666},{"step":579,"nonzero":0.9791666666666666},{"step":581,"nonzero":0.875},{"step":583,"nonzero":0.9583333333333334},{"step":585,"nonzero":1.0},{"step":587,"nonzero":1.0},{"step":589,"nonzero":0.9791666666666666},{"step":591,"nonzero":1.0},{"step":593,"nonzero":1.0},{"step":595,"nonzero":1.0},{"step":597,"nonzero":0.9583333333333334},{"step":599,"nonzero":0.9375}]},{"id":"cispo-repor-dora-226","label":"CISPO + REPO-R \u00b7 DoRA \u00b7 226 prompts","run_ids":["1790967377"],"loss":"CISPO","adapter":"DoRA","prompts":226,"repo_r":true,"holdout_by_checkpoint":[{"step":0,"holdout":-1.3664,"train_core":-0.9836,"valid_rate":0.9},{"step":100,"holdout":-0.3755,"train_core":-0.2094,"valid_rate":0.86},{"step":200,"holdout":-0.1223,"train_core":-0.0188,"valid_rate":0.96},{"step":300,"holdout":0.6011,"train_core":0.2761,"valid_rate":0.98},{"step":400,"holdout":1.1007,"train_core":0.5559,"valid_rate":0.98},{"step":500,"holdout":1.3328,"train_core":0.7074,"valid_rate":1.0},{"step":600,"holdout":1.1701,"train_core":0.6452,"valid_rate":1.0}],"selected_step":500,"selected_holdout":1.3328,"train_steps":[{"step":1,"reward":-2.015054941177368,"core":-0.6128399968147278,"cost":-0.08141078054904938,"kl":0.0,"entropy":0.5671490281820297},{"step":3,"reward":-2.440347671508789,"core":-0.7799794673919678,"cost":-0.07195515185594559,"kl":0.00073854552465491,"entropy":0.5408806204795837},{"step":5,"reward":-2.4211342334747314,"core":-0.746096134185791,"cost":-0.1366967409849167,"kl":0.002611725707538426,"entropy":0.5794654786586761},{"step":7,"reward":-1.0743192434310913,"core":-0.26671040058135986,"cost":-0.06256978958845139,"kl":0.004993017413653433,"entropy":0.5396430194377899},{"step":9,"reward":-1.6314929723739624,"core":-0.5115444660186768,"cost":-0.04642863944172859,"kl":0.00621095416136086,"entropy":0.5786535441875458},{"step":11,"reward":-1.9702516794204712,"core":-0.5578393340110779,"cost":-0.11937008053064346,"kl":0.006256696418859065,"entropy":0.505066841840744},{"step":13,"reward":-1.4573779106140137,"core":-0.44245603680610657,"cost":-0.05671730637550354,"kl":0.011403669835999608,"entropy":0.5138096287846565},{"step":15,"reward":-2.4100067615509033,"core":-0.6425248980522156,"cost":-0.22064878046512604,"kl":0.01705639809370041,"entropy":0.4213600903749466},{"step":17,"reward":-1.8395506143569946,"core":-0.547971785068512,"cost":-0.014112201519310474,"kl":0.021537464577704668,"entropy":0.49812988191843033},{"step":19,"reward":-2.000102996826172,"core":-0.6110371947288513,"cost":-0.05535535886883736,"kl":0.01579722505994141,"entropy":0.5466396734118462},{"step":21,"reward":-1.74514639377594,"core":-0.5295297503471375,"cost":-0.007105882745236158,"kl":0.017711794702336192,"entropy":0.512687437236309},{"step":23,"reward":-1.5099554061889648,"core":-0.4457283020019531,"cost":-0.04212662950158119,"kl":0.01718687266111374,"entropy":0.4756305292248726},{"step":25,"reward":-2.826870918273926,"core":-0.8259563446044922,"cost":-0.21804170310497284,"kl":0.013215029845014215,"entropy":0.3971443548798561},{"step":27,"reward":-3.2970314025878906,"core":-0.9569001793861389,"cost":-0.32479724287986755,"kl":0.0242842105217278,"entropy":0.5388441234827042},{"step":29,"reward":-1.3338035345077515,"core":-0.38924479484558105,"cost":-0.03408481553196907,"kl":0.02611663332208991,"entropy":0.4998457729816437},{"step":31,"reward":-1.285676121711731,"core":-0.39154931902885437,"cost":-0.011517509818077087,"kl":0.023696288000792265,"entropy":0.5415685325860977},{"step":33,"reward":-1.7723582983016968,"core":-0.4947119653224945,"cost":-0.10716661065816879,"kl":0.03294609347358346,"entropy":0.500070795416832},{"step":35,"reward":-1.5770683288574219,"core":-0.4502204358577728,"cost":-0.06594235450029373,"kl":0.03770518722012639,"entropy":0.5160528421401978},{"step":37,"reward":-1.3488746881484985,"core":-0.38182222843170166,"cost":-0.09007395058870316,"kl":0.041269184090197086,"entropy":0.5003904551267624},{"step":39,"reward":-1.32072114944458,"core":-0.35987234115600586,"cost":-0.05142321065068245,"kl":0.025459191761910915,"entropy":0.41019660234451294},{"step":41,"reward":-1.8329321146011353,"core":-0.5367733240127563,"cost":-0.0784853920340538,"kl":0.02585726184770465,"entropy":0.4899963140487671},{"step":43,"reward":-0.6658007502555847,"core":-0.13447146117687225,"cost":-0.0008488150197081268,"kl":0.03173482930287719,"entropy":0.5149584114551544},{"step":45,"reward":-1.000062346458435,"core":-0.23896460235118866,"cost":-0.002152347005903721,"kl":0.04374961834400892,"entropy":0.5894315242767334},{"step":47,"reward":-1.5376701354980469,"core":-0.4298640787601471,"cost":-0.06251808255910873,"kl":0.039743716828525066,"entropy":0.5542385578155518},{"step":49,"reward":-1.0394209623336792,"core":-0.2570621073246002,"cost":-0.03169594705104828,"kl":0.055996621027588844,"entropy":0.5845648497343063},{"step":51,"reward":-2.2122631072998047,"core":-0.6218176484107971,"cost":-0.1403428316116333,"kl":0.04709773790091276,"entropy":0.5724490880966187},{"step":53,"reward":-1.4879802465438843,"core":-0.3880026638507843,"cost":-0.009282045066356659,"kl":0.06156735960394144,"entropy":0.6439669132232666},{"step":55,"reward":-1.3405742645263672,"core":-0.3794616460800171,"cost":0.0,"kl":0.05939029809087515,"entropy":0.5418888553977013},{"step":57,"reward":-1.4422179460525513,"core":-0.365964412689209,"cost":-0.012462477199733257,"kl":0.08772887848317623,"entropy":0.528474971652031},{"step":59,"reward":-0.792783260345459,"core":-0.10978257656097412,"cost":-0.013913801871240139,"kl":0.08945326320827007,"entropy":0.5781567692756653},{"step":61,"reward":-0.9603431224822998,"core":-0.26323747634887695,"cost":0.0,"kl":0.08044476620852947,"entropy":0.5287886261940002},{"step":63,"reward":-0.7395862936973572,"core":-0.12916214764118195,"cost":-0.033438604325056076,"kl":0.08609216753393412,"entropy":0.57195845246315},{"step":65,"reward":-1.466531753540039,"core":-0.40485677123069763,"cost":-0.06542451679706573,"kl":0.08691387996077538,"entropy":0.4768199473619461},{"step":67,"reward":-1.2505348920822144,"core":-0.37736427783966064,"cost":-0.004572057165205479,"kl":0.08809278346598148,"entropy":0.600725382566452},{"step":69,"reward":-0.5678401589393616,"core":-0.0733848512172699,"cost":-0.001989839132875204,"kl":0.08595042116940022,"entropy":0.4664603918790817},{"step":71,"reward":-0.4885600507259369,"core":-0.06826529651880264,"cost":-0.0020003828685730696,"kl":0.2585138287395239,"entropy":0.5685197114944458},{"step":73,"reward":-0.8252347111701965,"core":-0.20494455099105835,"cost":0.0,"kl":0.08898886665701866,"entropy":0.49827588349580765},{"step":75,"reward":0.2616559565067291,"core":0.211361363530159,"cost":-0.05129649117588997,"kl":0.10936043784022331,"entropy":0.532875545322895},{"step":77,"reward":-1.0181292295455933,"core":-0.26700660586357117,"cost":0.0,"kl":0.11731206625699997,"entropy":0.643876776099205},{"step":79,"reward":-1.169976830482483,"core":-0.24885892868041992,"cost":-0.0718550756573677,"kl":0.11514193378388882,"entropy":0.53249691426754},{"step":81,"reward":-0.8862956166267395,"core":-0.1842167228460312,"cost":-0.017502561211586,"kl":0.1175836119800806,"entropy":0.622677281498909},{"step":83,"reward":-1.3416682481765747,"core":-0.38079187273979187,"cost":-0.039682988077402115,"kl":0.0936489887535572,"entropy":0.5181429907679558},{"step":85,"reward":-0.7056947350502014,"core":-0.11849109083414078,"cost":-0.018972108140587807,"kl":0.147178553044796,"entropy":0.5866300463676453},{"step":87,"reward":-0.8522235751152039,"core":-0.1994251012802124,"cost":-0.013965032994747162,"kl":0.08346892334520817,"entropy":0.5632291883230209},{"step":89,"reward":-1.1106399297714233,"core":-0.2729991376399994,"cost":-0.10777805000543594,"kl":0.09698650613427162,"entropy":0.5202042758464813},{"step":91,"reward":-0.182008758187294,"core":0.03316241875290871,"cost":0.0,"kl":0.15483490005135536,"entropy":0.5371576324105263},{"step":93,"reward":-1.1824687719345093,"core":-0.2939871847629547,"cost":-0.024480396881699562,"kl":0.09498926065862179,"entropy":0.4969476908445358},{"step":95,"reward":-2.1217029094696045,"core":-0.5766051411628723,"cost":-0.11453934758901596,"kl":0.10369466058909893,"entropy":0.49841246753931046},{"step":97,"reward":-0.6538683772087097,"core":-0.1439613550901413,"cost":-0.04710225388407707,"kl":0.10803464241325855,"entropy":0.5810141265392303},{"step":99,"reward":-0.5137032866477966,"core":-0.0238222386687994,"cost":-0.02136904001235962,"kl":0.14125862903892994,"entropy":0.613920047879219},{"step":101,"reward":-0.4039320945739746,"core":0.04806444048881531,"cost":-0.04554412141442299,"kl":0.11419838108122349,"entropy":0.5086728110909462},{"step":103,"reward":-0.6371457576751709,"core":-0.07879232615232468,"cost":-0.030848288908600807,"kl":0.13827016577124596,"entropy":0.5381855666637421},{"step":105,"reward":-0.8982667922973633,"core":-0.2044670730829239,"cost":-0.03464145585894585,"kl":0.11763240210711956,"entropy":0.4688768908381462},{"step":107,"reward":-0.4261382520198822,"core":0.0415828675031662,"cost":0.0,"kl":0.2108611725270748,"entropy":0.5048978105187416},{"step":109,"reward":0.17635543644428253,"core":0.20960955321788788,"cost":-0.0033263815566897392,"kl":0.16249513998627663,"entropy":0.45714281499385834},{"step":111,"reward":-0.48003748059272766,"core":-0.08261990547180176,"cost":-0.028552211821079254,"kl":0.14900994300842285,"entropy":0.47624966502189636},{"step":113,"reward":-0.5775846242904663,"core":-0.06848812848329544,"cost":-0.02669401466846466,"kl":0.15191950276494026,"entropy":0.49837934225797653},{"step":115,"reward":-0.7966135144233704,"core":-0.1381753534078598,"cost":-0.02068057842552662,"kl":0.1425506565719843,"entropy":0.4086318090558052},{"step":117,"reward":-0.2993071973323822,"core":0.036569297313690186,"cost":0.0,"kl":0.15024632960557938,"entropy":0.5159672647714615},{"step":119,"reward":-0.20510883629322052,"core":0.03462502360343933,"cost":-0.02739897184073925,"kl":0.15785450860857964,"entropy":0.443325512111187},{"step":121,"reward":0.3287961184978485,"core":0.23932218551635742,"cost":-0.02293783612549305,"kl":0.16885250806808472,"entropy":0.48330020904541016},{"step":123,"reward":0.058147866278886795,"core":0.17601720988750458,"cost":0.0,"kl":0.13863827474415302,"entropy":0.431412898004055},{"step":125,"reward":-0.6175068020820618,"core":-0.10863861441612244,"cost":-0.0163718294352293,"kl":0.18716783821582794,"entropy":0.49292169511318207},{"step":127,"reward":-1.8412466049194336,"core":-0.5112336277961731,"cost":-0.15368862450122833,"kl":0.142540680244565,"entropy":0.550294354557991},{"step":129,"reward":-1.1114033460617065,"core":-0.30222585797309875,"cost":-0.028501011431217194,"kl":0.16899393498897552,"entropy":0.5259312763810158},{"step":131,"reward":-1.2316029071807861,"core":-0.3625064790248871,"cost":-0.006686534732580185,"kl":0.15698038786649704,"entropy":0.5217476487159729},{"step":133,"reward":-0.4287933111190796,"core":-0.015174287371337414,"cost":-0.02006005309522152,"kl":0.19342128932476044,"entropy":0.5780849009752274},{"step":135,"reward":-0.1031448021531105,"core":0.03649109601974487,"cost":-0.02225133217871189,"kl":0.19442376121878624,"entropy":0.5715677589178085},{"step":137,"reward":-0.06585660576820374,"core":0.0948951244354248,"cost":0.0,"kl":0.1794380433857441,"entropy":0.5942151844501495},{"step":139,"reward":-0.05487780645489693,"core":0.08248760551214218,"cost":0.0,"kl":0.19335078820586205,"entropy":0.536889374256134},{"step":141,"reward":0.02638278342783451,"core":0.12965579330921173,"cost":0.0,"kl":0.21071791276335716,"entropy":0.5322393476963043},{"step":143,"reward":-0.5500085949897766,"core":-0.07750280946493149,"cost":-0.02023041620850563,"kl":0.28483304381370544,"entropy":0.6700664311647415},{"step":145,"reward":-0.4576662480831146,"core":-0.03793041408061981,"cost":-0.0012260187650099397,"kl":0.21033268049359322,"entropy":0.4866403788328171},{"step":147,"reward":-0.3412446975708008,"core":0.006925210356712341,"cost":-0.013332716189324856,"kl":0.26748181134462357,"entropy":0.5429646074771881},{"step":149,"reward":-0.05086668208241463,"core":0.048646047711372375,"cost":-0.02072521112859249,"kl":0.2505398504436016,"entropy":0.5834751129150391},{"step":151,"reward":-0.8405520915985107,"core":-0.20126140117645264,"cost":-0.015759853646159172,"kl":0.21055541187524796,"entropy":0.6821494102478027},{"step":153,"reward":-1.262138843536377,"core":-0.33931758999824524,"cost":-0.056270744651556015,"kl":0.16483682952821255,"entropy":0.5652168989181519},{"step":155,"reward":-1.4044424295425415,"core":-0.3315163552761078,"cost":-0.1510971188545227,"kl":0.19737506657838821,"entropy":0.5672533512115479},{"step":157,"reward":0.3958907127380371,"core":0.25474104285240173,"cost":-0.012361954897642136,"kl":0.21028224378824234,"entropy":0.5538261085748672},{"step":159,"reward":-0.2725096046924591,"core":0.05968277528882027,"cost":-0.055509090423583984,"kl":0.19504770264029503,"entropy":0.6058643907308578},{"step":161,"reward":-0.5032888650894165,"core":-0.04545782878994942,"cost":-0.008196192793548107,"kl":0.18968594074249268,"entropy":0.5854191333055496},{"step":163,"reward":-0.003999034408479929,"core":0.11263076215982437,"cost":0.0,"kl":0.16931845992803574,"entropy":0.5733610093593597},{"step":165,"reward":-1.5301307439804077,"core":-0.43115344643592834,"cost":-0.08364877849817276,"kl":0.17481908947229385,"entropy":0.557991087436676},{"step":167,"reward":0.13007478415966034,"core":0.25189825892448425,"cost":-0.02451639622449875,"kl":0.1957714706659317,"entropy":0.5098972618579865},{"step":169,"reward":-0.06447786837816238,"core":0.09935620427131653,"cost":-0.03652505949139595,"kl":0.16726132482290268,"entropy":0.6209427863359451},{"step":171,"reward":-0.07687351107597351,"core":0.12118259072303772,"cost":-0.016888625919818878,"kl":0.17755340784788132,"entropy":0.574983537197113},{"step":173,"reward":0.010360796935856342,"core":0.1005016565322876,"cost":-0.014174592681229115,"kl":0.2371995560824871,"entropy":0.5911728590726852},{"step":175,"reward":0.1617906540632248,"core":0.25133073329925537,"cost":-0.013137049973011017,"kl":0.20433517917990685,"entropy":0.609003409743309},{"step":177,"reward":-0.6002960801124573,"core":-0.04795609414577484,"cost":-0.04732111096382141,"kl":0.15437524765729904,"entropy":0.47499096393585205},{"step":179,"reward":0.5585908889770508,"core":0.374224990606308,"cost":-0.013926784507930279,"kl":0.16264081001281738,"entropy":0.4729432985186577},{"step":181,"reward":0.10570503026247025,"core":0.14325650036334991,"cost":-0.017567461356520653,"kl":0.15228654071688652,"entropy":0.48817551881074905},{"step":183,"reward":0.044038910418748856,"core":0.1386508047580719,"cost":-0.049554627388715744,"kl":0.26608802378177643,"entropy":0.5200445428490639},{"step":185,"reward":0.15645518898963928,"core":0.14649450778961182,"cost":-0.00998616311699152,"kl":0.1827159933745861,"entropy":0.5476776883006096},{"step":187,"reward":-0.17193575203418732,"core":0.07019305974245071,"cost":0.0,"kl":0.25045114010572433,"entropy":0.6074684858322144},{"step":189,"reward":-0.0997438132762909,"core":0.06218399479985237,"cost":-0.026659945026040077,"kl":0.19263532385230064,"entropy":0.5336679890751839},{"step":191,"reward":-0.1668805032968521,"core":0.09625830501317978,"cost":-0.034344982355833054,"kl":0.1873762421309948,"entropy":0.49939175695180893},{"step":193,"reward":0.2549149990081787,"core":0.22877003252506256,"cost":-0.007727503776550293,"kl":0.1657955963164568,"entropy":0.5886194407939911},{"step":195,"reward":-0.39522066712379456,"core":-0.036610543727874756,"cost":-0.007943549193441868,"kl":0.19754787161946297,"entropy":0.5776665657758713},{"step":197,"reward":-0.43443021178245544,"core":-0.01763744093477726,"cost":-0.007152831181883812,"kl":0.14530537463724613,"entropy":0.5483042374253273},{"step":199,"reward":-0.7959621548652649,"core":-0.1452619880437851,"cost":-0.06825863569974899,"kl":0.2251301296055317,"entropy":0.5956558585166931},{"step":201,"reward":0.12301629036664963,"core":0.2298365831375122,"cost":0.0,"kl":0.20963415876030922,"entropy":0.620219886302948},{"step":203,"reward":-0.08592069149017334,"core":0.13901318609714508,"cost":-0.015880705788731575,"kl":0.1647566482424736,"entropy":0.5666761696338654},{"step":205,"reward":-0.31948524713516235,"core":0.07993005216121674,"cost":-0.03131423890590668,"kl":0.1969534456729889,"entropy":0.6410960853099823},{"step":207,"reward":0.6396003365516663,"core":0.41427263617515564,"cost":0.0,"kl":0.14119679294526577,"entropy":0.5593797117471695},{"step":209,"reward":-0.7711710333824158,"core":-0.1037084087729454,"cost":-0.04559573903679848,"kl":0.1890406385064125,"entropy":0.6143198907375336},{"step":211,"reward":0.14054618775844574,"core":0.17905394732952118,"cost":0.0,"kl":0.20851604267954826,"entropy":0.5242673084139824},{"step":213,"reward":-0.8333015441894531,"core":-0.17453467845916748,"cost":-0.043208539485931396,"kl":0.19141901656985283,"entropy":0.6311180144548416},{"step":215,"reward":-0.24917404353618622,"core":0.0985814556479454,"cost":-0.03400074318051338,"kl":0.19616062939167023,"entropy":0.6204268485307693},{"step":217,"reward":0.21434897184371948,"core":0.1576441377401352,"cost":-0.017217596992850304,"kl":0.1995772048830986,"entropy":0.4647250100970268},{"step":219,"reward":0.18573622405529022,"core":0.16209284961223602,"cost":-0.0092899389564991,"kl":0.20411023497581482,"entropy":0.5058746859431267},{"step":221,"reward":-0.0838628038764,"core":0.12012355774641037,"cost":-0.02394910715520382,"kl":0.22699743509292603,"entropy":0.5376400500535965},{"step":223,"reward":-0.04207407310605049,"core":0.09089275449514389,"cost":0.0,"kl":0.19498128816485405,"entropy":0.5393455922603607},{"step":225,"reward":0.07161835581064224,"core":0.1959352344274521,"cost":0.0,"kl":0.1791384220123291,"entropy":0.5164449438452721},{"step":227,"reward":0.015444725751876831,"core":0.1095563992857933,"cost":0.0,"kl":0.2091507911682129,"entropy":0.5507155805826187},{"step":229,"reward":0.07395251840353012,"core":0.11821136623620987,"cost":-0.01395601686090231,"kl":0.19435475021600723,"entropy":0.4703940525650978},{"step":231,"reward":0.08356112241744995,"core":0.20381583273410797,"cost":-0.0018199514597654343,"kl":0.22395987436175346,"entropy":0.5492475628852844},{"step":233,"reward":-0.11179965734481812,"core":0.0984315276145935,"cost":0.0,"kl":0.17338375374674797,"entropy":0.5581226050853729},{"step":235,"reward":0.32533976435661316,"core":0.27989813685417175,"cost":0.0,"kl":0.1617780700325966,"entropy":0.5943949520587921},{"step":237,"reward":0.3159380257129669,"core":0.26057150959968567,"cost":-0.0017548493342474103,"kl":0.19782987982034683,"entropy":0.5624512732028961},{"step":239,"reward":0.5571221709251404,"core":0.44024065136909485,"cost":-0.018998675048351288,"kl":0.20702064782381058,"entropy":0.458547480404377},{"step":241,"reward":-0.13656124472618103,"core":0.07751811295747757,"cost":-0.047966692596673965,"kl":0.23561852425336838,"entropy":0.4975404590368271},{"step":243,"reward":0.4128447473049164,"core":0.2884485423564911,"cost":-0.0008295883890241385,"kl":0.19081033393740654,"entropy":0.5638274550437927},{"step":245,"reward":0.5852843523025513,"core":0.42749857902526855,"cost":-0.033724088221788406,"kl":0.1903546303510666,"entropy":0.4569706469774246},{"step":247,"reward":-0.41143372654914856,"core":-0.012008324265480042,"cost":-0.03540962189435959,"kl":0.19120686873793602,"entropy":0.5675115287303925},{"step":249,"reward":0.5402724742889404,"core":0.3119015693664551,"cost":-0.01467205211520195,"kl":0.20468075573444366,"entropy":0.5296963378787041},{"step":251,"reward":-0.1930512636899948,"core":0.1412414312362671,"cost":-0.027172857895493507,"kl":0.20759078487753868,"entropy":0.5506215691566467},{"step":253,"reward":0.17406845092773438,"core":0.1939430832862854,"cost":0.0,"kl":0.20248707383871078,"entropy":0.5324226915836334},{"step":255,"reward":0.9508152604103088,"core":0.5325183272361755,"cost":0.0,"kl":0.22654173523187637,"entropy":0.46134594827890396},{"step":257,"reward":-0.21185684204101562,"core":0.1064327284693718,"cost":-0.028981270268559456,"kl":0.21861481666564941,"entropy":0.5398657768964767},{"step":259,"reward":-0.28888967633247375,"core":0.047030944377183914,"cost":0.0,"kl":0.18453602492809296,"entropy":0.539551854133606},{"step":261,"reward":0.061400990933179855,"core":0.17257778346538544,"cost":0.0,"kl":0.24822812527418137,"entropy":0.5623204261064529},{"step":263,"reward":-0.4081660211086273,"core":0.03262339159846306,"cost":-0.028022395446896553,"kl":0.17967496439814568,"entropy":0.5647075921297073},{"step":265,"reward":-0.5553152561187744,"core":-0.04945676401257515,"cost":-0.01797126606106758,"kl":0.24676591157913208,"entropy":0.5591722875833511},{"step":267,"reward":0.30868467688560486,"core":0.2561890184879303,"cost":0.0,"kl":0.20866992324590683,"entropy":0.5867724865674973},{"step":269,"reward":0.6145703196525574,"core":0.3986237943172455,"cost":0.0,"kl":0.22650425136089325,"entropy":0.5470864325761795},{"step":271,"reward":-0.09081313014030457,"core":0.0913056954741478,"cost":0.0,"kl":0.2428177408874035,"entropy":0.5823652297258377},{"step":273,"reward":0.6274693012237549,"core":0.42207762598991394,"cost":0.0,"kl":0.17804895713925362,"entropy":0.5613420009613037},{"step":275,"reward":0.22900165617465973,"core":0.19225521385669708,"cost":0.0,"kl":0.20293880254030228,"entropy":0.5916207432746887},{"step":277,"reward":0.5532386898994446,"core":0.4418454170227051,"cost":0.0,"kl":0.20377153158187866,"entropy":0.7726766914129257},{"step":279,"reward":0.9534969329833984,"core":0.48269161581993103,"cost":0.0,"kl":0.2135978266596794,"entropy":0.5716946721076965},{"step":281,"reward":1.0875730514526367,"core":0.5224589705467224,"cost":0.0,"kl":0.23099637404084206,"entropy":0.5975574105978012},{"step":283,"reward":0.41195976734161377,"core":0.3063589632511139,"cost":0.0,"kl":0.16471955180168152,"entropy":0.5162154138088226},{"step":285,"reward":0.7289654612541199,"core":0.4186532199382782,"cost":0.0,"kl":0.2107430323958397,"entropy":0.5284873098134995},{"step":287,"reward":0.22115081548690796,"core":0.25666239857673645,"cost":0.0,"kl":0.21907704323530197,"entropy":0.5508056655526161},{"step":289,"reward":0.4173842668533325,"core":0.3688087463378906,"cost":0.0,"kl":0.20746934041380882,"entropy":0.529477596282959},{"step":291,"reward":0.5726675391197205,"core":0.3778074085712433,"cost":-0.002074882620945573,"kl":0.17018477991223335,"entropy":0.5352687686681747},{"step":293,"reward":1.0472296476364136,"core":0.5833916664123535,"cost":0.0,"kl":0.22348163649439812,"entropy":0.5000899136066437},{"step":295,"reward":0.7645149230957031,"core":0.42704126238822937,"cost":-0.014192656613886356,"kl":0.2384018748998642,"entropy":0.566267117857933},{"step":297,"reward":0.33652815222740173,"core":0.2782372236251831,"cost":-0.003977505024522543,"kl":0.24035851657390594,"entropy":0.4482670873403549},{"step":299,"reward":0.250137060880661,"core":0.2531959116458893,"cost":-0.0012322979746386409,"kl":0.26593440026044846,"entropy":0.5835347026586533},{"step":301,"reward":0.2603508532047272,"core":0.25162187218666077,"cost":0.0,"kl":0.2981936112046242,"entropy":0.4887511283159256},{"step":303,"reward":0.6625703573226929,"core":0.43470120429992676,"cost":0.0,"kl":0.3017372563481331,"entropy":0.4839605912566185},{"step":305,"reward":1.3057066202163696,"core":0.6959131360054016,"cost":0.0,"kl":0.345828615128994,"entropy":0.5325234830379486},{"step":307,"reward":1.0187069177627563,"core":0.4839310944080353,"cost":0.0,"kl":0.21583149582147598,"entropy":0.5406054258346558},{"step":309,"reward":0.9130447506904602,"core":0.46516740322113037,"cost":-0.0009055958944372833,"kl":0.31843411922454834,"entropy":0.5198389887809753},{"step":311,"reward":0.7817850708961487,"core":0.4134279191493988,"cost":0.0,"kl":0.29635632783174515,"entropy":0.6079807281494141},{"step":313,"reward":0.8229081034660339,"core":0.5389413833618164,"cost":0.0,"kl":0.3005494885146618,"entropy":0.583073228597641},{"step":315,"reward":1.3323127031326294,"core":0.7360413670539856,"cost":0.0,"kl":0.21231240034103394,"entropy":0.47802234441041946},{"step":317,"reward":0.7590446472167969,"core":0.45541438460350037,"cost":0.0,"kl":0.3766276091337204,"entropy":0.5645955801010132},{"step":319,"reward":1.2287284135818481,"core":0.7172228693962097,"cost":-0.0010764418402686715,"kl":0.30471061915159225,"entropy":0.4824150502681732},{"step":321,"reward":0.7982451915740967,"core":0.4799439609050751,"cost":0.0,"kl":0.3928050324320793,"entropy":0.5613244920969009},{"step":323,"reward":1.240173578262329,"core":0.6725578308105469,"cost":0.0,"kl":0.3386251851916313,"entropy":0.49530135095119476},{"step":325,"reward":0.528247058391571,"core":0.3692340850830078,"cost":-0.06314539164304733,"kl":0.4341055452823639,"entropy":0.6063304841518402},{"step":327,"reward":0.7265661358833313,"core":0.40064290165901184,"cost":0.0,"kl":0.32030578702688217,"entropy":0.6286008507013321},{"step":329,"reward":0.13215045630931854,"core":0.2395920306444168,"cost":-0.0018274210160598159,"kl":0.39653507620096207,"entropy":0.6440892368555069},{"step":331,"reward":1.1485041379928589,"core":0.6014314889907837,"cost":0.0,"kl":0.32092002779245377,"entropy":0.5828108042478561},{"step":333,"reward":0.8856002688407898,"core":0.5004709362983704,"cost":0.0,"kl":0.416461743414402,"entropy":0.5976029336452484},{"step":335,"reward":0.28024014830589294,"core":0.25093865394592285,"cost":-0.030639955773949623,"kl":0.3541894927620888,"entropy":0.6488805860280991},{"step":337,"reward":0.42186498641967773,"core":0.24026678502559662,"cost":-0.004580093082040548,"kl":0.33235543221235275,"entropy":0.6808156073093414},{"step":339,"reward":0.8037335276603699,"core":0.4925667941570282,"cost":0.0,"kl":0.35594336688518524,"entropy":0.6318759918212891},{"step":341,"reward":0.39184901118278503,"core":0.2884902060031891,"cost":-0.02103065513074398,"kl":0.35227298736572266,"entropy":0.6478331536054611},{"step":343,"reward":0.8023577332496643,"core":0.39498695731163025,"cost":-0.02887188456952572,"kl":0.3368801027536392,"entropy":0.5615251511335373},{"step":345,"reward":0.6359102129936218,"core":0.38717153668403625,"cost":-0.009103178977966309,"kl":0.35515712201595306,"entropy":0.6041739881038666},{"step":347,"reward":1.5418716669082642,"core":0.6686217784881592,"cost":0.0,"kl":0.2767065651714802,"entropy":0.47154103219509125},{"step":349,"reward":0.7503418922424316,"core":0.46403253078460693,"cost":-0.008212663233280182,"kl":0.3156949207186699,"entropy":0.6127124726772308},{"step":351,"reward":0.4901387393474579,"core":0.2960706055164337,"cost":-0.0004837973683606833,"kl":0.38182590156793594,"entropy":0.692796990275383},{"step":353,"reward":0.9139881134033203,"core":0.5517544746398926,"cost":0.0,"kl":0.3206074982881546,"entropy":0.5850251168012619},{"step":355,"reward":-0.46520528197288513,"core":-0.031503234058618546,"cost":0.0,"kl":0.2520183026790619,"entropy":0.644393265247345},{"step":357,"reward":0.9766449332237244,"core":0.6152858734130859,"cost":0.0,"kl":0.2733115814626217,"entropy":0.6377589106559753},{"step":359,"reward":1.5091676712036133,"core":0.7400515079498291,"cost":0.0,"kl":0.34863483160734177,"entropy":0.589663952589035},{"step":361,"reward":0.7090163230895996,"core":0.45437121391296387,"cost":0.0,"kl":0.2955305799841881,"entropy":0.5518606156110764},{"step":363,"reward":0.369605153799057,"core":0.274217814207077,"cost":0.0,"kl":0.24801291525363922,"entropy":0.6575299054384232},{"step":365,"reward":0.4847519099712372,"core":0.369110107421875,"cost":-0.01600443385541439,"kl":0.35912565141916275,"entropy":0.5561230331659317},{"step":367,"reward":1.2296174764633179,"core":0.6300299763679504,"cost":-0.015408550389111042,"kl":0.3427671864628792,"entropy":0.5105634704232216},{"step":369,"reward":0.44786742329597473,"core":0.32064804434776306,"cost":0.0,"kl":0.33552025258541107,"entropy":0.592067152261734},{"step":371,"reward":0.8839476704597473,"core":0.49223998188972473,"cost":0.0,"kl":0.31139541044831276,"entropy":0.5324418842792511},{"step":373,"reward":1.1120911836624146,"core":0.6507244110107422,"cost":0.0,"kl":0.3346451669931412,"entropy":0.5815166532993317},{"step":375,"reward":1.1433840990066528,"core":0.6227499842643738,"cost":0.0,"kl":0.3486993685364723,"entropy":0.5709662884473801},{"step":377,"reward":0.6234425902366638,"core":0.3866443634033203,"cost":0.0,"kl":0.22619084641337395,"entropy":0.5062755569815636},{"step":379,"reward":0.2960177958011627,"core":0.31049931049346924,"cost":-0.0035351335536688566,"kl":0.26047827303409576,"entropy":0.5680313557386398},{"step":381,"reward":-0.44690951704978943,"core":0.02607383392751217,"cost":-0.04546944424510002,"kl":0.3079035058617592,"entropy":0.5780862271785736},{"step":383,"reward":1.325736165046692,"core":0.6940008997917175,"cost":0.0,"kl":0.2397887445986271,"entropy":0.47663426399230957},{"step":385,"reward":0.9080072045326233,"core":0.5612683296203613,"cost":0.0,"kl":0.3204636238515377,"entropy":0.6057685911655426},{"step":387,"reward":0.6981341242790222,"core":0.42912089824676514,"cost":0.0,"kl":0.5270519033074379,"entropy":0.5639167875051498},{"step":389,"reward":0.9520053863525391,"core":0.6014111638069153,"cost":0.0,"kl":0.2630666755139828,"entropy":0.5924474000930786},{"step":391,"reward":0.7601838707923889,"core":0.492313951253891,"cost":-0.017157064750790596,"kl":0.27625299617648125,"entropy":0.4608282893896103},{"step":393,"reward":1.0849488973617554,"core":0.5404102206230164,"cost":0.0,"kl":0.3721473440527916,"entropy":0.5220687538385391},{"step":395,"reward":1.2836360931396484,"core":0.6488522887229919,"cost":0.0,"kl":0.3081237003207207,"entropy":0.5426872223615646},{"step":397,"reward":-0.09709582477807999,"core":0.09827753901481628,"cost":-0.05302171781659126,"kl":0.37181877344846725,"entropy":0.5588063895702362},{"step":399,"reward":0.8084077835083008,"core":0.49061307311058044,"cost":-0.007525115739554167,"kl":0.34329431504011154,"entropy":0.5287556275725365},{"step":401,"reward":1.169155240058899,"core":0.6040971279144287,"cost":0.0,"kl":0.5805634111166,"entropy":0.5241172388195992},{"step":403,"reward":1.1926227807998657,"core":0.6283921003341675,"cost":0.0,"kl":0.27131230384111404,"entropy":0.5141308158636093},{"step":405,"reward":0.5260272026062012,"core":0.3669957220554352,"cost":0.0,"kl":0.28992053121328354,"entropy":0.527046836912632},{"step":407,"reward":1.0109957456588745,"core":0.5009855628013611,"cost":0.0,"kl":0.34148214384913445,"entropy":0.5837777704000473},{"step":409,"reward":1.0150569677352905,"core":0.5445742011070251,"cost":-0.0053800600580871105,"kl":0.2727414034307003,"entropy":0.5029585286974907},{"step":411,"reward":0.8872436881065369,"core":0.5410192012786865,"cost":-0.003572238376364112,"kl":0.3044568747282028,"entropy":0.5061303898692131},{"step":413,"reward":0.48179617524147034,"core":0.37252604961395264,"cost":0.0,"kl":0.30760103464126587,"entropy":0.5651472955942154},{"step":415,"reward":0.6833033561706543,"core":0.4733560085296631,"cost":-0.01786041259765625,"kl":0.2739104591310024,"entropy":0.5177177265286446},{"step":417,"reward":0.7908009886741638,"core":0.4593733847141266,"cost":0.0,"kl":0.27982765063643456,"entropy":0.5260056182742119},{"step":419,"reward":0.7159046530723572,"core":0.4882480800151825,"cost":-0.031727537512779236,"kl":0.27239157259464264,"entropy":0.6105741709470749},{"step":421,"reward":0.9504430890083313,"core":0.5579173564910889,"cost":-0.019192948937416077,"kl":0.31526054069399834,"entropy":0.5464381724596024},{"step":423,"reward":0.7023183703422546,"core":0.4807220697402954,"cost":-0.014519392512738705,"kl":0.3732827976346016,"entropy":0.5506995022296906},{"step":425,"reward":1.010353684425354,"core":0.5649416446685791,"cost":-0.0063700564205646515,"kl":0.313513919711113,"entropy":0.5792135000228882},{"step":427,"reward":0.6485640406608582,"core":0.419413298368454,"cost":-0.012427921406924725,"kl":0.30568603053689003,"entropy":0.5623880624771118},{"step":429,"reward":1.1416829824447632,"core":0.6828524470329285,"cost":-0.00839098822325468,"kl":0.2545703500509262,"entropy":0.5674692392349243},{"step":431,"reward":1.1185083389282227,"core":0.5699784159660339,"cost":-0.004709395579993725,"kl":0.32653023302555084,"entropy":0.54932451993227},{"step":433,"reward":0.5565926432609558,"core":0.458750456571579,"cost":-0.040342554450035095,"kl":0.27239789068698883,"entropy":0.6122152209281921},{"step":435,"reward":0.4636670649051666,"core":0.3786633014678955,"cost":0.0,"kl":0.3981967382133007,"entropy":0.6242439299821854},{"step":437,"reward":1.204909086227417,"core":0.7394906878471375,"cost":0.0,"kl":0.38025758415460587,"entropy":0.5429579317569733},{"step":439,"reward":0.41455724835395813,"core":0.28031203150749207,"cost":-0.0023616987746208906,"kl":0.26748254522681236,"entropy":0.5943206995725632},{"step":441,"reward":0.30334681272506714,"core":0.2994392216205597,"cost":-0.023535283282399178,"kl":0.4317474886775017,"entropy":0.609930157661438},{"step":443,"reward":0.5128584504127502,"core":0.36675381660461426,"cost":-0.00628190441057086,"kl":0.32807738333940506,"entropy":0.5747255086898804},{"step":445,"reward":1.1204184293746948,"core":0.6410057544708252,"cost":-0.0123568931594491,"kl":0.38851457089185715,"entropy":0.5192573070526123},{"step":447,"reward":1.0066806077957153,"core":0.6543436646461487,"cost":0.0,"kl":0.331197164952755,"entropy":0.46614470332860947},{"step":449,"reward":1.1633682250976562,"core":0.6686846613883972,"cost":0.0,"kl":0.3146367333829403,"entropy":0.5059167370200157},{"step":451,"reward":0.8537155985832214,"core":0.5775654315948486,"cost":-0.023047350347042084,"kl":0.3077279329299927,"entropy":0.5645270496606827},{"step":453,"reward":1.206703543663025,"core":0.6240897178649902,"cost":0.0,"kl":0.353179894387722,"entropy":0.46624670922756195},{"step":455,"reward":0.8640467524528503,"core":0.5161322951316833,"cost":-0.00533335143700242,"kl":0.28320011124014854,"entropy":0.5462214350700378},{"step":457,"reward":0.6320862174034119,"core":0.49737560749053955,"cost":0.0,"kl":0.35793904960155487,"entropy":0.6014837324619293},{"step":459,"reward":0.8786776661872864,"core":0.5275277495384216,"cost":0.0,"kl":0.40151602774858475,"entropy":0.580070823431015},{"step":461,"reward":0.8008365631103516,"core":0.46772393584251404,"cost":0.0,"kl":0.408860020339489,"entropy":0.6068852841854095},{"step":463,"reward":1.5369328260421753,"core":0.7683960795402527,"cost":0.0,"kl":0.3729184418916702,"entropy":0.5266881883144379},{"step":465,"reward":0.5002332925796509,"core":0.38445401191711426,"cost":0.0,"kl":0.3689512312412262,"entropy":0.6024594753980637},{"step":467,"reward":0.6760122179985046,"core":0.4219905436038971,"cost":0.0,"kl":0.45278724282979965,"entropy":0.527717262506485},{"step":469,"reward":0.9710877537727356,"core":0.5776516199111938,"cost":-0.03087085671722889,"kl":0.38944822549819946,"entropy":0.558232381939888},{"step":471,"reward":0.632450520992279,"core":0.3749559819698334,"cost":-0.00020273844711482525,"kl":0.38088615983724594,"entropy":0.5524518191814423},{"step":473,"reward":0.10968092828989029,"core":0.1883496791124344,"cost":-0.03173400089144707,"kl":0.3314458727836609,"entropy":0.5670554339885712},{"step":475,"reward":1.0208630561828613,"core":0.5182290077209473,"cost":-0.01538961660116911,"kl":0.2581876330077648,"entropy":0.55340176820755},{"step":477,"reward":0.6962170004844666,"core":0.4445413053035736,"cost":0.0,"kl":0.3795599713921547,"entropy":0.5777910649776459},{"step":479,"reward":0.2919187843799591,"core":0.3518693447113037,"cost":0.0,"kl":0.33120086789131165,"entropy":0.6526640057563782},{"step":481,"reward":0.7715066075325012,"core":0.46646562218666077,"cost":0.0,"kl":0.32136377692222595,"entropy":0.5857667028903961},{"step":483,"reward":0.8578241467475891,"core":0.5364983677864075,"cost":0.0,"kl":0.24997689202427864,"entropy":0.5995170474052429},{"step":485,"reward":0.32447049021720886,"core":0.3740030527114868,"cost":-0.03999767079949379,"kl":0.32475272566080093,"entropy":0.6125213950872421},{"step":487,"reward":1.030070185661316,"core":0.49211540818214417,"cost":-0.023476295173168182,"kl":0.35487286001443863,"entropy":0.6822674572467804},{"step":489,"reward":0.9360649585723877,"core":0.5154281258583069,"cost":0.0,"kl":0.3697996363043785,"entropy":0.6064621061086655},{"step":491,"reward":0.8154528737068176,"core":0.5387352108955383,"cost":0.0,"kl":0.3809535801410675,"entropy":0.6694158911705017},{"step":493,"reward":0.7810311913490295,"core":0.4515460431575775,"cost":-0.0038495471235364676,"kl":0.3496114984154701,"entropy":0.5137756243348122},{"step":495,"reward":0.49567365646362305,"core":0.35605761408805847,"cost":-0.0014764908701181412,"kl":0.263959888368845,"entropy":0.6518794745206833},{"step":497,"reward":0.8438921570777893,"core":0.49955543875694275,"cost":0.0,"kl":0.29359984025359154,"entropy":0.599385991692543},{"step":499,"reward":1.2252548933029175,"core":0.6579992771148682,"cost":-0.01697855442762375,"kl":0.31839393824338913,"entropy":0.6147865504026413},{"step":501,"reward":1.04228937625885,"core":0.5516061782836914,"cost":0.0,"kl":0.3217949867248535,"entropy":0.6314707845449448},{"step":503,"reward":0.6613768935203552,"core":0.47093522548675537,"cost":-0.022021545097231865,"kl":0.28246451169252396,"entropy":0.5861694514751434},{"step":505,"reward":0.6167228817939758,"core":0.4007514417171478,"cost":-0.023938171565532684,"kl":0.2502124719321728,"entropy":0.5793996602296829},{"step":507,"reward":0.7471778988838196,"core":0.41817232966423035,"cost":-0.01776685193181038,"kl":0.2587623856961727,"entropy":0.5442293435335159},{"step":509,"reward":0.894069492816925,"core":0.4943939745426178,"cost":0.0,"kl":0.2805544100701809,"entropy":0.6269983649253845},{"step":511,"reward":1.4715529680252075,"core":0.6835822463035583,"cost":0.0,"kl":0.28235629573464394,"entropy":0.5687315762042999},{"step":513,"reward":0.944364070892334,"core":0.5260066986083984,"cost":0.0,"kl":0.2419198751449585,"entropy":0.49844300001859665},{"step":515,"reward":0.14619937539100647,"core":0.20308096706867218,"cost":0.0,"kl":0.28321753069758415,"entropy":0.6185986548662186},{"step":517,"reward":0.7213724255561829,"core":0.4176967442035675,"cost":0.0,"kl":0.31602899730205536,"entropy":0.5630037859082222},{"step":519,"reward":0.6746513843536377,"core":0.4588046073913574,"cost":0.0,"kl":0.2838115580379963,"entropy":0.6144398301839828},{"step":521,"reward":0.7961335182189941,"core":0.5091390609741211,"cost":-0.03453252464532852,"kl":0.29095274955034256,"entropy":0.5422427132725716},{"step":523,"reward":0.9530035853385925,"core":0.5727759003639221,"cost":-0.005608778912574053,"kl":0.31320740655064583,"entropy":0.6341570764780045},{"step":525,"reward":0.2997535467147827,"core":0.28255516290664673,"cost":0.0,"kl":0.2848989740014076,"entropy":0.6057300269603729},{"step":527,"reward":0.7380674481391907,"core":0.4972686469554901,"cost":0.0,"kl":0.20659782737493515,"entropy":0.629159152507782},{"step":529,"reward":1.0301283597946167,"core":0.5308151245117188,"cost":0.0,"kl":0.31258054077625275,"entropy":0.5031407102942467},{"step":531,"reward":0.7778204083442688,"core":0.44178667664527893,"cost":0.0,"kl":0.30078500881791115,"entropy":0.5649334788322449},{"step":533,"reward":0.751436710357666,"core":0.4229503571987152,"cost":-0.0065801371820271015,"kl":0.310397494584322,"entropy":0.5968146324157715},{"step":535,"reward":1.3240690231323242,"core":0.747029721736908,"cost":0.0,"kl":0.2671121433377266,"entropy":0.4381319135427475},{"step":537,"reward":0.8775558471679688,"core":0.5070423483848572,"cost":0.0,"kl":0.31901058554649353,"entropy":0.5513936281204224},{"step":539,"reward":1.051690936088562,"core":0.588684618473053,"cost":0.0,"kl":0.32011839747428894,"entropy":0.547961637377739},{"step":541,"reward":1.2347699403762817,"core":0.6150367856025696,"cost":-0.0037185519468039274,"kl":0.2854899652302265,"entropy":0.5932750403881073},{"step":543,"reward":1.1035523414611816,"core":0.6089743375778198,"cost":-0.008815793320536613,"kl":0.2766708545386791,"entropy":0.5062209516763687},{"step":545,"reward":1.0148485898971558,"core":0.523501992225647,"cost":0.0,"kl":0.3518140837550163,"entropy":0.5675897747278214},{"step":547,"reward":0.4721479117870331,"core":0.3816104829311371,"cost":0.0,"kl":0.32484646141529083,"entropy":0.5409500524401665},{"step":549,"reward":1.3459892272949219,"core":0.7008482813835144,"cost":0.0,"kl":0.32044998556375504,"entropy":0.553699254989624},{"step":551,"reward":0.6658455729484558,"core":0.4277435839176178,"cost":0.0,"kl":0.29554491862654686,"entropy":0.5854099690914154},{"step":553,"reward":1.1559094190597534,"core":0.6257285475730896,"cost":0.0,"kl":0.29379433393478394,"entropy":0.554001122713089},{"step":555,"reward":1.003505825996399,"core":0.6533286571502686,"cost":-0.01115157175809145,"kl":0.29751142114400864,"entropy":0.4709283113479614},{"step":557,"reward":0.6773473620414734,"core":0.4519578218460083,"cost":0.0,"kl":0.3179576024413109,"entropy":0.5953217893838882},{"step":559,"reward":0.4568847119808197,"core":0.3858775794506073,"cost":0.0,"kl":0.320782795548439,"entropy":0.6073287278413773},{"step":561,"reward":0.8976395726203918,"core":0.5549187064170837,"cost":-0.013081535696983337,"kl":0.27805450558662415,"entropy":0.6029775142669678},{"step":563,"reward":0.7245814800262451,"core":0.5051898956298828,"cost":0.0,"kl":0.3368091657757759,"entropy":0.5590972751379013},{"step":565,"reward":0.7070464491844177,"core":0.42238959670066833,"cost":0.0,"kl":0.2755458727478981,"entropy":0.5371029227972031},{"step":567,"reward":0.6194255948066711,"core":0.36033669114112854,"cost":0.0,"kl":0.2871387377381325,"entropy":0.5640816688537598},{"step":569,"reward":1.086295485496521,"core":0.5513240694999695,"cost":0.0,"kl":0.21528701856732368,"entropy":0.5972465425729752},{"step":571,"reward":0.9011291861534119,"core":0.5573824048042297,"cost":0.0,"kl":0.28486986085772514,"entropy":0.5724683851003647},{"step":573,"reward":0.5718005299568176,"core":0.3047438859939575,"cost":-0.013248272240161896,"kl":0.32815614342689514,"entropy":0.5469019114971161},{"step":575,"reward":0.8462378978729248,"core":0.5243231654167175,"cost":0.0,"kl":0.24892796576023102,"entropy":0.6004865020513535},{"step":577,"reward":0.5073685646057129,"core":0.35997316241264343,"cost":0.0,"kl":0.2611565813422203,"entropy":0.6118511706590652},{"step":579,"reward":1.0569602251052856,"core":0.5584565997123718,"cost":-0.004478922579437494,"kl":0.2615112476050854,"entropy":0.6346688866615295},{"step":581,"reward":-0.13277626037597656,"core":0.13006728887557983,"cost":-0.0013126482954248786,"kl":0.3412647843360901,"entropy":0.5900672525167465},{"step":583,"reward":-0.1513940840959549,"core":0.13165336847305298,"cost":0.0,"kl":0.2725324146449566,"entropy":0.6198172718286514},{"step":585,"reward":1.3973231315612793,"core":0.7835467457771301,"cost":0.0,"kl":0.2382512018084526,"entropy":0.5123680308461189},{"step":587,"reward":1.0521982908248901,"core":0.5258621573448181,"cost":-0.04034663736820221,"kl":0.30781181156635284,"entropy":0.5275011956691742},{"step":589,"reward":0.57734614610672,"core":0.42930594086647034,"cost":-0.011032634414732456,"kl":0.29090503603219986,"entropy":0.5977060943841934},{"step":591,"reward":1.118436574935913,"core":0.703167736530304,"cost":0.0,"kl":0.3748152479529381,"entropy":0.500341571867466},{"step":593,"reward":0.4048748016357422,"core":0.33804476261138916,"cost":0.0,"kl":0.5956773534417152,"entropy":0.525589182972908},{"step":595,"reward":1.1579899787902832,"core":0.7075446248054504,"cost":0.0,"kl":0.2766822576522827,"entropy":0.5930992662906647},{"step":597,"reward":0.11721355468034744,"core":0.2323039025068283,"cost":-0.009004469029605389,"kl":0.2649013586342335,"entropy":0.5878530889749527},{"step":599,"reward":0.6465420722961426,"core":0.46044477820396423,"cost":-0.004660698119550943,"kl":0.39638974517583847,"entropy":0.5923870205879211}],"nonzero_advantage":[{"step":1,"nonzero":0.2916666666666667},{"step":3,"nonzero":0.6041666666666666},{"step":5,"nonzero":0.5},{"step":7,"nonzero":0.7916666666666666},{"step":9,"nonzero":0.6458333333333334},{"step":11,"nonzero":0.5625},{"step":13,"nonzero":0.6458333333333334},{"step":15,"nonzero":0.4791666666666667},{"step":17,"nonzero":0.6458333333333334},{"step":19,"nonzero":0.6666666666666666},{"step":21,"nonzero":0.5208333333333334},{"step":23,"nonzero":0.5416666666666666},{"step":25,"nonzero":0.3333333333333333},{"step":27,"nonzero":0.3125},{"step":29,"nonzero":0.6875},{"step":31,"nonzero":0.4791666666666667},{"step":33,"nonzero":0.625},{"step":35,"nonzero":0.7291666666666666},{"step":37,"nonzero":0.5833333333333334},{"step":39,"nonzero":0.6458333333333334},{"step":41,"nonzero":0.4375},{"step":43,"nonzero":0.875},{"step":45,"nonzero":0.7708333333333334},{"step":47,"nonzero":0.5833333333333334},{"step":49,"nonzero":0.7083333333333334},{"step":51,"nonzero":0.5625},{"step":53,"nonzero":0.8541666666666666},{"step":55,"nonzero":0.8125},{"step":57,"nonzero":0.8125},{"step":59,"nonzero":0.9166666666666666},{"step":61,"nonzero":0.7291666666666666},{"step":63,"nonzero":0.7916666666666666},{"step":65,"nonzero":0.5},{"step":67,"nonzero":0.5833333333333334},{"step":69,"nonzero":0.6041666666666666},{"step":71,"nonzero":0.8333333333333334},{"step":73,"nonzero":0.7708333333333334},{"step":75,"nonzero":0.9791666666666666},{"step":77,"nonzero":0.6041666666666666},{"step":79,"nonzero":0.8541666666666666},{"step":81,"nonzero":0.9375},{"step":83,"nonzero":0.7916666666666666},{"step":85,"nonzero":0.75},{"step":87,"nonzero":0.8125},{"step":89,"nonzero":0.6666666666666666},{"step":91,"nonzero":0.9166666666666666},{"step":93,"nonzero":0.8541666666666666},{"step":95,"nonzero":0.4583333333333333},{"step":97,"nonzero":0.8958333333333334},{"step":99,"nonzero":0.8958333333333334},{"step":101,"nonzero":0.7708333333333334},{"step":103,"nonzero":0.8541666666666666},{"step":105,"nonzero":0.6875},{"step":107,"nonzero":0.9375},{"step":109,"nonzero":1.0},{"step":111,"nonzero":0.8125},{"step":113,"nonzero":0.7291666666666666},{"step":115,"nonzero":0.8333333333333334},{"step":117,"nonzero":0.9583333333333334},{"step":119,"nonzero":0.875},{"step":121,"nonzero":1.0},{"step":123,"nonzero":0.9166666666666666},{"step":125,"nonzero":0.9583333333333334},{"step":127,"nonzero":0.5208333333333334},{"step":129,"nonzero":0.8333333333333334},{"step":131,"nonzero":0.5208333333333334},{"step":133,"nonzero":0.75},{"step":135,"nonzero":0.9375},{"step":137,"nonzero":0.8541666666666666},{"step":139,"nonzero":0.875},{"step":141,"nonzero":1.0},{"step":143,"nonzero":0.8125},{"step":145,"nonzero":0.9166666666666666},{"step":147,"nonzero":0.9166666666666666},{"step":149,"nonzero":0.8958333333333334},{"step":151,"nonzero":0.7083333333333334},{"step":153,"nonzero":0.5833333333333334},{"step":155,"nonzero":0.7708333333333334},{"step":157,"nonzero":1.0},{"step":159,"nonzero":1.0},{"step":161,"nonzero":0.6458333333333334},{"step":163,"nonzero":1.0},{"step":165,"nonzero":0.5625},{"step":167,"nonzero":0.8958333333333334},{"step":169,"nonzero":0.9375},{"step":171,"nonzero":0.9791666666666666},{"step":173,"nonzero":0.9583333333333334},{"step":175,"nonzero":0.9375},{"step":177,"nonzero":0.7083333333333334},{"step":179,"nonzero":0.9583333333333334},{"step":181,"nonzero":0.9375},{"step":183,"nonzero":0.9166666666666666},{"step":185,"nonzero":0.9375},{"step":187,"nonzero":0.8541666666666666},{"step":189,"nonzero":0.75},{"step":191,"nonzero":0.8125},{"step":193,"nonzero":1.0},{"step":195,"nonzero":0.8125},{"step":197,"nonzero":0.8958333333333334},{"step":199,"nonzero":0.9166666666666666},{"step":201,"nonzero":0.8125},{"step":203,"nonzero":0.9583333333333334},{"step":205,"nonzero":0.9375},{"step":207,"nonzero":0.9583333333333334},{"step":209,"nonzero":0.75},{"step":211,"nonzero":0.75},{"step":213,"nonzero":0.875},{"step":215,"nonzero":0.7916666666666666},{"step":217,"nonzero":0.8958333333333334},{"step":219,"nonzero":0.9375},{"step":221,"nonzero":0.8333333333333334},{"step":223,"nonzero":1.0},{"step":225,"nonzero":0.9583333333333334},{"step":227,"nonzero":0.9791666666666666},{"step":229,"nonzero":0.9583333333333334},{"step":231,"nonzero":0.875},{"step":233,"nonzero":0.8541666666666666},{"step":235,"nonzero":0.9791666666666666},{"step":237,"nonzero":0.9791666666666666},{"step":239,"nonzero":0.875},{"step":241,"nonzero":0.8958333333333334},{"step":243,"nonzero":0.875},{"step":245,"nonzero":0.9583333333333334},{"step":247,"nonzero":0.8541666666666666},{"step":249,"nonzero":0.8958333333333334},{"step":251,"nonzero":0.9166666666666666},{"step":253,"nonzero":0.9791666666666666},{"step":255,"nonzero":0.9791666666666666},{"step":257,"nonzero":0.7708333333333334},{"step":259,"nonzero":0.8541666666666666},{"step":261,"nonzero":0.9583333333333334},{"step":263,"nonzero":0.75},{"step":265,"nonzero":0.8541666666666666},{"step":267,"nonzero":0.8958333333333334},{"step":269,"nonzero":1.0},{"step":271,"nonzero":0.9375},{"step":273,"nonzero":0.9375},{"step":275,"nonzero":0.9166666666666666},{"step":277,"nonzero":1.0},{"step":279,"nonzero":1.0},{"step":281,"nonzero":1.0},{"step":283,"nonzero":0.9583333333333334},{"step":285,"nonzero":0.9583333333333334},{"step":287,"nonzero":0.9791666666666666},{"step":289,"nonzero":0.9375},{"step":291,"nonzero":0.9375},{"step":293,"nonzero":1.0},{"step":295,"nonzero":0.9375},{"step":297,"nonzero":0.9583333333333334},{"step":299,"nonzero":0.9166666666666666},{"step":301,"nonzero":0.875},{"step":303,"nonzero":0.9791666666666666},{"step":305,"nonzero":1.0},{"step":307,"nonzero":1.0},{"step":309,"nonzero":1.0},{"step":311,"nonzero":0.9583333333333334},{"step":313,"nonzero":1.0},{"step":315,"nonzero":1.0},{"step":317,"nonzero":0.9791666666666666},{"step":319,"nonzero":1.0},{"step":321,"nonzero":0.9791666666666666},{"step":323,"nonzero":1.0},{"step":325,"nonzero":0.9375},{"step":327,"nonzero":1.0},{"step":329,"nonzero":0.9166666666666666},{"step":331,"nonzero":1.0},{"step":333,"nonzero":0.9791666666666666},{"step":335,"nonzero":0.9166666666666666},{"step":337,"nonzero":0.9791666666666666},{"step":339,"nonzero":1.0},{"step":341,"nonzero":0.9791666666666666},{"step":343,"nonzero":1.0},{"step":345,"nonzero":0.9791666666666666},{"step":347,"nonzero":1.0},{"step":349,"nonzero":1.0},{"step":351,"nonzero":0.9791666666666666},{"step":353,"nonzero":0.9583333333333334},{"step":355,"nonzero":1.0},{"step":357,"nonzero":1.0},{"step":359,"nonzero":1.0},{"step":361,"nonzero":0.9791666666666666},{"step":363,"nonzero":1.0},{"step":365,"nonzero":0.9791666666666666},{"step":367,"nonzero":1.0},{"step":369,"nonzero":0.9791666666666666},{"step":371,"nonzero":0.9791666666666666},{"step":373,"nonzero":0.9791666666666666},{"step":375,"nonzero":1.0},{"step":377,"nonzero":0.9583333333333334},{"step":379,"nonzero":0.9375},{"step":381,"nonzero":0.9375},{"step":383,"nonzero":1.0},{"step":385,"nonzero":0.9791666666666666},{"step":387,"nonzero":0.9791666666666666},{"step":389,"nonzero":1.0},{"step":391,"nonzero":1.0},{"step":393,"nonzero":1.0},{"step":395,"nonzero":1.0},{"step":397,"nonzero":0.8958333333333334},{"step":399,"nonzero":0.9791666666666666},{"step":401,"nonzero":1.0},{"step":403,"nonzero":1.0},{"step":405,"nonzero":1.0},{"step":407,"nonzero":1.0},{"step":409,"nonzero":0.9791666666666666},{"step":411,"nonzero":0.9583333333333334},{"step":413,"nonzero":1.0},{"step":415,"nonzero":0.9791666666666666},{"step":417,"nonzero":1.0},{"step":419,"nonzero":0.9375},{"step":421,"nonzero":1.0},{"step":423,"nonzero":0.9791666666666666},{"step":425,"nonzero":1.0},{"step":427,"nonzero":0.9583333333333334},{"step":429,"nonzero":1.0},{"step":431,"nonzero":1.0},{"step":433,"nonzero":0.8958333333333334},{"step":435,"nonzero":0.9583333333333334},{"step":437,"nonzero":1.0},{"step":439,"nonzero":0.8958333333333334},{"step":441,"nonzero":0.9583333333333334},{"step":443,"nonzero":1.0},{"step":445,"nonzero":1.0},{"step":447,"nonzero":0.9791666666666666},{"step":449,"nonzero":1.0},{"step":451,"nonzero":0.9791666666666666},{"step":453,"nonzero":1.0},{"step":455,"nonzero":1.0},{"step":457,"nonzero":1.0},{"step":459,"nonzero":0.9791666666666666},{"step":461,"nonzero":0.9583333333333334},{"step":463,"nonzero":1.0},{"step":465,"nonzero":0.9791666666666666},{"step":467,"nonzero":1.0},{"step":469,"nonzero":1.0},{"step":471,"nonzero":0.9375},{"step":473,"nonzero":0.9791666666666666},{"step":475,"nonzero":1.0},{"step":477,"nonzero":0.9791666666666666},{"step":479,"nonzero":1.0},{"step":481,"nonzero":1.0},{"step":483,"nonzero":1.0},{"step":485,"nonzero":0.9583333333333334},{"step":487,"nonzero":1.0},{"step":489,"nonzero":1.0},{"step":491,"nonzero":1.0},{"step":493,"nonzero":0.9791666666666666},{"step":495,"nonzero":1.0},{"step":497,"nonzero":0.9791666666666666},{"step":499,"nonzero":1.0},{"step":501,"nonzero":1.0},{"step":503,"nonzero":1.0},{"step":505,"nonzero":0.9375},{"step":507,"nonzero":0.9791666666666666},{"step":509,"nonzero":1.0},{"step":511,"nonzero":1.0},{"step":513,"nonzero":0.9375},{"step":515,"nonzero":0.8958333333333334},{"step":517,"nonzero":0.9791666666666666},{"step":519,"nonzero":0.9791666666666666},{"step":521,"nonzero":0.9791666666666666},{"step":523,"nonzero":1.0},{"step":525,"nonzero":0.9583333333333334},{"step":527,"nonzero":0.9791666666666666},{"step":529,"nonzero":1.0},{"step":531,"nonzero":1.0},{"step":533,"nonzero":0.9583333333333334},{"step":535,"nonzero":1.0},{"step":537,"nonzero":1.0},{"step":539,"nonzero":1.0},{"step":541,"nonzero":0.9791666666666666},{"step":543,"nonzero":0.9375},{"step":545,"nonzero":1.0},{"step":547,"nonzero":0.9583333333333334},{"step":549,"nonzero":1.0},{"step":551,"nonzero":0.9791666666666666},{"step":553,"nonzero":1.0},{"step":555,"nonzero":1.0},{"step":557,"nonzero":1.0},{"step":559,"nonzero":0.9583333333333334},{"step":561,"nonzero":1.0},{"step":563,"nonzero":0.9791666666666666},{"step":565,"nonzero":0.9166666666666666},{"step":567,"nonzero":0.9583333333333334},{"step":569,"nonzero":1.0},{"step":571,"nonzero":1.0},{"step":573,"nonzero":0.9791666666666666},{"step":575,"nonzero":1.0},{"step":577,"nonzero":0.9791666666666666},{"step":579,"nonzero":1.0},{"step":581,"nonzero":0.9375},{"step":583,"nonzero":0.9791666666666666},{"step":585,"nonzero":1.0},{"step":587,"nonzero":1.0},{"step":589,"nonzero":0.8958333333333334},{"step":591,"nonzero":1.0},{"step":593,"nonzero":1.0},{"step":595,"nonzero":1.0},{"step":597,"nonzero":0.9166666666666666},{"step":599,"nonzero":0.9791666666666666}]},{"id":"dapo-repor-dora-226","label":"DAPO + REPO-R \u00b7 DoRA \u00b7 226 prompts","run_ids":["1790973987"],"loss":"DAPO","adapter":"DoRA","prompts":226,"repo_r":true,"holdout_by_checkpoint":[{"step":0,"holdout":-1.3664,"train_core":-0.9836,"valid_rate":0.9},{"step":100,"holdout":-0.2873,"train_core":-0.2047,"valid_rate":0.92},{"step":200,"holdout":-0.1622,"train_core":-0.1855,"valid_rate":0.96},{"step":300,"holdout":0.4806,"train_core":0.2142,"valid_rate":0.94},{"step":400,"holdout":0.7787,"train_core":0.3298,"valid_rate":0.96},{"step":500,"holdout":0.6939,"train_core":0.3305,"valid_rate":0.94},{"step":600,"holdout":0.6802,"train_core":0.3274,"valid_rate":0.96}],"selected_step":500,"selected_holdout":0.6939,"train_steps":[{"step":1,"reward":-2.015054941177368,"core":-0.6128399968147278,"cost":-0.08141078054904938,"kl":0.0,"entropy":0.5671490281820297},{"step":3,"reward":-2.4309113025665283,"core":-0.7745025753974915,"cost":-0.07165638357400894,"kl":0.0008887773874448612,"entropy":0.5407039895653725},{"step":5,"reward":-2.420438289642334,"core":-0.7494697570800781,"cost":-0.12121579051017761,"kl":0.0018945227493532002,"entropy":0.5742806047201157},{"step":7,"reward":-1.2469230890274048,"core":-0.36142924427986145,"cost":-0.04351918399333954,"kl":0.0032495628693141043,"entropy":0.5141580551862717},{"step":9,"reward":-1.3628921508789062,"core":-0.4184962213039398,"cost":-0.06166806444525719,"kl":0.004791218612808734,"entropy":0.5361299067735672},{"step":11,"reward":-2.233140707015991,"core":-0.669661283493042,"cost":-0.15129408240318298,"kl":0.008631667704321444,"entropy":0.5188380032777786},{"step":13,"reward":-1.0998810529708862,"core":-0.3325134813785553,"cost":-0.0011122633004561067,"kl":0.01388353225775063,"entropy":0.556360512971878},{"step":15,"reward":-1.629839301109314,"core":-0.392106294631958,"cost":-0.1361345797777176,"kl":0.018245964078232646,"entropy":0.5139903947710991},{"step":17,"reward":-1.3358917236328125,"core":-0.37999799847602844,"cost":0.0,"kl":0.025469631887972355,"entropy":0.5781601071357727},{"step":19,"reward":-1.6132112741470337,"core":-0.462197870016098,"cost":-0.061958760023117065,"kl":0.048605759628117085,"entropy":0.6715573519468307},{"step":21,"reward":-1.5853019952774048,"core":-0.4496726095676422,"cost":-0.023417778313159943,"kl":0.05036022048443556,"entropy":0.600068986415863},{"step":23,"reward":-1.6666332483291626,"core":-0.4960083067417145,"cost":-0.03003259189426899,"kl":0.08044045325368643,"entropy":0.6149289608001709},{"step":25,"reward":-2.1375999450683594,"core":-0.5687134861946106,"cost":-0.18074287474155426,"kl":0.07035506330430508,"entropy":0.4932536631822586},{"step":27,"reward":-2.1901440620422363,"core":-0.6222714185714722,"cost":-0.12894736230373383,"kl":0.09794531390070915,"entropy":0.5935398787260056},{"step":29,"reward":-1.221051812171936,"core":-0.33011147379875183,"cost":-0.004948537331074476,"kl":0.13016856461763382,"entropy":0.5981479436159134},{"step":31,"reward":-1.0894478559494019,"core":-0.26000070571899414,"cost":-0.06703975796699524,"kl":0.12765195593237877,"entropy":0.6282693296670914},{"step":33,"reward":-0.5611335635185242,"core":-0.08898985385894775,"cost":-0.006535134743899107,"kl":0.14254169911146164,"entropy":0.5814045965671539},{"step":35,"reward":-0.46308937668800354,"core":-0.09127864986658096,"cost":-0.021544568240642548,"kl":0.12747525423765182,"entropy":0.5187646523118019},{"step":37,"reward":-0.7895233035087585,"core":-0.2082209587097168,"cost":-0.025735830888152122,"kl":0.15702706947922707,"entropy":0.5032849237322807},{"step":39,"reward":-0.7149341702461243,"core":-0.13107578456401825,"cost":-0.0647859051823616,"kl":0.11584794707596302,"entropy":0.39229776710271835},{"step":41,"reward":-1.0701147317886353,"core":-0.28626444935798645,"cost":-0.03534899652004242,"kl":0.1441393867135048,"entropy":0.4517723619937897},{"step":43,"reward":-0.7061713337898254,"core":-0.16179607808589935,"cost":0.0,"kl":0.11618407443165779,"entropy":0.49624431878328323},{"step":45,"reward":-1.000209927558899,"core":-0.21715675294399261,"cost":-0.03466835990548134,"kl":0.16995708271861076,"entropy":0.5036064460873604},{"step":47,"reward":-0.4700113832950592,"core":-0.028272995725274086,"cost":-0.029593028128147125,"kl":0.16169740445911884,"entropy":0.4927271381020546},{"step":49,"reward":-0.6405765414237976,"core":-0.1258041113615036,"cost":-0.0315011627972126,"kl":0.1614413857460022,"entropy":0.5137585997581482},{"step":51,"reward":-0.42004236578941345,"core":0.0514226071536541,"cost":-0.07682125270366669,"kl":0.15777356550097466,"entropy":0.5102220550179482},{"step":53,"reward":-1.2732783555984497,"core":-0.296555757522583,"cost":-0.0042713419534265995,"kl":0.11626731790602207,"entropy":0.6119256466627121},{"step":55,"reward":-0.8317370414733887,"core":-0.1572428196668625,"cost":-0.05968150123953819,"kl":0.11984713189303875,"entropy":0.49465084075927734},{"step":57,"reward":-0.07335226982831955,"core":0.09749099612236023,"cost":-0.0035174123477190733,"kl":0.17140435613691807,"entropy":0.5146345496177673},{"step":59,"reward":-0.678074061870575,"core":-0.03358258679509163,"cost":-0.05829824134707451,"kl":0.09731880761682987,"entropy":0.558644637465477},{"step":61,"reward":-1.097669005393982,"core":-0.28991493582725525,"cost":-0.021702654659748077,"kl":0.13484898954629898,"entropy":0.6041399985551834},{"step":63,"reward":-1.0037075281143188,"core":-0.19445450603961945,"cost":-0.04386594519019127,"kl":0.11188561841845512,"entropy":0.6576078236103058},{"step":65,"reward":-1.4372581243515015,"core":-0.3710278272628784,"cost":-0.052928805351257324,"kl":0.1456432081758976,"entropy":0.5081198886036873},{"step":67,"reward":-1.281166672706604,"core":-0.35619059205055237,"cost":-0.10131531953811646,"kl":0.1333625242114067,"entropy":0.6838267892599106},{"step":69,"reward":-1.2085613012313843,"core":-0.31321391463279724,"cost":-0.004252341575920582,"kl":0.07383988238871098,"entropy":0.5825652033090591},{"step":71,"reward":-0.18938545882701874,"core":0.04996269568800926,"cost":-0.00998871959745884,"kl":0.10147163085639477,"entropy":0.6349270939826965},{"step":73,"reward":-1.0737642049789429,"core":-0.28233376145362854,"cost":0.0,"kl":0.09747044742107391,"entropy":0.5095837339758873},{"step":75,"reward":-0.246668741106987,"core":0.024593735113739967,"cost":-0.022607704624533653,"kl":0.13679983466863632,"entropy":0.5373866111040115},{"step":77,"reward":-0.6374698281288147,"core":-0.09885131567716599,"cost":-0.033736106008291245,"kl":0.10705375298857689,"entropy":0.5897165536880493},{"step":79,"reward":-0.5026220679283142,"core":0.007699776440858841,"cost":-0.04752865806221962,"kl":0.09630076587200165,"entropy":0.4728177338838577},{"step":81,"reward":-0.411927729845047,"core":0.023545444011688232,"cost":-0.008157077245414257,"kl":0.12360720336437225,"entropy":0.5565978586673737},{"step":83,"reward":-0.813744068145752,"core":-0.1965736597776413,"cost":-0.03266768530011177,"kl":0.13113234099000692,"entropy":0.38275624066591263},{"step":85,"reward":0.2270643711090088,"core":0.22320057451725006,"cost":-0.004866262897849083,"kl":0.125841299071908,"entropy":0.47145847976207733},{"step":87,"reward":-0.5434052348136902,"core":-0.053261443972587585,"cost":-0.028479287400841713,"kl":0.10686038061976433,"entropy":0.4765308201313019},{"step":89,"reward":-0.9278824925422668,"core":-0.23328781127929688,"cost":-0.08710932731628418,"kl":0.13708271645009518,"entropy":0.38201675564050674},{"step":91,"reward":-0.8859558701515198,"core":-0.20046788454055786,"cost":-0.011905194260179996,"kl":0.3034412395209074,"entropy":0.4626820981502533},{"step":93,"reward":-0.212286576628685,"core":0.05095163360238075,"cost":-0.01180887222290039,"kl":0.08337939158082008,"entropy":0.4375610277056694},{"step":95,"reward":-0.9743921160697937,"core":-0.20010648667812347,"cost":-0.011480707675218582,"kl":0.13641804456710815,"entropy":0.43513117730617523},{"step":97,"reward":-0.5776755809783936,"core":-0.10749407857656479,"cost":-0.0340801365673542,"kl":0.11031713895499706,"entropy":0.578736200928688},{"step":99,"reward":-0.8654667735099792,"core":-0.1875273436307907,"cost":-0.001396283507347107,"kl":0.11136133037507534,"entropy":0.5591671168804169},{"step":101,"reward":-1.1261084079742432,"core":-0.25208982825279236,"cost":-0.03524671867489815,"kl":0.09977472759783268,"entropy":0.5185535028576851},{"step":103,"reward":-0.5847526788711548,"core":-0.05301281809806824,"cost":-0.03824213147163391,"kl":0.15251831524074078,"entropy":0.49554845690727234},{"step":105,"reward":-1.55059814453125,"core":-0.41235923767089844,"cost":-0.045615676790475845,"kl":0.08633432723581791,"entropy":0.5001069083809853},{"step":107,"reward":0.17026835680007935,"core":0.26923713088035583,"cost":-0.0009290424641221762,"kl":0.13650242425501347,"entropy":0.5191752910614014},{"step":109,"reward":-0.6407411694526672,"core":-0.07048987597227097,"cost":-0.0652596578001976,"kl":0.12842347472906113,"entropy":0.5642099976539612},{"step":111,"reward":-0.14817775785923004,"core":0.025006664916872978,"cost":0.0,"kl":0.1040810588747263,"entropy":0.5093336701393127},{"step":113,"reward":0.022076020017266273,"core":0.13803450763225555,"cost":0.0,"kl":0.10772612690925598,"entropy":0.5977503955364227},{"step":115,"reward":-1.245932936668396,"core":-0.3010871112346649,"cost":-0.02588302455842495,"kl":0.10495076142251492,"entropy":0.5332461446523666},{"step":117,"reward":-0.24107728898525238,"core":0.0379524789750576,"cost":-0.010920938104391098,"kl":0.09576947428286076,"entropy":0.5954747945070267},{"step":119,"reward":-0.4799485206604004,"core":-0.09577640891075134,"cost":-0.006776444148272276,"kl":0.12051918916404247,"entropy":0.6127244085073471},{"step":121,"reward":0.5955566763877869,"core":0.40099430084228516,"cost":-0.034350838512182236,"kl":0.13253672420978546,"entropy":0.5589761137962341},{"step":123,"reward":-0.2874487638473511,"core":0.07639900594949722,"cost":-0.03305435553193092,"kl":0.11149455048143864,"entropy":0.5117927119135857},{"step":125,"reward":-0.7499664425849915,"core":-0.10830572992563248,"cost":-0.018053613603115082,"kl":0.09783795103430748,"entropy":0.5698112845420837},{"step":127,"reward":-1.8054218292236328,"core":-0.48936644196510315,"cost":-0.17638327181339264,"kl":0.12534505128860474,"entropy":0.6586817502975464},{"step":129,"reward":-1.1754881143569946,"core":-0.33507075905799866,"cost":-0.03299563750624657,"kl":0.11905593238770962,"entropy":0.6235408037900925},{"step":131,"reward":-0.624069094657898,"core":-0.12205176800489426,"cost":-0.00391300767660141,"kl":0.13862772658467293,"entropy":0.5252981260418892},{"step":133,"reward":-0.1284628063440323,"core":0.10022672265768051,"cost":0.0,"kl":0.16939633339643478,"entropy":0.6777442842721939},{"step":135,"reward":-0.8139092326164246,"core":-0.22504717111587524,"cost":-0.0013441332848742604,"kl":0.15116997435688972,"entropy":0.6457981020212173},{"step":137,"reward":-0.9289267659187317,"core":-0.20886339247226715,"cost":-0.008683260530233383,"kl":0.11000512354075909,"entropy":0.6047445833683014},{"step":139,"reward":-0.5789910554885864,"core":-0.0686912015080452,"cost":0.0,"kl":0.11432486586272717,"entropy":0.5769813656806946},{"step":141,"reward":-0.8619982600212097,"core":-0.1631992906332016,"cost":0.0,"kl":0.1437238808721304,"entropy":0.5932202786207199},{"step":143,"reward":-0.844987154006958,"core":-0.21430356800556183,"cost":-0.017981253564357758,"kl":0.12632093019783497,"entropy":0.660307988524437},{"step":145,"reward":-0.8923432230949402,"core":-0.13420182466506958,"cost":-0.04131662845611572,"kl":0.11561308801174164,"entropy":0.38655389845371246},{"step":147,"reward":-0.6090938448905945,"core":-0.07897897809743881,"cost":-0.012718964368104935,"kl":0.11327245086431503,"entropy":0.5160593837499619},{"step":149,"reward":-0.6403177380561829,"core":-0.1421964317560196,"cost":-0.009588814340531826,"kl":0.10845607332885265,"entropy":0.49854540824890137},{"step":151,"reward":-0.42349398136138916,"core":-0.002093295333907008,"cost":0.0,"kl":0.12377996370196342,"entropy":0.5613420605659485},{"step":153,"reward":-0.9877298474311829,"core":-0.24872522056102753,"cost":-0.03316211327910423,"kl":0.09901276975870132,"entropy":0.4667551890015602},{"step":155,"reward":-1.2708503007888794,"core":-0.30420616269111633,"cost":-0.08098211884498596,"kl":0.14861171506345272,"entropy":0.503574974834919},{"step":157,"reward":-0.22138746082782745,"core":0.015862420201301575,"cost":-0.0028576208278536797,"kl":0.14092566817998886,"entropy":0.4951530247926712},{"step":159,"reward":-1.4408507347106934,"core":-0.3735835552215576,"cost":-0.054476141929626465,"kl":0.13175483234226704,"entropy":0.4749467819929123},{"step":161,"reward":-0.48847249150276184,"core":-0.037194982171058655,"cost":-0.014850817620754242,"kl":0.10578320175409317,"entropy":0.5259166285395622},{"step":163,"reward":0.3504638671875,"core":0.309692919254303,"cost":0.0,"kl":0.11043049395084381,"entropy":0.5382034331560135},{"step":165,"reward":-1.271574854850769,"core":-0.37025371193885803,"cost":-0.010855305008590221,"kl":0.1341745499521494,"entropy":0.5100760385394096},{"step":167,"reward":-0.023201191797852516,"core":0.2544773817062378,"cost":-0.06358078867197037,"kl":0.1476810723543167,"entropy":0.5562043935060501},{"step":169,"reward":-0.32898613810539246,"core":0.018905440345406532,"cost":-0.05054446682333946,"kl":0.10141302831470966,"entropy":0.5821196883916855},{"step":171,"reward":-0.03612002730369568,"core":0.13353820145130157,"cost":-0.014288835227489471,"kl":0.12247073836624622,"entropy":0.587992399930954},{"step":173,"reward":-0.2706308364868164,"core":0.02983974479138851,"cost":-0.018596617504954338,"kl":0.11906505562365055,"entropy":0.5797469168901443},{"step":175,"reward":-0.06468439847230911,"core":0.15515916049480438,"cost":0.0,"kl":0.13888056576251984,"entropy":0.6494967639446259},{"step":177,"reward":-0.8751840591430664,"core":-0.14936350286006927,"cost":-0.013807565905153751,"kl":0.15812925808131695,"entropy":0.5529816895723343},{"step":179,"reward":0.6077549457550049,"core":0.39678049087524414,"cost":0.0,"kl":0.125404117628932,"entropy":0.5225656628608704},{"step":181,"reward":0.6517971158027649,"core":0.3456486165523529,"cost":0.0,"kl":0.156870536506176,"entropy":0.570443257689476},{"step":183,"reward":-0.54115229845047,"core":-0.08572564274072647,"cost":-0.003911370877176523,"kl":0.15743691101670265,"entropy":0.5473365038633347},{"step":185,"reward":0.5090442895889282,"core":0.27773305773735046,"cost":0.0,"kl":0.1540483944118023,"entropy":0.547747015953064},{"step":187,"reward":-0.7980279922485352,"core":-0.16837476193904877,"cost":-0.008373693563044071,"kl":0.21456316486001015,"entropy":0.6495379060506821},{"step":189,"reward":-0.6991791129112244,"core":-0.18090593814849854,"cost":-0.0311704371124506,"kl":0.10817215405404568,"entropy":0.5144009441137314},{"step":191,"reward":-0.48769137263298035,"core":0.008334964513778687,"cost":-0.07310456037521362,"kl":0.13821169920265675,"entropy":0.48265034705400467},{"step":193,"reward":0.021722570061683655,"core":0.20263636112213135,"cost":-0.02895471639931202,"kl":0.14450794830918312,"entropy":0.5710962116718292},{"step":195,"reward":-1.237398624420166,"core":-0.34390103816986084,"cost":0.0,"kl":0.11535620503127575,"entropy":0.5088917464017868},{"step":197,"reward":-0.27392739057540894,"core":0.0602509044110775,"cost":-0.003125944407656789,"kl":0.11985585279762745,"entropy":0.507869616150856},{"step":199,"reward":-0.3627126216888428,"core":0.029534662142395973,"cost":-0.0536038838326931,"kl":0.19108207523822784,"entropy":0.5547935292124748},{"step":201,"reward":0.37344661355018616,"core":0.35094764828681946,"cost":-0.003979288041591644,"kl":0.14282814040780067,"entropy":0.4892224445939064},{"step":203,"reward":0.0587860532104969,"core":0.21211862564086914,"cost":-0.01941760443150997,"kl":0.143243245780468,"entropy":0.48041724413633347},{"step":205,"reward":0.05701860785484314,"core":0.2275426834821701,"cost":0.0,"kl":0.17791370674967766,"entropy":0.5792839676141739},{"step":207,"reward":0.26550254225730896,"core":0.3144312798976898,"cost":0.0,"kl":0.18840793520212173,"entropy":0.5156762301921844},{"step":209,"reward":-0.024646351113915443,"core":0.13425548374652863,"cost":-0.01156532671302557,"kl":0.162993635982275,"entropy":0.5370768010616302},{"step":211,"reward":-0.29353830218315125,"core":0.04817124083638191,"cost":-0.030729347839951515,"kl":0.15415434911847115,"entropy":0.5310996025800705},{"step":213,"reward":-0.11769469827413559,"core":0.09448635578155518,"cost":0.0,"kl":0.25510260462760925,"entropy":0.5586582720279694},{"step":215,"reward":0.0016818046569824219,"core":0.17216944694519043,"cost":-0.011240091174840927,"kl":0.17380721494555473,"entropy":0.5508740991353989},{"step":217,"reward":0.3045778274536133,"core":0.18291594088077545,"cost":-0.0013273890363052487,"kl":0.12678072601556778,"entropy":0.3771311044692993},{"step":219,"reward":0.5398597121238708,"core":0.34925758838653564,"cost":-0.003778188256546855,"kl":0.18121444806456566,"entropy":0.5329314842820168},{"step":221,"reward":-0.05764101818203926,"core":0.12273592501878738,"cost":0.0,"kl":0.2587229162454605,"entropy":0.5553947687149048},{"step":223,"reward":-0.1220925971865654,"core":0.05604644492268562,"cost":0.0,"kl":0.13566674292087555,"entropy":0.6216614544391632},{"step":225,"reward":-0.08044765889644623,"core":0.18559880554676056,"cost":0.0,"kl":0.21696557104587555,"entropy":0.5800066292285919},{"step":227,"reward":-0.39057207107543945,"core":0.008001104928553104,"cost":0.0,"kl":0.17848936840891838,"entropy":0.5592359527945518},{"step":229,"reward":0.20904946327209473,"core":0.2298821210861206,"cost":-0.024469466879963875,"kl":0.2749501056969166,"entropy":0.5080207958817482},{"step":231,"reward":0.012956957332789898,"core":0.20277123153209686,"cost":-0.033411551266908646,"kl":0.23140283301472664,"entropy":0.5942794531583786},{"step":233,"reward":0.42331576347351074,"core":0.3188876807689667,"cost":0.0,"kl":0.23450808599591255,"entropy":0.594012439250946},{"step":235,"reward":0.17864514887332916,"core":0.23945803940296173,"cost":-0.011150941252708435,"kl":0.2182026505470276,"entropy":0.6354247480630875},{"step":237,"reward":0.20937134325504303,"core":0.2612183094024658,"cost":-0.03157857060432434,"kl":0.206538375467062,"entropy":0.6597975641489029},{"step":239,"reward":0.18198931217193604,"core":0.26966944336891174,"cost":0.0,"kl":0.27763423323631287,"entropy":0.5845329910516739},{"step":241,"reward":-0.06739876419305801,"core":0.07622913271188736,"cost":-0.036046262830495834,"kl":0.23660530894994736,"entropy":0.5804894715547562},{"step":243,"reward":0.28003406524658203,"core":0.2907136380672455,"cost":-0.00801179651170969,"kl":0.24375156685709953,"entropy":0.6173514127731323},{"step":245,"reward":0.6102266311645508,"core":0.4443303346633911,"cost":0.0,"kl":0.20845099538564682,"entropy":0.5863557904958725},{"step":247,"reward":-0.3202354907989502,"core":0.06203126907348633,"cost":-0.037084322422742844,"kl":0.2769325338304043,"entropy":0.6146687865257263},{"step":249,"reward":0.06072211265563965,"core":0.17930598556995392,"cost":-0.0448758602142334,"kl":0.2874051630496979,"entropy":0.569624200463295},{"step":251,"reward":0.22095054388046265,"core":0.33380770683288574,"cost":0.0,"kl":0.23805148527026176,"entropy":0.6024213880300522},{"step":253,"reward":0.8255956172943115,"core":0.4392329752445221,"cost":0.0,"kl":0.22294197231531143,"entropy":0.5241774171590805},{"step":255,"reward":0.6056634783744812,"core":0.41667425632476807,"cost":-0.007463347166776657,"kl":0.24739161506295204,"entropy":0.5287009328603745},{"step":257,"reward":0.2238370031118393,"core":0.29255661368370056,"cost":0.0,"kl":0.2984725311398506,"entropy":0.6012663692235947},{"step":259,"reward":0.06744521856307983,"core":0.2261533886194229,"cost":0.0,"kl":0.18792686983942986,"entropy":0.5777512341737747},{"step":261,"reward":0.3657704293727875,"core":0.342866450548172,"cost":-0.012160996906459332,"kl":0.27413808554410934,"entropy":0.6311326622962952},{"step":263,"reward":-0.31124451756477356,"core":0.03996112570166588,"cost":-0.019249489530920982,"kl":0.2328895591199398,"entropy":0.5746131390333176},{"step":265,"reward":0.14312969148159027,"core":0.23741553723812103,"cost":0.0,"kl":0.3300855979323387,"entropy":0.5570622235536575},{"step":267,"reward":0.22293667495250702,"core":0.24547278881072998,"cost":0.0,"kl":0.22920798882842064,"entropy":0.5713308304548264},{"step":269,"reward":0.1971466988325119,"core":0.22205258905887604,"cost":0.0,"kl":0.23466822504997253,"entropy":0.5376891493797302},{"step":271,"reward":0.7360208034515381,"core":0.4304135739803314,"cost":-0.008789187297224998,"kl":0.3130852095782757,"entropy":0.5588477104902267},{"step":273,"reward":1.2011348009109497,"core":0.6591551303863525,"cost":0.0,"kl":0.2569703347980976,"entropy":0.564315602183342},{"step":275,"reward":0.6383506059646606,"core":0.34308895468711853,"cost":0.0,"kl":0.2984800860285759,"entropy":0.5500450730323792},{"step":277,"reward":1.0398889780044556,"core":0.600601851940155,"cost":-0.021053744480013847,"kl":0.28268326446413994,"entropy":0.5466776639223099},{"step":279,"reward":0.9917727112770081,"core":0.5256771445274353,"cost":0.0,"kl":0.296132355928421,"entropy":0.5416241362690926},{"step":281,"reward":0.9027544856071472,"core":0.46241721510887146,"cost":0.0,"kl":0.29231710359454155,"entropy":0.5885031074285507},{"step":283,"reward":0.29546284675598145,"core":0.32518595457077026,"cost":0.0,"kl":0.22182323783636093,"entropy":0.5600363910198212},{"step":285,"reward":1.3038243055343628,"core":0.700506865978241,"cost":0.0,"kl":0.29822202026844025,"entropy":0.5509204864501953},{"step":287,"reward":0.2528608739376068,"core":0.3191467225551605,"cost":0.0,"kl":0.2755820192396641,"entropy":0.5844527333974838},{"step":289,"reward":0.3477989137172699,"core":0.3626846373081207,"cost":0.0,"kl":0.23924756422638893,"entropy":0.5719298869371414},{"step":291,"reward":0.5852497816085815,"core":0.3678531348705292,"cost":0.0,"kl":0.19718200713396072,"entropy":0.5141138881444931},{"step":293,"reward":1.1101099252700806,"core":0.629809558391571,"cost":0.0,"kl":0.25153666362166405,"entropy":0.5732052624225616},{"step":295,"reward":0.23327000439167023,"core":0.25594502687454224,"cost":-0.005332187283784151,"kl":0.2438911870121956,"entropy":0.6037687808275223},{"step":297,"reward":-0.021363893523812294,"core":0.1447032392024994,"cost":-0.03678680956363678,"kl":0.22784480452537537,"entropy":0.4969567805528641},{"step":299,"reward":0.2403174191713333,"core":0.24210739135742188,"cost":0.0,"kl":0.2521795444190502,"entropy":0.55378358066082},{"step":301,"reward":0.5128468871116638,"core":0.37509241700172424,"cost":0.0,"kl":0.25243471190333366,"entropy":0.5447127595543861},{"step":303,"reward":0.13243480026721954,"core":0.2067864090204239,"cost":0.0,"kl":0.2716054469347,"entropy":0.5638451874256134},{"step":305,"reward":0.6501988768577576,"core":0.44190993905067444,"cost":0.0,"kl":0.21024514362215996,"entropy":0.5686631947755814},{"step":307,"reward":0.23611371219158173,"core":0.2154906988143921,"cost":0.0,"kl":0.17547153308987617,"entropy":0.5084347128868103},{"step":309,"reward":0.5602063536643982,"core":0.34598278999328613,"cost":0.0,"kl":0.23299966752529144,"entropy":0.48489151149988174},{"step":311,"reward":0.8261410593986511,"core":0.4696890413761139,"cost":0.0,"kl":0.18373920395970345,"entropy":0.5714992582798004},{"step":313,"reward":0.9379788041114807,"core":0.5553267598152161,"cost":0.0,"kl":0.2222897671163082,"entropy":0.5740872770547867},{"step":315,"reward":1.2488065958023071,"core":0.7194134593009949,"cost":0.0,"kl":0.21947269514203072,"entropy":0.5146278217434883},{"step":317,"reward":0.7105298042297363,"core":0.4726039171218872,"cost":0.0,"kl":0.25303666293621063,"entropy":0.588226392865181},{"step":319,"reward":1.4422550201416016,"core":0.8044835925102234,"cost":-0.005395147483795881,"kl":0.23695142939686775,"entropy":0.4856649488210678},{"step":321,"reward":0.15691661834716797,"core":0.2813698351383209,"cost":-0.01728040538728237,"kl":0.24498392641544342,"entropy":0.5087575018405914},{"step":323,"reward":0.8796718716621399,"core":0.5014840364456177,"cost":0.0,"kl":0.2917790487408638,"entropy":0.5299004018306732},{"step":325,"reward":0.6768622398376465,"core":0.43519434332847595,"cost":-0.017773086205124855,"kl":0.33019252493977547,"entropy":0.608427494764328},{"step":327,"reward":0.3130214512348175,"core":0.2502639889717102,"cost":-0.01413197536021471,"kl":0.28538260608911514,"entropy":0.5781724005937576},{"step":329,"reward":-0.3025767505168915,"core":0.04849584028124809,"cost":0.0,"kl":0.20334943011403084,"entropy":0.5491360723972321},{"step":331,"reward":0.3180272877216339,"core":0.2695114016532898,"cost":0.0,"kl":0.35369962453842163,"entropy":0.5515847653150558},{"step":333,"reward":-0.07514480501413345,"core":0.12910966575145721,"cost":0.0,"kl":0.21220958977937698,"entropy":0.4978441148996353},{"step":335,"reward":0.28377291560173035,"core":0.2653979957103729,"cost":-0.09623929113149643,"kl":0.19639050215482712,"entropy":0.5035063922405243},{"step":337,"reward":0.9149553775787354,"core":0.40446630120277405,"cost":0.0,"kl":0.1970921903848648,"entropy":0.5737039595842361},{"step":339,"reward":0.07319337874650955,"core":0.2209300845861435,"cost":-0.014138009399175644,"kl":0.2523524612188339,"entropy":0.5634655207395554},{"step":341,"reward":0.9276748299598694,"core":0.5243884921073914,"cost":0.0,"kl":0.23103730380535126,"entropy":0.4910193681716919},{"step":343,"reward":1.0302871465682983,"core":0.5401942133903503,"cost":-0.014953979291021824,"kl":0.21769949421286583,"entropy":0.4613538235425949},{"step":345,"reward":0.07109164446592331,"core":0.10243195295333862,"cost":0.0,"kl":0.16493502259254456,"entropy":0.48911555856466293},{"step":347,"reward":1.3841195106506348,"core":0.5870195031166077,"cost":-0.018188005313277245,"kl":0.18162793293595314,"entropy":0.369293637573719},{"step":349,"reward":0.36514416337013245,"core":0.30829569697380066,"cost":-0.01106050331145525,"kl":0.14484711177647114,"entropy":0.4966853857040405},{"step":351,"reward":-0.13195835053920746,"core":0.09586787223815918,"cost":-0.027841515839099884,"kl":0.20198319479823112,"entropy":0.5621838867664337},{"step":353,"reward":0.8075973391532898,"core":0.5150262713432312,"cost":0.0,"kl":0.18936344236135483,"entropy":0.5081087127327919},{"step":355,"reward":-0.30583369731903076,"core":0.04458194598555565,"cost":-0.002779831178486347,"kl":0.19276827201247215,"entropy":0.5027918666601181},{"step":357,"reward":0.9387310147285461,"core":0.6521701216697693,"cost":0.0,"kl":0.1787358559668064,"entropy":0.5127066224813461},{"step":359,"reward":0.669968843460083,"core":0.4784505367279053,"cost":0.0,"kl":0.30312539637088776,"entropy":0.4909743368625641},{"step":361,"reward":0.3569220006465912,"core":0.34256401658058167,"cost":0.0,"kl":0.23014574125409126,"entropy":0.5141262412071228},{"step":363,"reward":0.35696467757225037,"core":0.30031582713127136,"cost":-0.0015173930441960692,"kl":0.19225537404417992,"entropy":0.6251554489135742},{"step":365,"reward":0.7126893997192383,"core":0.4870389401912689,"cost":0.0,"kl":0.26882612332701683,"entropy":0.5088823884725571},{"step":367,"reward":0.6675341129302979,"core":0.5324567556381226,"cost":-0.011163047514855862,"kl":0.19233860075473785,"entropy":0.43558961898088455},{"step":369,"reward":0.46204233169555664,"core":0.3290896415710449,"cost":-0.003861365607008338,"kl":0.25711994245648384,"entropy":0.5501420050859451},{"step":371,"reward":0.2804202735424042,"core":0.26950541138648987,"cost":-0.0007858308963477612,"kl":0.20844748616218567,"entropy":0.46126890182495117},{"step":373,"reward":0.7278124690055847,"core":0.526908814907074,"cost":-0.0022930833511054516,"kl":0.20780384168028831,"entropy":0.5657324492931366},{"step":375,"reward":1.0610827207565308,"core":0.5928781032562256,"cost":0.0,"kl":0.2283993437886238,"entropy":0.5797199904918671},{"step":377,"reward":0.693331241607666,"core":0.37886977195739746,"cost":0.0,"kl":0.22875872999429703,"entropy":0.5351029932498932},{"step":379,"reward":0.053571414202451706,"core":0.24534957110881805,"cost":0.0,"kl":0.1301144976168871,"entropy":0.5443321242928505},{"step":381,"reward":0.003337552072480321,"core":0.19732315838336945,"cost":-0.04925380274653435,"kl":0.24460053443908691,"entropy":0.6082763820886612},{"step":383,"reward":1.1314550638198853,"core":0.6431698203086853,"cost":0.0,"kl":0.2870376221835613,"entropy":0.5800911039113998},{"step":385,"reward":0.3474853038787842,"core":0.3771909773349762,"cost":-0.03707503527402878,"kl":0.2079446204006672,"entropy":0.6406190544366837},{"step":387,"reward":0.6056647300720215,"core":0.439854234457016,"cost":0.0,"kl":0.2885267250239849,"entropy":0.6435373723506927},{"step":389,"reward":0.7709231376647949,"core":0.533774197101593,"cost":-0.014852073974907398,"kl":0.2121332809329033,"entropy":0.629522904753685},{"step":391,"reward":0.7497709393501282,"core":0.5138940215110779,"cost":0.0,"kl":0.23441097140312195,"entropy":0.5089773908257484},{"step":393,"reward":0.2547822892665863,"core":0.22487731277942657,"cost":-9.86788381851511e-06,"kl":0.2681402117013931,"entropy":0.635570764541626},{"step":395,"reward":0.6163936257362366,"core":0.40772807598114014,"cost":0.0,"kl":0.23695539683103561,"entropy":0.6465373933315277},{"step":397,"reward":0.0031547248363494873,"core":0.09725966304540634,"cost":-0.0016949675045907497,"kl":0.25075120851397514,"entropy":0.5835125595331192},{"step":399,"reward":0.6885180473327637,"core":0.46137118339538574,"cost":0.0,"kl":0.2412632741034031,"entropy":0.5958002358675003},{"step":401,"reward":0.612421452999115,"core":0.38139811158180237,"cost":0.0,"kl":0.2371288649737835,"entropy":0.5708961933851242},{"step":403,"reward":0.8592727780342102,"core":0.5258662700653076,"cost":-0.030792199075222015,"kl":0.21445102617144585,"entropy":0.588295042514801},{"step":405,"reward":0.7131936550140381,"core":0.4435659945011139,"cost":0.0,"kl":0.24026508629322052,"entropy":0.5886612385511398},{"step":407,"reward":1.1612825393676758,"core":0.5561115145683289,"cost":-0.006859665270894766,"kl":0.2576840743422508,"entropy":0.6086118668317795},{"step":409,"reward":1.2599304914474487,"core":0.6571221947669983,"cost":-0.0009299991652369499,"kl":0.2674005888402462,"entropy":0.5814877897500992},{"step":411,"reward":0.7451589703559875,"core":0.4984380900859833,"cost":-0.0006965851061977446,"kl":0.22413459420204163,"entropy":0.556441992521286},{"step":413,"reward":0.6076682209968567,"core":0.42465531826019287,"cost":0.0,"kl":0.22945290803909302,"entropy":0.5426755994558334},{"step":415,"reward":0.7090272903442383,"core":0.4568811357021332,"cost":0.0,"kl":0.21068356186151505,"entropy":0.6131879836320877},{"step":417,"reward":0.1265193372964859,"core":0.21893596649169922,"cost":-0.0009699634392745793,"kl":0.3478201888501644,"entropy":0.5664083063602448},{"step":419,"reward":1.1779624223709106,"core":0.691781222820282,"cost":0.0,"kl":0.23290061578154564,"entropy":0.6295117288827896},{"step":421,"reward":0.5068582892417908,"core":0.4010944068431854,"cost":-0.01268017292022705,"kl":0.2747397758066654,"entropy":0.5668878853321075},{"step":423,"reward":0.24650071561336517,"core":0.2746848165988922,"cost":0.0,"kl":0.22386808320879936,"entropy":0.6061518639326096},{"step":425,"reward":0.5810999274253845,"core":0.4267909824848175,"cost":0.0,"kl":0.24970851838588715,"entropy":0.5957348644733429},{"step":427,"reward":0.08369499444961548,"core":0.18945825099945068,"cost":-0.014444573782384396,"kl":0.22528427094221115,"entropy":0.6191080957651138},{"step":429,"reward":0.8473987579345703,"core":0.5828564167022705,"cost":0.0,"kl":0.18149985373020172,"entropy":0.5843337327241898},{"step":431,"reward":0.6954848766326904,"core":0.3848130404949188,"cost":-0.026487378403544426,"kl":0.25920890644192696,"entropy":0.5706891566514969},{"step":433,"reward":0.2725450396537781,"core":0.2576186954975128,"cost":0.0,"kl":0.19154999777674675,"entropy":0.6082488000392914},{"step":435,"reward":0.6500142812728882,"core":0.41715964674949646,"cost":-0.006544440519064665,"kl":0.21772155165672302,"entropy":0.5728239268064499},{"step":437,"reward":1.07716965675354,"core":0.6316351294517517,"cost":0.0,"kl":0.2640087902545929,"entropy":0.555911660194397},{"step":439,"reward":0.7813217043876648,"core":0.4148194491863251,"cost":0.0,"kl":0.23144057393074036,"entropy":0.6194622218608856},{"step":441,"reward":0.5452620983123779,"core":0.4004432260990143,"cost":-0.011625547893345356,"kl":0.20872826501727104,"entropy":0.5827572196722031},{"step":443,"reward":0.14927048981189728,"core":0.25255534052848816,"cost":-0.00772879971191287,"kl":0.2685563266277313,"entropy":0.5831588059663773},{"step":445,"reward":0.930197536945343,"core":0.549455463886261,"cost":0.0,"kl":0.26025449857115746,"entropy":0.5414847284555435},{"step":447,"reward":1.0562392473220825,"core":0.6972439885139465,"cost":0.0,"kl":0.23696257546544075,"entropy":0.5005107298493385},{"step":449,"reward":1.2329946756362915,"core":0.754263699054718,"cost":0.0,"kl":0.24112558364868164,"entropy":0.48629186302423477},{"step":451,"reward":0.3801988661289215,"core":0.3510188162326813,"cost":-0.011809065006673336,"kl":0.2200249284505844,"entropy":0.6232151240110397},{"step":453,"reward":0.8410606384277344,"core":0.4930359125137329,"cost":0.0,"kl":0.2997531145811081,"entropy":0.4889129176735878},{"step":455,"reward":0.9680449366569519,"core":0.5821756720542908,"cost":-0.009634003974497318,"kl":0.3136148788034916,"entropy":0.5771058350801468},{"step":457,"reward":0.7817193865776062,"core":0.5844761729240417,"cost":-0.02288275957107544,"kl":0.19687741994857788,"entropy":0.5630888938903809},{"step":459,"reward":0.7963661551475525,"core":0.5023157596588135,"cost":0.0,"kl":0.3341452479362488,"entropy":0.5617460459470749},{"step":461,"reward":0.07723067700862885,"core":0.12402182072401047,"cost":0.0,"kl":0.25913309678435326,"entropy":0.6164464950561523},{"step":463,"reward":0.7163861393928528,"core":0.4092657268047333,"cost":0.0,"kl":0.3515312597155571,"entropy":0.5430122315883636},{"step":465,"reward":0.4673938453197479,"core":0.34306588768959045,"cost":-0.00968074519187212,"kl":0.2107277810573578,"entropy":0.5582015961408615},{"step":467,"reward":0.4800971746444702,"core":0.3134559392929077,"cost":0.0,"kl":0.2987285330891609,"entropy":0.5234085991978645},{"step":469,"reward":1.1344492435455322,"core":0.5637615323066711,"cost":0.0,"kl":0.1925010159611702,"entropy":0.5228291004896164},{"step":471,"reward":0.3426358997821808,"core":0.21401925384998322,"cost":0.0,"kl":0.2831581123173237,"entropy":0.5631276965141296},{"step":473,"reward":-0.3381350040435791,"core":0.015918085351586342,"cost":-0.02708623744547367,"kl":0.17309065908193588,"entropy":0.475373312830925},{"step":475,"reward":0.7968043684959412,"core":0.4336063861846924,"cost":-0.01892860233783722,"kl":0.20830013230443,"entropy":0.48510613292455673},{"step":477,"reward":0.4558013677597046,"core":0.3679593503475189,"cost":0.0,"kl":0.33998946473002434,"entropy":0.5349816381931305},{"step":479,"reward":0.6840817332267761,"core":0.4854315221309662,"cost":-0.02877316065132618,"kl":0.211977306753397,"entropy":0.582945704460144},{"step":481,"reward":0.7987434267997742,"core":0.4999931752681732,"cost":-0.00838708970695734,"kl":0.21870136260986328,"entropy":0.5725783854722977},{"step":483,"reward":0.7707765102386475,"core":0.4826160967350006,"cost":0.0,"kl":0.36958396807312965,"entropy":0.5494211912155151},{"step":485,"reward":0.5151382684707642,"core":0.4200022518634796,"cost":-0.015080251730978489,"kl":0.2032580003142357,"entropy":0.5456797033548355},{"step":487,"reward":0.3697117865085602,"core":0.2684875428676605,"cost":-0.0010274528758600354,"kl":0.24883177876472473,"entropy":0.6022958606481552},{"step":489,"reward":0.529446005821228,"core":0.40991654992103577,"cost":0.0,"kl":0.30698365718126297,"entropy":0.521265298128128},{"step":491,"reward":0.7374410629272461,"core":0.5071583390235901,"cost":0.0,"kl":0.33666183799505234,"entropy":0.5437557548284531},{"step":493,"reward":-0.7132111191749573,"core":-0.14263340830802917,"cost":-0.05019882321357727,"kl":0.1984512247145176,"entropy":0.46098901331424713},{"step":495,"reward":0.611519992351532,"core":0.4286890923976898,"cost":-0.03083251602947712,"kl":0.2005668245255947,"entropy":0.5450420528650284},{"step":497,"reward":0.43927276134490967,"core":0.3237290382385254,"cost":-0.0019333148375153542,"kl":0.19651975110173225,"entropy":0.5505979359149933},{"step":499,"reward":1.091997504234314,"core":0.684273898601532,"cost":-0.0011415501357987523,"kl":0.26225194334983826,"entropy":0.5968599170446396},{"step":501,"reward":0.36848437786102295,"core":0.3401974141597748,"cost":0.0,"kl":0.29312678426504135,"entropy":0.6388998329639435},{"step":503,"reward":0.762589156627655,"core":0.4961582124233246,"cost":0.0,"kl":0.23161251470446587,"entropy":0.4952751398086548},{"step":505,"reward":0.7807701230049133,"core":0.5234550833702087,"cost":-0.01928786374628544,"kl":0.17989103868603706,"entropy":0.5232865363359451},{"step":507,"reward":0.6645441651344299,"core":0.3790891170501709,"cost":-0.006462855730205774,"kl":0.23840146511793137,"entropy":0.4984557777643204},{"step":509,"reward":0.1254166215658188,"core":0.2446722537279129,"cost":-0.05342371389269829,"kl":0.18791833519935608,"entropy":0.5715547651052475},{"step":511,"reward":1.0537022352218628,"core":0.558901846408844,"cost":0.0,"kl":0.2760499641299248,"entropy":0.5656337738037109},{"step":513,"reward":0.692818820476532,"core":0.4433545768260956,"cost":0.0,"kl":0.2454686388373375,"entropy":0.4753450080752373},{"step":515,"reward":-0.49394187331199646,"core":0.004138594027608633,"cost":-0.004695085808634758,"kl":0.2456357665359974,"entropy":0.5735345631837845},{"step":517,"reward":0.7239828109741211,"core":0.4004436731338501,"cost":0.0,"kl":0.21292348578572273,"entropy":0.5277149006724358},{"step":519,"reward":0.8356750011444092,"core":0.5468140840530396,"cost":0.0,"kl":0.16760925948619843,"entropy":0.5485576242208481},{"step":521,"reward":0.7410764098167419,"core":0.4634898900985718,"cost":-0.004750341642647982,"kl":0.20907320454716682,"entropy":0.5561656057834625},{"step":523,"reward":0.9613439440727234,"core":0.5821962356567383,"cost":0.0,"kl":0.25279172882437706,"entropy":0.5877895206212997},{"step":525,"reward":0.28061801195144653,"core":0.32101500034332275,"cost":0.0,"kl":0.18093682825565338,"entropy":0.5306370854377747},{"step":527,"reward":1.2826796770095825,"core":0.7324156761169434,"cost":-0.00854461919516325,"kl":0.16396477818489075,"entropy":0.6072721779346466},{"step":529,"reward":0.8304006457328796,"core":0.43687471747398376,"cost":-0.0002538115077186376,"kl":0.17567266896367073,"entropy":0.4639165997505188},{"step":531,"reward":0.040452536195516586,"core":0.1618943065404892,"cost":-0.0016487740213051438,"kl":0.23476146161556244,"entropy":0.5799297988414764},{"step":533,"reward":0.46946874260902405,"core":0.3182072639465332,"cost":-0.0014256075955927372,"kl":0.2463492564857006,"entropy":0.5591491758823395},{"step":535,"reward":1.323684573173523,"core":0.7379913330078125,"cost":0.0,"kl":0.29110728576779366,"entropy":0.4545917585492134},{"step":537,"reward":0.12317579984664917,"core":0.21134914457798004,"cost":0.0,"kl":0.2514260932803154,"entropy":0.5617192834615707},{"step":539,"reward":0.6518124938011169,"core":0.40542253851890564,"cost":0.0,"kl":0.2095012664794922,"entropy":0.571209192276001},{"step":541,"reward":1.1364967823028564,"core":0.5282779932022095,"cost":0.0,"kl":0.19965286925435066,"entropy":0.5865757763385773},{"step":543,"reward":1.0062025785446167,"core":0.6268911957740784,"cost":-0.002116648480296135,"kl":0.2064841277897358,"entropy":0.5401715487241745},{"step":545,"reward":0.5305641293525696,"core":0.3722357749938965,"cost":-0.008002390153706074,"kl":0.2766320928931236,"entropy":0.5499307215213776},{"step":547,"reward":0.140680730342865,"core":0.23714418709278107,"cost":0.0,"kl":0.23051060363650322,"entropy":0.554503008723259},{"step":549,"reward":0.8605346083641052,"core":0.5283588767051697,"cost":-0.005288723390549421,"kl":0.26481932401657104,"entropy":0.5204731374979019},{"step":551,"reward":-0.018657376989722252,"core":0.2018168568611145,"cost":-0.02151823230087757,"kl":0.23982411995530128,"entropy":0.5937997102737427},{"step":553,"reward":0.928144633769989,"core":0.5784510970115662,"cost":0.0,"kl":0.19057713821530342,"entropy":0.5540837496519089},{"step":555,"reward":1.0655899047851562,"core":0.6739771366119385,"cost":-0.009024015627801418,"kl":0.27176448702812195,"entropy":0.49344898015260696},{"step":557,"reward":0.3074645698070526,"core":0.2829933166503906,"cost":0.0,"kl":0.3314288258552551,"entropy":0.5927204191684723},{"step":559,"reward":0.4474720060825348,"core":0.31226691603660583,"cost":0.0,"kl":0.29768022522330284,"entropy":0.6161260157823563},{"step":561,"reward":0.9677505493164062,"core":0.5457125306129456,"cost":0.0,"kl":0.21498418226838112,"entropy":0.6050094366073608},{"step":563,"reward":0.46932461857795715,"core":0.4017384946346283,"cost":0.0,"kl":0.227526243776083,"entropy":0.5585429966449738},{"step":565,"reward":0.24753589928150177,"core":0.2972008287906647,"cost":-0.0005784541135653853,"kl":0.19087938964366913,"entropy":0.5276327654719353},{"step":567,"reward":0.0584336519241333,"core":0.17540405690670013,"cost":-0.008716887794435024,"kl":0.2512547895312309,"entropy":0.5501896813511848},{"step":569,"reward":1.3155874013900757,"core":0.7019648551940918,"cost":0.0,"kl":0.21795770153403282,"entropy":0.5417640432715416},{"step":571,"reward":0.4179438054561615,"core":0.373183012008667,"cost":0.0,"kl":0.31013165414333344,"entropy":0.6224492937326431},{"step":573,"reward":1.166940450668335,"core":0.5516904592514038,"cost":-0.0004236571257933974,"kl":0.28874095156788826,"entropy":0.5528886467218399},{"step":575,"reward":1.0580428838729858,"core":0.6146683096885681,"cost":0.0,"kl":0.2414521798491478,"entropy":0.5993564277887344},{"step":577,"reward":0.8123040199279785,"core":0.4604664742946625,"cost":0.0,"kl":0.20920203626155853,"entropy":0.6137315928936005},{"step":579,"reward":0.7745044231414795,"core":0.492196649312973,"cost":-0.0222002062946558,"kl":0.20378470420837402,"entropy":0.5677581876516342},{"step":581,"reward":-0.7471769452095032,"core":-0.08674019575119019,"cost":-0.04157204553484917,"kl":0.3025795556604862,"entropy":0.5952936857938766},{"step":583,"reward":0.2867359519004822,"core":0.3007255792617798,"cost":-0.029071152210235596,"kl":0.23750458285212517,"entropy":0.5941193997859955},{"step":585,"reward":0.9233492016792297,"core":0.5555291771888733,"cost":-0.030322877690196037,"kl":0.20185577496886253,"entropy":0.5306228995323181},{"step":587,"reward":0.9028828740119934,"core":0.5080797076225281,"cost":-0.019695186987519264,"kl":0.20609644800424576,"entropy":0.5018159970641136},{"step":589,"reward":0.7251648902893066,"core":0.4849729537963867,"cost":0.0,"kl":0.21831494197249413,"entropy":0.6492436826229095},{"step":591,"reward":1.0527561902999878,"core":0.6836828589439392,"cost":0.0,"kl":0.3607541546225548,"entropy":0.5528942495584488},{"step":593,"reward":0.23998849093914032,"core":0.22147266566753387,"cost":0.0,"kl":0.17183683440089226,"entropy":0.4818228483200073},{"step":595,"reward":1.099272608757019,"core":0.6502106189727783,"cost":0.0,"kl":0.26223529130220413,"entropy":0.574153408408165},{"step":597,"reward":-0.0001684129238128662,"core":0.1505947709083557,"cost":0.0,"kl":0.19754130765795708,"entropy":0.5972196906805038},{"step":599,"reward":0.5310672521591187,"core":0.44054996967315674,"cost":-0.025076620280742645,"kl":0.2525387220084667,"entropy":0.6092239320278168}],"nonzero_advantage":[{"step":1,"nonzero":0.2916666666666667},{"step":3,"nonzero":0.5833333333333334},{"step":5,"nonzero":0.3958333333333333},{"step":7,"nonzero":0.8125},{"step":9,"nonzero":0.5},{"step":11,"nonzero":0.4375},{"step":13,"nonzero":0.7083333333333334},{"step":15,"nonzero":0.7291666666666666},{"step":17,"nonzero":0.5833333333333334},{"step":19,"nonzero":0.6458333333333334},{"step":21,"nonzero":0.3125},{"step":23,"nonzero":0.3541666666666667},{"step":25,"nonzero":0.4583333333333333},{"step":27,"nonzero":0.5625},{"step":29,"nonzero":0.75},{"step":31,"nonzero":0.5625},{"step":33,"nonzero":0.8958333333333334},{"step":35,"nonzero":0.8541666666666666},{"step":37,"nonzero":0.5625},{"step":39,"nonzero":0.875},{"step":41,"nonzero":0.625},{"step":43,"nonzero":0.7083333333333334},{"step":45,"nonzero":0.875},{"step":47,"nonzero":0.7291666666666666},{"step":49,"nonzero":0.8958333333333334},{"step":51,"nonzero":0.8958333333333334},{"step":53,"nonzero":0.9375},{"step":55,"nonzero":0.75},{"step":57,"nonzero":0.9166666666666666},{"step":59,"nonzero":0.8333333333333334},{"step":61,"nonzero":0.75},{"step":63,"nonzero":0.8541666666666666},{"step":65,"nonzero":0.6041666666666666},{"step":67,"nonzero":0.6875},{"step":69,"nonzero":0.5833333333333334},{"step":71,"nonzero":0.875},{"step":73,"nonzero":0.8333333333333334},{"step":75,"nonzero":1.0},{"step":77,"nonzero":0.9166666666666666},{"step":79,"nonzero":0.7708333333333334},{"step":81,"nonzero":0.9583333333333334},{"step":83,"nonzero":0.75},{"step":85,"nonzero":0.9166666666666666},{"step":87,"nonzero":0.8125},{"step":89,"nonzero":0.8541666666666666},{"step":91,"nonzero":0.6458333333333334},{"step":93,"nonzero":0.8541666666666666},{"step":95,"nonzero":0.6041666666666666},{"step":97,"nonzero":0.8541666666666666},{"step":99,"nonzero":0.8958333333333334},{"step":101,"nonzero":0.8125},{"step":103,"nonzero":0.8958333333333334},{"step":105,"nonzero":0.5416666666666666},{"step":107,"nonzero":0.9583333333333334},{"step":109,"nonzero":0.8333333333333334},{"step":111,"nonzero":0.9375},{"step":113,"nonzero":0.8541666666666666},{"step":115,"nonzero":0.5833333333333334},{"step":117,"nonzero":0.7291666666666666},{"step":119,"nonzero":0.8333333333333334},{"step":121,"nonzero":0.9583333333333334},{"step":123,"nonzero":0.9583333333333334},{"step":125,"nonzero":0.8958333333333334},{"step":127,"nonzero":0.5833333333333334},{"step":129,"nonzero":0.7916666666666666},{"step":131,"nonzero":0.6875},{"step":133,"nonzero":0.9583333333333334},{"step":135,"nonzero":0.8125},{"step":137,"nonzero":0.7708333333333334},{"step":139,"nonzero":0.7708333333333334},{"step":141,"nonzero":0.8541666666666666},{"step":143,"nonzero":0.6875},{"step":145,"nonzero":0.6875},{"step":147,"nonzero":0.7708333333333334},{"step":149,"nonzero":0.875},{"step":151,"nonzero":0.6458333333333334},{"step":153,"nonzero":0.7916666666666666},{"step":155,"nonzero":0.75},{"step":157,"nonzero":0.8333333333333334},{"step":159,"nonzero":0.7916666666666666},{"step":161,"nonzero":0.5833333333333334},{"step":163,"nonzero":1.0},{"step":165,"nonzero":0.5833333333333334},{"step":167,"nonzero":0.875},{"step":169,"nonzero":0.9166666666666666},{"step":171,"nonzero":0.8541666666666666},{"step":173,"nonzero":0.7916666666666666},{"step":175,"nonzero":1.0},{"step":177,"nonzero":0.7708333333333334},{"step":179,"nonzero":0.9583333333333334},{"step":181,"nonzero":1.0},{"step":183,"nonzero":0.8125},{"step":185,"nonzero":0.9583333333333334},{"step":187,"nonzero":0.7291666666666666},{"step":189,"nonzero":0.7083333333333334},{"step":191,"nonzero":0.9166666666666666},{"step":193,"nonzero":0.9583333333333334},{"step":195,"nonzero":0.7916666666666666},{"step":197,"nonzero":0.9166666666666666},{"step":199,"nonzero":0.8541666666666666},{"step":201,"nonzero":1.0},{"step":203,"nonzero":0.9166666666666666},{"step":205,"nonzero":0.9375},{"step":207,"nonzero":0.9791666666666666},{"step":209,"nonzero":0.9583333333333334},{"step":211,"nonzero":0.7916666666666666},{"step":213,"nonzero":0.9375},{"step":215,"nonzero":0.9166666666666666},{"step":217,"nonzero":0.9791666666666666},{"step":219,"nonzero":0.9791666666666666},{"step":221,"nonzero":0.9375},{"step":223,"nonzero":0.9583333333333334},{"step":225,"nonzero":0.875},{"step":227,"nonzero":0.5625},{"step":229,"nonzero":0.8333333333333334},{"step":231,"nonzero":0.9791666666666666},{"step":233,"nonzero":0.9375},{"step":235,"nonzero":0.9375},{"step":237,"nonzero":0.8958333333333334},{"step":239,"nonzero":0.9166666666666666},{"step":241,"nonzero":0.8958333333333334},{"step":243,"nonzero":0.875},{"step":245,"nonzero":0.9375},{"step":247,"nonzero":0.875},{"step":249,"nonzero":0.8333333333333334},{"step":251,"nonzero":0.9583333333333334},{"step":253,"nonzero":1.0},{"step":255,"nonzero":0.8541666666666666},{"step":257,"nonzero":0.8958333333333334},{"step":259,"nonzero":0.9583333333333334},{"step":261,"nonzero":0.9375},{"step":263,"nonzero":0.8541666666666666},{"step":265,"nonzero":0.8958333333333334},{"step":267,"nonzero":0.9583333333333334},{"step":269,"nonzero":0.9375},{"step":271,"nonzero":1.0},{"step":273,"nonzero":1.0},{"step":275,"nonzero":0.8125},{"step":277,"nonzero":1.0},{"step":279,"nonzero":0.9791666666666666},{"step":281,"nonzero":1.0},{"step":283,"nonzero":0.9583333333333334},{"step":285,"nonzero":1.0},{"step":287,"nonzero":0.9583333333333334},{"step":289,"nonzero":0.9791666666666666},{"step":291,"nonzero":0.9791666666666666},{"step":293,"nonzero":1.0},{"step":295,"nonzero":0.8125},{"step":297,"nonzero":0.8125},{"step":299,"nonzero":0.9375},{"step":301,"nonzero":0.9166666666666666},{"step":303,"nonzero":0.9791666666666666},{"step":305,"nonzero":1.0},{"step":307,"nonzero":0.9791666666666666},{"step":309,"nonzero":0.9791666666666666},{"step":311,"nonzero":0.9375},{"step":313,"nonzero":1.0},{"step":315,"nonzero":1.0},{"step":317,"nonzero":0.9375},{"step":319,"nonzero":1.0},{"step":321,"nonzero":0.8125},{"step":323,"nonzero":1.0},{"step":325,"nonzero":1.0},{"step":327,"nonzero":0.9166666666666666},{"step":329,"nonzero":0.8541666666666666},{"step":331,"nonzero":0.9375},{"step":333,"nonzero":0.8958333333333334},{"step":335,"nonzero":0.9375},{"step":337,"nonzero":1.0},{"step":339,"nonzero":0.9375},{"step":341,"nonzero":0.9583333333333334},{"step":343,"nonzero":1.0},{"step":345,"nonzero":0.9583333333333334},{"step":347,"nonzero":1.0},{"step":349,"nonzero":0.9583333333333334},{"step":351,"nonzero":0.9583333333333334},{"step":353,"nonzero":0.9166666666666666},{"step":355,"nonzero":0.8333333333333334},{"step":357,"nonzero":1.0},{"step":359,"nonzero":0.9583333333333334},{"step":361,"nonzero":0.9583333333333334},{"step":363,"nonzero":0.9583333333333334},{"step":365,"nonzero":1.0},{"step":367,"nonzero":0.9791666666666666},{"step":369,"nonzero":1.0},{"step":371,"nonzero":0.7916666666666666},{"step":373,"nonzero":0.8958333333333334},{"step":375,"nonzero":0.9791666666666666},{"step":377,"nonzero":0.9166666666666666},{"step":379,"nonzero":0.9166666666666666},{"step":381,"nonzero":0.9166666666666666},{"step":383,"nonzero":0.9583333333333334},{"step":385,"nonzero":0.9583333333333334},{"step":387,"nonzero":1.0},{"step":389,"nonzero":0.9791666666666666},{"step":391,"nonzero":0.9583333333333334},{"step":393,"nonzero":0.9375},{"step":395,"nonzero":1.0},{"step":397,"nonzero":0.9166666666666666},{"step":399,"nonzero":0.9791666666666666},{"step":401,"nonzero":0.9583333333333334},{"step":403,"nonzero":1.0},{"step":405,"nonzero":1.0},{"step":407,"nonzero":1.0},{"step":409,"nonzero":1.0},{"step":411,"nonzero":0.9791666666666666},{"step":413,"nonzero":0.9583333333333334},{"step":415,"nonzero":0.9791666666666666},{"step":417,"nonzero":0.9166666666666666},{"step":419,"nonzero":0.9791666666666666},{"step":421,"nonzero":0.9791666666666666},{"step":423,"nonzero":0.9583333333333334},{"step":425,"nonzero":0.9791666666666666},{"step":427,"nonzero":0.9166666666666666},{"step":429,"nonzero":1.0},{"step":431,"nonzero":0.9791666666666666},{"step":433,"nonzero":0.9375},{"step":435,"nonzero":1.0},{"step":437,"nonzero":0.9791666666666666},{"step":439,"nonzero":0.9583333333333334},{"step":441,"nonzero":1.0},{"step":443,"nonzero":0.9166666666666666},{"step":445,"nonzero":0.9791666666666666},{"step":447,"nonzero":1.0},{"step":449,"nonzero":1.0},{"step":451,"nonzero":0.9375},{"step":453,"nonzero":0.9166666666666666},{"step":455,"nonzero":1.0},{"step":457,"nonzero":0.9166666666666666},{"step":459,"nonzero":1.0},{"step":461,"nonzero":0.875},{"step":463,"nonzero":0.9375},{"step":465,"nonzero":0.9166666666666666},{"step":467,"nonzero":0.9791666666666666},{"step":469,"nonzero":1.0},{"step":471,"nonzero":0.8541666666666666},{"step":473,"nonzero":0.875},{"step":475,"nonzero":0.9791666666666666},{"step":477,"nonzero":0.9791666666666666},{"step":479,"nonzero":0.9791666666666666},{"step":481,"nonzero":1.0},{"step":483,"nonzero":1.0},{"step":485,"nonzero":0.9791666666666666},{"step":487,"nonzero":0.9791666666666666},{"step":489,"nonzero":1.0},{"step":491,"nonzero":1.0},{"step":493,"nonzero":0.7916666666666666},{"step":495,"nonzero":0.9375},{"step":497,"nonzero":1.0},{"step":499,"nonzero":1.0},{"step":501,"nonzero":0.9791666666666666},{"step":503,"nonzero":1.0},{"step":505,"nonzero":1.0},{"step":507,"nonzero":0.875},{"step":509,"nonzero":0.9583333333333334},{"step":511,"nonzero":1.0},{"step":513,"nonzero":0.8958333333333334},{"step":515,"nonzero":0.7916666666666666},{"step":517,"nonzero":1.0},{"step":519,"nonzero":0.9791666666666666},{"step":521,"nonzero":0.9583333333333334},{"step":523,"nonzero":0.9583333333333334},{"step":525,"nonzero":0.9791666666666666},{"step":527,"nonzero":1.0},{"step":529,"nonzero":1.0},{"step":531,"nonzero":0.9583333333333334},{"step":533,"nonzero":0.875},{"step":535,"nonzero":1.0},{"step":537,"nonzero":0.8958333333333334},{"step":539,"nonzero":1.0},{"step":541,"nonzero":1.0},{"step":543,"nonzero":1.0},{"step":545,"nonzero":0.9791666666666666},{"step":547,"nonzero":0.8541666666666666},{"step":549,"nonzero":0.9791666666666666},{"step":551,"nonzero":0.8541666666666666},{"step":553,"nonzero":1.0},{"step":555,"nonzero":1.0},{"step":557,"nonzero":0.9791666666666666},{"step":559,"nonzero":0.9583333333333334},{"step":561,"nonzero":1.0},{"step":563,"nonzero":0.8958333333333334},{"step":565,"nonzero":0.9166666666666666},{"step":567,"nonzero":0.7916666666666666},{"step":569,"nonzero":1.0},{"step":571,"nonzero":0.9166666666666666},{"step":573,"nonzero":1.0},{"step":575,"nonzero":1.0},{"step":577,"nonzero":0.9375},{"step":579,"nonzero":1.0},{"step":581,"nonzero":0.9583333333333334},{"step":583,"nonzero":0.9166666666666666},{"step":585,"nonzero":1.0},{"step":587,"nonzero":1.0},{"step":589,"nonzero":1.0},{"step":591,"nonzero":0.9791666666666666},{"step":593,"nonzero":0.9583333333333334},{"step":595,"nonzero":1.0},{"step":597,"nonzero":0.9583333333333334},{"step":599,"nonzero":1.0}]}],"environment_b":[{"id":"b-floor0","label":"Advantage floor at 0 (first slice)","run_id":"1791045171","cpcv_floor":0.0,"train_steps":[{"step":1,"reward":-13.691131591796875,"core":-4.519927978515625,"cost":-0.08229321986436844,"kl":0.0,"entropy":0.46514444798231125},{"step":3,"reward":-6.6489081382751465,"core":-2.168501853942871,"cost":-0.026844466105103493,"kl":0.0,"entropy":0.5031708776950836},{"step":5,"reward":-16.778303146362305,"core":-5.519523620605469,"cost":-0.15398754179477692,"kl":0.0,"entropy":0.48862289637327194},{"step":7,"reward":-12.535252571105957,"core":-4.154280185699463,"cost":-0.05645408108830452,"kl":0.0,"entropy":0.5186695531010628},{"step":9,"reward":-12.600050926208496,"core":-4.124881744384766,"cost":-0.1294647604227066,"kl":0.0,"entropy":0.5288020372390747},{"step":11,"reward":-11.580638885498047,"core":-3.792008638381958,"cost":-0.0959746241569519,"kl":0.0,"entropy":0.44090986996889114},{"step":13,"reward":-11.957697868347168,"core":-3.9302265644073486,"cost":-0.09962377697229385,"kl":0.0005543634833884425,"entropy":0.487661711871624},{"step":15,"reward":-16.154094696044922,"core":-5.3063507080078125,"cost":-0.14480595290660858,"kl":0.0007760578882880509,"entropy":0.494719460606575},{"step":17,"reward":-8.841741561889648,"core":-2.8751494884490967,"cost":-0.05994458869099617,"kl":0.0011859039950650185,"entropy":0.48292917013168335},{"step":19,"reward":-13.803447723388672,"core":-4.519752025604248,"cost":-0.1359005719423294,"kl":0.0013958184863440692,"entropy":0.41880902647972107},{"step":21,"reward":-8.994585990905762,"core":-2.9428870677948,"cost":-0.0733126625418663,"kl":0.0032846799003891647,"entropy":0.4896969497203827},{"step":23,"reward":-14.57366943359375,"core":-4.756994724273682,"cost":-0.1894191950559616,"kl":0.008015691884793341,"entropy":0.43590255081653595},{"step":25,"reward":-12.257789611816406,"core":-4.017605304718018,"cost":-0.07667507976293564,"kl":0.01594374398700893,"entropy":0.4063796177506447},{"step":27,"reward":-12.457215309143066,"core":-4.053889751434326,"cost":-0.14135508239269257,"kl":0.024365941528230906,"entropy":0.36305755376815796},{"step":29,"reward":-14.874862670898438,"core":-4.927863597869873,"cost":-0.057517316192388535,"kl":0.02998571004718542,"entropy":0.39648963510990143},{"step":31,"reward":-10.326281547546387,"core":-3.3939380645751953,"cost":-0.03484125807881355,"kl":0.04055414255708456,"entropy":0.41107598692178726},{"step":33,"reward":-20.1583309173584,"core":-6.614889621734619,"cost":-0.24286265671253204,"kl":0.02426755102351308,"entropy":0.36127979308366776},{"step":35,"reward":-8.691088676452637,"core":-2.841236114501953,"cost":-0.03950050100684166,"kl":0.027758695650845766,"entropy":0.4208711311221123},{"step":37,"reward":-9.026602745056152,"core":-2.9619922637939453,"cost":-0.04268703982234001,"kl":0.022784923668950796,"entropy":0.48334403336048126},{"step":39,"reward":-8.169272422790527,"core":-2.686171770095825,"cost":-0.028589263558387756,"kl":0.026901497971266508,"entropy":0.49026573449373245},{"step":41,"reward":-10.463696479797363,"core":-3.4470646381378174,"cost":-0.023321116343140602,"kl":0.020916287787258625,"entropy":0.5165163651108742},{"step":43,"reward":-7.435020446777344,"core":-2.4391565322875977,"cost":-0.04174748435616493,"kl":0.022847862914204597,"entropy":0.5001233890652657},{"step":45,"reward":-10.995524406433105,"core":-3.6251614093780518,"cost":-0.05049269273877144,"kl":0.014828207669779658,"entropy":0.5071963965892792},{"step":47,"reward":-10.041023254394531,"core":-3.316743850708008,"cost":-0.03563358634710312,"kl":0.01703766197897494,"entropy":0.5797276645898819},{"step":49,"reward":-9.772917747497559,"core":-3.2178544998168945,"cost":-0.06732203811407089,"kl":0.02060046372935176,"entropy":0.5788328647613525},{"step":51,"reward":-10.206892967224121,"core":-3.376185417175293,"cost":-0.02903410792350769,"kl":0.016886372584849596,"entropy":0.5655150860548019},{"step":53,"reward":-11.798815727233887,"core":-3.890947103500366,"cost":-0.07212116569280624,"kl":0.01661741267889738,"entropy":0.5560417920351028},{"step":55,"reward":-5.87039041519165,"core":-1.935556411743164,"cost":0.0,"kl":0.01796504994854331,"entropy":0.5815047025680542},{"step":57,"reward":-14.373149871826172,"core":-4.703907012939453,"cost":-0.15446047484874725,"kl":0.016598548740148544,"entropy":0.47320932894945145},{"step":59,"reward":-10.5322265625,"core":-3.4523143768310547,"cost":-0.07720286399126053,"kl":0.014945391099900007,"entropy":0.5213619843125343},{"step":61,"reward":-5.508768081665039,"core":-1.7888301610946655,"cost":-0.007210202049463987,"kl":0.013457056134939194,"entropy":0.5391527265310287},{"step":63,"reward":-4.834719181060791,"core":-1.53030264377594,"cost":-0.03418834134936333,"kl":0.01273297646548599,"entropy":0.44990822672843933},{"step":65,"reward":-11.800236701965332,"core":-3.8438169956207275,"cost":-0.11267002671957016,"kl":0.015420263400301337,"entropy":0.42713312059640884},{"step":67,"reward":-10.854450225830078,"core":-3.5713958740234375,"cost":-0.05921951308846474,"kl":0.012981677195057273,"entropy":0.5182740539312363},{"step":69,"reward":-14.397419929504395,"core":-4.749277591705322,"cost":-0.08244010806083679,"kl":0.011844618478789926,"entropy":0.4660148173570633},{"step":71,"reward":-8.453408241271973,"core":-2.7617576122283936,"cost":-0.040008265525102615,"kl":0.014139682054519653,"entropy":0.4499261826276779},{"step":73,"reward":-19.667327880859375,"core":-6.455036163330078,"cost":-0.2345653772354126,"kl":0.009426855947822332,"entropy":0.4788353145122528},{"step":75,"reward":-12.226351737976074,"core":-3.998502731323242,"cost":-0.08657166361808777,"kl":0.016723361797630787,"entropy":0.6144375950098038},{"step":77,"reward":-12.451717376708984,"core":-4.102394104003906,"cost":-0.08224431425333023,"kl":0.011726192780770361,"entropy":0.5316344350576401},{"step":79,"reward":-19.117807388305664,"core":-6.285257816314697,"cost":-0.19833408296108246,"kl":0.016447424422949553,"entropy":0.5045108050107956},{"step":81,"reward":-8.519986152648926,"core":-2.7899951934814453,"cost":-0.05924653634428978,"kl":0.023320158943533897,"entropy":0.5668911635875702},{"step":83,"reward":-12.029423713684082,"core":-3.940032958984375,"cost":-0.09303673356771469,"kl":0.019597836770117283,"entropy":0.5212415978312492},{"step":85,"reward":-14.93154525756836,"core":-4.913280963897705,"cost":-0.11474428325891495,"kl":0.013230539858341217,"entropy":0.5180042609572411},{"step":87,"reward":-16.161712646484375,"core":-5.302116870880127,"cost":-0.21354706585407257,"kl":0.016086848452687263,"entropy":0.5764726996421814},{"step":89,"reward":-9.662262916564941,"core":-3.1708614826202393,"cost":-0.05969882011413574,"kl":0.0175120304338634,"entropy":0.5470473617315292},{"step":91,"reward":-12.778302192687988,"core":-4.186822414398193,"cost":-0.10846620798110962,"kl":0.018618641421198845,"entropy":0.6108837723731995},{"step":93,"reward":-7.58147668838501,"core":-2.4492456912994385,"cost":-0.05334590747952461,"kl":0.018314510118216276,"entropy":0.5466235131025314},{"step":95,"reward":-5.086984157562256,"core":-1.6766465902328491,"cost":-0.001004367251880467,"kl":0.015879334416240454,"entropy":0.5756528824567795},{"step":97,"reward":-11.037338256835938,"core":-3.631272554397583,"cost":-0.07119851559400558,"kl":0.014677990693598986,"entropy":0.4600367844104767},{"step":99,"reward":-16.9141788482666,"core":-5.574638366699219,"cost":-0.11157593131065369,"kl":0.014727619243785739,"entropy":0.5829643607139587},{"step":101,"reward":-11.994980812072754,"core":-3.936796188354492,"cost":-0.09786996990442276,"kl":0.00999330054037273,"entropy":0.4969404265284538},{"step":103,"reward":-21.51215171813965,"core":-7.051100254058838,"cost":-0.2571231424808502,"kl":0.010826245183125138,"entropy":0.45497720688581467},{"step":105,"reward":-13.296875,"core":-4.374061584472656,"cost":-0.08060917258262634,"kl":0.014584523160010576,"entropy":0.5986317843198776},{"step":107,"reward":-10.182941436767578,"core":-3.3518686294555664,"cost":-0.059756577014923096,"kl":0.011207905132323503,"entropy":0.4935530200600624},{"step":109,"reward":-9.365412712097168,"core":-3.03826904296875,"cost":-0.09524765610694885,"kl":0.011926152277737856,"entropy":0.419227734208107},{"step":111,"reward":-11.114184379577637,"core":-3.643451690673828,"cost":-0.07454413175582886,"kl":0.010021310299634933,"entropy":0.5297740995883942},{"step":113,"reward":-9.415359497070312,"core":-3.0979416370391846,"cost":-0.010974426753818989,"kl":0.014530812972225249,"entropy":0.48393871635198593},{"step":115,"reward":-25.38498878479004,"core":-8.356402397155762,"cost":-0.2630290389060974,"kl":0.01079586404375732,"entropy":0.4782113805413246},{"step":117,"reward":-17.06353187561035,"core":-5.600819110870361,"cost":-0.15766765177249908,"kl":0.012529927538707852,"entropy":0.42033421248197556},{"step":119,"reward":-6.3843674659729,"core":-2.1042659282684326,"cost":-0.019266964867711067,"kl":0.011493918602354825,"entropy":0.5002636462450027},{"step":121,"reward":-11.703548431396484,"core":-3.830989122390747,"cost":-0.10743105411529541,"kl":0.0055921191815286875,"entropy":0.4775404632091522},{"step":123,"reward":-11.303558349609375,"core":-3.7276954650878906,"cost":-0.0669318214058876,"kl":0.008533533429726958,"entropy":0.5935186892747879},{"step":125,"reward":-9.375859260559082,"core":-3.067638635635376,"cost":-0.07615462690591812,"kl":0.009452017024159431,"entropy":0.5339464247226715},{"step":127,"reward":-8.466471672058105,"core":-2.769757032394409,"cost":-0.02988770790398121,"kl":0.006933600758202374,"entropy":0.46702196449041367},{"step":129,"reward":-8.400067329406738,"core":-2.7648093700408936,"cost":-0.03554077073931694,"kl":0.01057720952667296,"entropy":0.5224549695849419},{"step":131,"reward":-13.18273639678955,"core":-4.353682041168213,"cost":-0.0813896581530571,"kl":0.01021641050465405,"entropy":0.5603727996349335},{"step":133,"reward":-11.665352821350098,"core":-3.8274834156036377,"cost":-0.10264220088720322,"kl":0.012566578574478626,"entropy":0.49418405443429947},{"step":135,"reward":-7.2358880043029785,"core":-2.391688108444214,"cost":0.0,"kl":0.008980143466033041,"entropy":0.5531255304813385},{"step":137,"reward":-9.152057647705078,"core":-3.029916763305664,"cost":-0.008452336303889751,"kl":0.00999562966171652,"entropy":0.5011691600084305},{"step":139,"reward":-9.678844451904297,"core":-3.182874917984009,"cost":-0.032549139112234116,"kl":0.013438134221360087,"entropy":0.5357213914394379},{"step":141,"reward":-12.331794738769531,"core":-4.054635524749756,"cost":-0.12488102912902832,"kl":0.010361484833993018,"entropy":0.4852265492081642},{"step":143,"reward":-7.081919193267822,"core":-2.3314478397369385,"cost":-0.04626308009028435,"kl":0.012571222614496946,"entropy":0.5310428887605667},{"step":145,"reward":-7.087552547454834,"core":-2.3302314281463623,"cost":-0.0025512585416436195,"kl":0.01687880326062441,"entropy":0.48804886639118195},{"step":147,"reward":-13.401775360107422,"core":-4.424810409545898,"cost":-0.07440437376499176,"kl":0.021062499843537807,"entropy":0.5161532238125801},{"step":149,"reward":-15.16390609741211,"core":-5.003452301025391,"cost":-0.11894229799509048,"kl":0.02012004633434117,"entropy":0.48415564000606537},{"step":151,"reward":-9.12136459350586,"core":-2.9901740550994873,"cost":-0.0688561201095581,"kl":0.02751572383567691,"entropy":0.5809275656938553},{"step":153,"reward":-12.312207221984863,"core":-4.048976898193359,"cost":-0.07745841145515442,"kl":0.02126715239137411,"entropy":0.4645843580365181},{"step":155,"reward":-8.24718189239502,"core":-2.7299673557281494,"cost":-0.028167864307761192,"kl":0.024448870215564966,"entropy":0.5039497017860413},{"step":157,"reward":-13.10095500946045,"core":-4.300105571746826,"cost":-0.10929278284311295,"kl":0.029419641941785812,"entropy":0.4868423715233803},{"step":159,"reward":-18.047136306762695,"core":-5.93634033203125,"cost":-0.1952289342880249,"kl":0.02665972337126732,"entropy":0.5094601437449455},{"step":161,"reward":-5.14778470993042,"core":-1.6883693933486938,"cost":0.0,"kl":0.02370448550209403,"entropy":0.4930333122611046},{"step":163,"reward":-14.84740924835205,"core":-4.905223846435547,"cost":-0.10906953364610672,"kl":0.02829855354502797,"entropy":0.5428410172462463},{"step":165,"reward":-7.480217456817627,"core":-2.4645748138427734,"cost":-0.01790226809680462,"kl":0.02934517152607441,"entropy":0.4964246600866318},{"step":167,"reward":-7.407596588134766,"core":-2.4468343257904053,"cost":-0.038177382200956345,"kl":0.026799915824085474,"entropy":0.5073464512825012},{"step":169,"reward":-4.5549139976501465,"core":-1.4838981628417969,"cost":0.0,"kl":0.02560576470568776,"entropy":0.49456847459077835},{"step":171,"reward":-13.987519264221191,"core":-4.598987102508545,"cost":-0.10636524111032486,"kl":0.023883571848273277,"entropy":0.4891743138432503},{"step":173,"reward":-9.42455768585205,"core":-3.104621171951294,"cost":-0.03681517392396927,"kl":0.024772124830633402,"entropy":0.5085639879107475},{"step":175,"reward":-14.281044006347656,"core":-4.6928911209106445,"cost":-0.11672595143318176,"kl":0.020788810681551695,"entropy":0.46600838750600815},{"step":177,"reward":-11.776028633117676,"core":-3.875159502029419,"cost":-0.05521591007709503,"kl":0.01268996438011527,"entropy":0.4256296306848526},{"step":179,"reward":-7.963174343109131,"core":-2.5768864154815674,"cost":-0.03529723733663559,"kl":0.02083678194321692,"entropy":0.41847478598356247},{"step":181,"reward":-13.703857421875,"core":-4.50705623626709,"cost":-0.12184297293424606,"kl":0.014185997657477856,"entropy":0.431877464056015},{"step":183,"reward":-5.652442455291748,"core":-1.81631600856781,"cost":-0.023729538545012474,"kl":0.023185070604085922,"entropy":0.3998090922832489},{"step":185,"reward":-13.53128719329834,"core":-4.455343723297119,"cost":-0.09973336011171341,"kl":0.01542136026546359,"entropy":0.45555775612592697},{"step":187,"reward":-7.738572597503662,"core":-2.530893325805664,"cost":-0.02324547804892063,"kl":0.02098140213638544,"entropy":0.5540119484066963},{"step":189,"reward":-11.163688659667969,"core":-3.6712138652801514,"cost":-0.08916449546813965,"kl":0.025125660467892885,"entropy":0.48300478607416153},{"step":191,"reward":-8.661184310913086,"core":-2.830099105834961,"cost":-0.027025757357478142,"kl":0.014877234818413854,"entropy":0.45503073185682297},{"step":193,"reward":-8.302404403686523,"core":-2.7169151306152344,"cost":-0.026994353160262108,"kl":0.01920521818101406,"entropy":0.4998120591044426},{"step":195,"reward":-10.917023658752441,"core":-3.575467348098755,"cost":-0.08447844535112381,"kl":0.02143864706158638,"entropy":0.4829547256231308},{"step":197,"reward":-4.756866455078125,"core":-1.5407835245132446,"cost":-0.0035059747751802206,"kl":0.021757776383310556,"entropy":0.5316720455884933},{"step":199,"reward":-6.250984191894531,"core":-2.0304672718048096,"cost":-0.016785984858870506,"kl":0.017479847418144345,"entropy":0.5058234184980392},{"step":201,"reward":-7.119095325469971,"core":-2.342848539352417,"cost":-0.0366530604660511,"kl":0.02068306040018797,"entropy":0.4372509568929672},{"step":203,"reward":-7.1527276039123535,"core":-2.3281469345092773,"cost":-0.03374764695763588,"kl":0.015290816780179739,"entropy":0.5044994503259659},{"step":205,"reward":-7.073554515838623,"core":-2.314969062805176,"cost":-0.03586262837052345,"kl":0.014782795449718833,"entropy":0.5269734859466553},{"step":207,"reward":-6.972483158111572,"core":-2.2649755477905273,"cost":-0.021565625444054604,"kl":0.012548717902973294,"entropy":0.49775275588035583},{"step":209,"reward":-18.926513671875,"core":-6.227756977081299,"cost":-0.21277247369289398,"kl":0.011995218694210052,"entropy":0.48464710265398026},{"step":211,"reward":-14.723617553710938,"core":-4.848113536834717,"cost":-0.12514036893844604,"kl":0.014114189893007278,"entropy":0.5043743923306465},{"step":213,"reward":-9.82869815826416,"core":-3.2264652252197266,"cost":-0.03866409882903099,"kl":0.010912269353866577,"entropy":0.480782687664032},{"step":215,"reward":-15.316458702087402,"core":-5.04511833190918,"cost":-0.12760122120380402,"kl":0.012579736765474081,"entropy":0.52044328302145},{"step":217,"reward":-6.991041660308838,"core":-2.2949769496917725,"cost":-0.020475199446082115,"kl":0.014435472316108644,"entropy":0.4932412803173065},{"step":219,"reward":-10.552088737487793,"core":-3.475947141647339,"cost":-0.026169156655669212,"kl":0.013093942776322365,"entropy":0.5383106023073196},{"step":221,"reward":-8.33306884765625,"core":-2.7025129795074463,"cost":-0.08203848451375961,"kl":0.015454883454367518,"entropy":0.5514265298843384},{"step":223,"reward":-10.314433097839355,"core":-3.367879867553711,"cost":-0.10112785547971725,"kl":0.008609827258624136,"entropy":0.60493303835392},{"step":225,"reward":-12.938260078430176,"core":-4.265727996826172,"cost":-0.08721183985471725,"kl":0.011752883205190301,"entropy":0.5544285774230957},{"step":227,"reward":-11.958808898925781,"core":-3.937967538833618,"cost":-0.035602912306785583,"kl":0.00999602279625833,"entropy":0.5079920887947083},{"step":229,"reward":-18.022153854370117,"core":-5.930076599121094,"cost":-0.19805912673473358,"kl":0.012125087901949883,"entropy":0.4849204272031784},{"step":231,"reward":-15.145263671875,"core":-4.957123279571533,"cost":-0.16907788813114166,"kl":0.010877491673454642,"entropy":0.47290360927581787},{"step":233,"reward":-7.225719451904297,"core":-2.36895751953125,"cost":-0.01709640771150589,"kl":0.017857942264527082,"entropy":0.5507120788097382},{"step":235,"reward":-13.320984840393066,"core":-4.3558478355407715,"cost":-0.13132049143314362,"kl":0.014140303246676922,"entropy":0.5830952972173691},{"step":237,"reward":-4.676139831542969,"core":-1.5339956283569336,"cost":0.0,"kl":0.014144246699288487,"entropy":0.5282184779644012},{"step":239,"reward":-10.359360694885254,"core":-3.377335786819458,"cost":-0.09023789316415787,"kl":0.013565610395744443,"entropy":0.48894745111465454},{"step":241,"reward":-7.926740646362305,"core":-2.5807180404663086,"cost":-0.046662867069244385,"kl":0.0136002276558429,"entropy":0.5861821621656418},{"step":243,"reward":-7.8081231117248535,"core":-2.5561439990997314,"cost":-0.027401015162467957,"kl":0.011248578084632754,"entropy":0.5319816023111343},{"step":245,"reward":-15.524819374084473,"core":-5.088733196258545,"cost":-0.13931715488433838,"kl":0.011171302059665322,"entropy":0.4940377026796341},{"step":247,"reward":-11.544222831726074,"core":-3.768892526626587,"cost":-0.13815568387508392,"kl":0.01442858437076211,"entropy":0.5466073900461197},{"step":249,"reward":-8.028084754943848,"core":-2.6185429096221924,"cost":-0.028747381642460823,"kl":0.019488368183374405,"entropy":0.487426221370697},{"step":251,"reward":-10.15335750579834,"core":-3.317207098007202,"cost":-0.0705694705247879,"kl":0.016201489372178912,"entropy":0.4794340580701828},{"step":253,"reward":-8.8836030960083,"core":-2.923463821411133,"cost":-0.047338444739580154,"kl":0.01596100302413106,"entropy":0.5035249441862106},{"step":255,"reward":-8.830018043518066,"core":-2.894282579421997,"cost":-0.03307707607746124,"kl":0.011862614890560508,"entropy":0.521396242082119},{"step":257,"reward":-10.9374418258667,"core":-3.5833027362823486,"cost":-0.09824217110872269,"kl":0.01176004484295845,"entropy":0.4839414730668068},{"step":259,"reward":-5.458015441894531,"core":-1.7423118352890015,"cost":-0.011039483360946178,"kl":0.013778942404314876,"entropy":0.45897671580314636},{"step":261,"reward":-21.58973503112793,"core":-7.088235855102539,"cost":-0.2775760591030121,"kl":0.011931080836802721,"entropy":0.4961124286055565},{"step":263,"reward":-16.627138137817383,"core":-5.4264702796936035,"cost":-0.1961825042963028,"kl":0.015560093335807323,"entropy":0.4936985597014427},{"step":265,"reward":-8.788058280944824,"core":-2.8907134532928467,"cost":-0.01824723370373249,"kl":0.0146204789634794,"entropy":0.5115610882639885},{"step":267,"reward":-8.251819610595703,"core":-2.721376419067383,"cost":-0.010936003178358078,"kl":0.017695054411888123,"entropy":0.4315740689635277},{"step":269,"reward":-8.639229774475098,"core":-2.8248507976531982,"cost":-0.06593433022499084,"kl":0.017166030127555132,"entropy":0.5383455604314804},{"step":271,"reward":-9.309885025024414,"core":-3.064234972000122,"cost":-0.03949068486690521,"kl":0.014595200307667255,"entropy":0.5149170309305191},{"step":273,"reward":-8.870768547058105,"core":-2.9179115295410156,"cost":-0.01955585554242134,"kl":0.028167505748569965,"entropy":0.5318680256605148},{"step":275,"reward":-6.04990816116333,"core":-1.955395221710205,"cost":-0.0386083759367466,"kl":0.016239545308053493,"entropy":0.5249025523662567},{"step":277,"reward":-10.793004035949707,"core":-3.540771484375,"cost":-0.07915721833705902,"kl":0.015858728205785155,"entropy":0.591452419757843},{"step":279,"reward":-12.959144592285156,"core":-4.273570537567139,"cost":-0.03359687328338623,"kl":0.022346994373947382,"entropy":0.5066563561558723},{"step":281,"reward":-8.523728370666504,"core":-2.7960236072540283,"cost":-0.020533202216029167,"kl":0.014752611052244902,"entropy":0.5286757946014404},{"step":283,"reward":-19.633771896362305,"core":-6.444482803344727,"cost":-0.25017282366752625,"kl":0.014827910112217069,"entropy":0.530550166964531},{"step":285,"reward":-11.368141174316406,"core":-3.7412233352661133,"cost":-0.03826485574245453,"kl":0.011047267122194171,"entropy":0.5268106758594513},{"step":287,"reward":-12.712753295898438,"core":-4.176371097564697,"cost":-0.08927146345376968,"kl":0.011848277878016233,"entropy":0.4356679618358612},{"step":289,"reward":-5.566675662994385,"core":-1.7563632726669312,"cost":-0.01900390535593033,"kl":0.013969217659905553,"entropy":0.46524258702993393},{"step":291,"reward":-7.596381664276123,"core":-2.4752652645111084,"cost":-0.04347316920757294,"kl":0.013691934291273355,"entropy":0.525072306394577},{"step":293,"reward":-12.6386079788208,"core":-4.1493000984191895,"cost":-0.10082327574491501,"kl":0.011188342235982418,"entropy":0.5423456281423569},{"step":295,"reward":-7.8862624168396,"core":-2.5900580883026123,"cost":-0.0158249344676733,"kl":0.015599915757775307,"entropy":0.49881819635629654},{"step":297,"reward":-8.360042572021484,"core":-2.7040956020355225,"cost":-0.08208667486906052,"kl":0.017512777354568243,"entropy":0.49109997600317},{"step":299,"reward":-10.237744331359863,"core":-3.369661569595337,"cost":-0.059661585837602615,"kl":0.017952999798581004,"entropy":0.5342877432703972},{"step":301,"reward":-9.66257095336914,"core":-3.164598226547241,"cost":-0.06652507930994034,"kl":0.01837855507619679,"entropy":0.5829441696405411},{"step":303,"reward":-5.5880818367004395,"core":-1.8135565519332886,"cost":-0.007814700715243816,"kl":0.014437154866755009,"entropy":0.586362436413765},{"step":305,"reward":-5.31298303604126,"core":-1.729392170906067,"cost":-0.001926226424984634,"kl":0.0243201507255435,"entropy":0.5286571606993675},{"step":307,"reward":-17.726045608520508,"core":-5.830451488494873,"cost":-0.16107822954654694,"kl":0.01252437406219542,"entropy":0.5160445496439934},{"step":309,"reward":-7.728842258453369,"core":-2.544464111328125,"cost":-0.019087499007582664,"kl":0.014015517197549343,"entropy":0.5625783354043961},{"step":311,"reward":-8.01838207244873,"core":-2.6151068210601807,"cost":-0.04527899995446205,"kl":0.016430665273219347,"entropy":0.5468702614307404},{"step":313,"reward":-8.33012866973877,"core":-2.7169411182403564,"cost":-0.045984163880348206,"kl":0.020069819409400225,"entropy":0.46929801255464554},{"step":315,"reward":-9.976994514465332,"core":-3.2671306133270264,"cost":-0.01600772701203823,"kl":0.013711552834138274,"entropy":0.5169558525085449},{"step":317,"reward":-5.506366729736328,"core":-1.791393756866455,"cost":0.0,"kl":0.02191845327615738,"entropy":0.4851624667644501},{"step":319,"reward":-7.925685882568359,"core":-2.5946128368377686,"cost":-0.03285440430045128,"kl":0.016433876706287265,"entropy":0.48921550810337067},{"step":321,"reward":-4.873514652252197,"core":-1.5588464736938477,"cost":-0.024029957130551338,"kl":0.010744812898337841,"entropy":0.46270883083343506},{"step":323,"reward":-10.21277141571045,"core":-3.3545405864715576,"cost":-0.059535253793001175,"kl":0.011035785311833024,"entropy":0.4374084174633026},{"step":325,"reward":-11.480781555175781,"core":-3.77909779548645,"cost":-0.09209736436605453,"kl":0.015801507513970137,"entropy":0.6590010151267052},{"step":327,"reward":-11.51512622833252,"core":-3.793970823287964,"cost":-0.0678860992193222,"kl":0.011052371934056282,"entropy":0.4126812815666199},{"step":329,"reward":-11.59149169921875,"core":-3.7976768016815186,"cost":-0.09615959972143173,"kl":0.013081840937957168,"entropy":0.590517446398735},{"step":331,"reward":-22.442794799804688,"core":-7.412073612213135,"cost":-0.2109588235616684,"kl":0.013700853800401092,"entropy":0.5307704210281372},{"step":333,"reward":-8.546510696411133,"core":-2.7770321369171143,"cost":-0.07332620024681091,"kl":0.012958520790562034,"entropy":0.5366480797529221},{"step":335,"reward":-12.759366035461426,"core":-4.154299259185791,"cost":-0.14356757700443268,"kl":0.012931483332067728,"entropy":0.5277454033493996},{"step":337,"reward":-11.43859577178955,"core":-3.7791900634765625,"cost":-0.06061801314353943,"kl":0.016383894253522158,"entropy":0.628344714641571},{"step":339,"reward":-10.94411563873291,"core":-3.56361985206604,"cost":-0.07690192013978958,"kl":0.011585697997361422,"entropy":0.478048712015152},{"step":341,"reward":-7.781251907348633,"core":-2.5212743282318115,"cost":-0.0802440270781517,"kl":0.01925750821828842,"entropy":0.4863707274198532},{"step":343,"reward":-17.31267738342285,"core":-5.702564239501953,"cost":-0.17062145471572876,"kl":0.01752048614434898,"entropy":0.5380338579416275},{"step":345,"reward":-10.567978858947754,"core":-3.4784364700317383,"cost":-0.05899911746382713,"kl":0.016659998800605536,"entropy":0.5459039360284805},{"step":347,"reward":-7.561009883880615,"core":-2.4219162464141846,"cost":-0.02950984053313732,"kl":0.017647989094257355,"entropy":0.46663615852594376},{"step":349,"reward":-9.967124938964844,"core":-3.2359778881073,"cost":-0.07927169650793076,"kl":0.017426026752218604,"entropy":0.5153714790940285},{"step":351,"reward":-8.173355102539062,"core":-2.6795480251312256,"cost":-0.03489597514271736,"kl":0.0176696740090847,"entropy":0.5156508684158325},{"step":353,"reward":-7.540566921234131,"core":-2.4501535892486572,"cost":-0.0407741516828537,"kl":0.02063132682815194,"entropy":0.49536918848752975},{"step":355,"reward":-10.886902809143066,"core":-3.5213563442230225,"cost":-0.16025276482105255,"kl":0.022724237758666277,"entropy":0.4640393406152725},{"step":357,"reward":-8.018918991088867,"core":-2.5840911865234375,"cost":-0.053641319274902344,"kl":0.021780282724648714,"entropy":0.47801338881254196},{"step":359,"reward":-15.15490436553955,"core":-4.9829421043396,"cost":-0.1245037317276001,"kl":0.016191462287679315,"entropy":0.43533754348754883},{"step":361,"reward":-8.20981502532959,"core":-2.69777512550354,"cost":-0.021594258025288582,"kl":0.014353537000715733,"entropy":0.5146594420075417},{"step":363,"reward":-12.322502136230469,"core":-4.076126575469971,"cost":-0.06209355220198631,"kl":0.021474292036145926,"entropy":0.5827806442975998},{"step":365,"reward":-7.9494948387146,"core":-2.6158416271209717,"cost":-0.04333223029971123,"kl":0.02654342632740736,"entropy":0.5515987277030945},{"step":367,"reward":-12.473891258239746,"core":-4.102381706237793,"cost":-0.03454633057117462,"kl":0.01639930810779333,"entropy":0.47843771427869797},{"step":369,"reward":-7.979917049407959,"core":-2.602254629135132,"cost":-0.03369053825736046,"kl":0.01642168709076941,"entropy":0.5181884318590164},{"step":371,"reward":-11.202128410339355,"core":-3.7070682048797607,"cost":-0.035916149616241455,"kl":0.027942472137510777,"entropy":0.5089334025979042},{"step":373,"reward":-6.343796253204346,"core":-2.0430519580841064,"cost":-0.057430725544691086,"kl":0.015851816395297647,"entropy":0.5330184400081635},{"step":375,"reward":-11.139752388000488,"core":-3.6655731201171875,"cost":-0.03910169377923012,"kl":0.017679642885923386,"entropy":0.5527909249067307},{"step":377,"reward":-7.6816582679748535,"core":-2.5103352069854736,"cost":-0.05307379364967346,"kl":0.019830288365483284,"entropy":0.50645961612463},{"step":379,"reward":-9.313216209411621,"core":-3.032355308532715,"cost":-0.07065794616937637,"kl":0.023580276407301426,"entropy":0.48738182336091995},{"step":381,"reward":-11.33942699432373,"core":-3.6881206035614014,"cost":-0.141911581158638,"kl":0.021720158401876688,"entropy":0.5588598251342773},{"step":383,"reward":-13.735037803649902,"core":-4.517704486846924,"cost":-0.1244218572974205,"kl":0.024021313525736332,"entropy":0.539661318063736},{"step":385,"reward":-6.124125003814697,"core":-1.9717334508895874,"cost":-0.020105253905057907,"kl":0.016520330915227532,"entropy":0.5218542069196701},{"step":387,"reward":-3.467228651046753,"core":-1.108618140220642,"cost":0.0,"kl":0.016388228395953774,"entropy":0.48695439100265503},{"step":389,"reward":-14.514659881591797,"core":-4.757134914398193,"cost":-0.10869058966636658,"kl":0.012168389046564698,"entropy":0.4585587754845619},{"step":391,"reward":-9.881335258483887,"core":-3.2578423023223877,"cost":-0.03892480954527855,"kl":0.013418469578027725,"entropy":0.5667886883020401},{"step":393,"reward":-12.444987297058105,"core":-4.055624008178711,"cost":-0.1482553333044052,"kl":0.013374669011682272,"entropy":0.4950016885995865},{"step":395,"reward":-9.114161491394043,"core":-2.9957783222198486,"cost":-0.04277653619647026,"kl":0.017083894927054644,"entropy":0.538595661520958},{"step":397,"reward":-10.052361488342285,"core":-3.292198896408081,"cost":-0.039481449872255325,"kl":0.020166601752862334,"entropy":0.49331002682447433},{"step":399,"reward":-5.356916904449463,"core":-1.7230366468429565,"cost":0.0,"kl":0.012898645363748074,"entropy":0.49245819449424744},{"step":401,"reward":-11.13448715209961,"core":-3.6604690551757812,"cost":-0.07162852585315704,"kl":0.015114447334781289,"entropy":0.5253636240959167},{"step":403,"reward":-10.540058135986328,"core":-3.4257984161376953,"cost":-0.09078576415777206,"kl":0.012979291146621108,"entropy":0.5055523365736008},{"step":405,"reward":-4.943602085113525,"core":-1.590822696685791,"cost":0.0,"kl":0.019928710535168648,"entropy":0.5270651876926422},{"step":407,"reward":-12.885643005371094,"core":-4.259562015533447,"cost":-0.08154777437448502,"kl":0.014042992610484362,"entropy":0.568707138299942},{"step":409,"reward":-7.86884069442749,"core":-2.5803403854370117,"cost":-0.03105045296251774,"kl":0.024866527877748013,"entropy":0.52472273260355},{"step":411,"reward":-9.04357624053955,"core":-2.93802809715271,"cost":-0.05768762156367302,"kl":0.012624365510419011,"entropy":0.4395340085029602},{"step":413,"reward":-2.7637996673583984,"core":-0.8417637944221497,"cost":0.0,"kl":0.03912512492388487,"entropy":0.4938315823674202},{"step":415,"reward":-6.105230808258057,"core":-1.9950023889541626,"cost":-0.021040355786681175,"kl":0.01965324254706502,"entropy":0.527717188000679},{"step":417,"reward":-14.196602821350098,"core":-4.632848262786865,"cost":-0.1484372764825821,"kl":0.01967551652342081,"entropy":0.43260888010263443},{"step":419,"reward":-8.984066009521484,"core":-2.94572377204895,"cost":-0.06488656997680664,"kl":0.0211869936902076,"entropy":0.46623823046684265},{"step":421,"reward":-9.576539039611816,"core":-3.1418721675872803,"cost":-0.04196612164378166,"kl":0.016431751661002636,"entropy":0.4550245478749275},{"step":423,"reward":-25.849267959594727,"core":-8.483075141906738,"cost":-0.3452918827533722,"kl":0.023431597976014018,"entropy":0.4819900244474411},{"step":425,"reward":-8.064586639404297,"core":-2.6238973140716553,"cost":-0.023134062066674232,"kl":0.021534343250095844,"entropy":0.45064184814691544},{"step":427,"reward":-7.787025451660156,"core":-2.5401554107666016,"cost":-0.04922783002257347,"kl":0.017053320072591305,"entropy":0.5387832671403885}],"nonzero_advantage":[{"step":1,"nonzero":0.0},{"step":3,"nonzero":0.0},{"step":5,"nonzero":0.0},{"step":7,"nonzero":0.0},{"step":9,"nonzero":0.0},{"step":11,"nonzero":0.041666666666666664},{"step":13,"nonzero":0.14583333333333334},{"step":15,"nonzero":0.0},{"step":17,"nonzero":0.0},{"step":19,"nonzero":0.08333333333333333},{"step":21,"nonzero":0.125},{"step":23,"nonzero":0.3125},{"step":25,"nonzero":0.0},{"step":27,"nonzero":0.0},{"step":29,"nonzero":0.0},{"step":31,"nonzero":0.0},{"step":33,"nonzero":0.0},{"step":35,"nonzero":0.08333333333333333},{"step":37,"nonzero":0.0},{"step":39,"nonzero":0.0},{"step":41,"nonzero":0.0},{"step":43,"nonzero":0.0},{"step":45,"nonzero":0.041666666666666664},{"step":47,"nonzero":0.0},{"step":49,"nonzero":0.0},{"step":51,"nonzero":0.0},{"step":53,"nonzero":0.0},{"step":55,"nonzero":0.20833333333333334},{"step":57,"nonzero":0.14583333333333334},{"step":59,"nonzero":0.0},{"step":61,"nonzero":0.08333333333333333},{"step":63,"nonzero":0.0},{"step":65,"nonzero":0.0},{"step":67,"nonzero":0.0},{"step":69,"nonzero":0.0},{"step":71,"nonzero":0.0},{"step":73,"nonzero":0.0625},{"step":75,"nonzero":0.041666666666666664},{"step":77,"nonzero":0.0},{"step":79,"nonzero":0.0},{"step":81,"nonzero":0.0},{"step":83,"nonzero":0.041666666666666664},{"step":85,"nonzero":0.0},{"step":87,"nonzero":0.0},{"step":89,"nonzero":0.125},{"step":91,"nonzero":0.125},{"step":93,"nonzero":0.0},{"step":95,"nonzero":0.0},{"step":97,"nonzero":0.14583333333333334},{"step":99,"nonzero":0.0},{"step":101,"nonzero":0.0},{"step":103,"nonzero":0.0},{"step":105,"nonzero":0.0},{"step":107,"nonzero":0.0},{"step":109,"nonzero":0.0},{"step":111,"nonzero":0.0},{"step":113,"nonzero":0.0},{"step":115,"nonzero":0.0},{"step":117,"nonzero":0.0},{"step":119,"nonzero":0.041666666666666664},{"step":121,"nonzero":0.10416666666666667},{"step":123,"nonzero":0.0},{"step":125,"nonzero":0.0},{"step":127,"nonzero":0.0},{"step":129,"nonzero":0.14583333333333334},{"step":131,"nonzero":0.041666666666666664},{"step":133,"nonzero":0.2708333333333333},{"step":135,"nonzero":0.0},{"step":137,"nonzero":0.0},{"step":139,"nonzero":0.0},{"step":141,"nonzero":0.041666666666666664},{"step":143,"nonzero":0.0},{"step":145,"nonzero":0.0},{"step":147,"nonzero":0.10416666666666667},{"step":149,"nonzero":0.041666666666666664},{"step":151,"nonzero":0.125},{"step":153,"nonzero":0.0},{"step":155,"nonzero":0.125},{"step":157,"nonzero":0.0},{"step":159,"nonzero":0.0},{"step":161,"nonzero":0.10416666666666667},{"step":163,"nonzero":0.0},{"step":165,"nonzero":0.0},{"step":167,"nonzero":0.125},{"step":169,"nonzero":0.125},{"step":171,"nonzero":0.0},{"step":173,"nonzero":0.0},{"step":175,"nonzero":0.0},{"step":177,"nonzero":0.08333333333333333},{"step":179,"nonzero":0.10416666666666667},{"step":181,"nonzero":0.0},{"step":183,"nonzero":0.08333333333333333},{"step":185,"nonzero":0.0},{"step":187,"nonzero":0.0},{"step":189,"nonzero":0.0},{"step":191,"nonzero":0.0},{"step":193,"nonzero":0.0},{"step":195,"nonzero":0.14583333333333334},{"step":197,"nonzero":0.16666666666666666},{"step":199,"nonzero":0.0},{"step":201,"nonzero":0.22916666666666666},{"step":203,"nonzero":0.0},{"step":205,"nonzero":0.0},{"step":207,"nonzero":0.0},{"step":209,"nonzero":0.0},{"step":211,"nonzero":0.0},{"step":213,"nonzero":0.0},{"step":215,"nonzero":0.041666666666666664},{"step":217,"nonzero":0.041666666666666664},{"step":219,"nonzero":0.0},{"step":221,"nonzero":0.1875},{"step":223,"nonzero":0.0},{"step":225,"nonzero":0.0},{"step":227,"nonzero":0.0625},{"step":229,"nonzero":0.1875},{"step":231,"nonzero":0.0},{"step":233,"nonzero":0.22916666666666666},{"step":235,"nonzero":0.20833333333333334},{"step":237,"nonzero":0.10416666666666667},{"step":239,"nonzero":0.0},{"step":241,"nonzero":0.16666666666666666},{"step":243,"nonzero":0.041666666666666664},{"step":245,"nonzero":0.0},{"step":247,"nonzero":0.0},{"step":249,"nonzero":0.0},{"step":251,"nonzero":0.0},{"step":253,"nonzero":0.041666666666666664},{"step":255,"nonzero":0.10416666666666667},{"step":257,"nonzero":0.0},{"step":259,"nonzero":0.0625},{"step":261,"nonzero":0.0},{"step":263,"nonzero":0.0},{"step":265,"nonzero":0.0},{"step":267,"nonzero":0.08333333333333333},{"step":269,"nonzero":0.0},{"step":271,"nonzero":0.0},{"step":273,"nonzero":0.0},{"step":275,"nonzero":0.0},{"step":277,"nonzero":0.0625},{"step":279,"nonzero":0.0},{"step":281,"nonzero":0.0},{"step":283,"nonzero":0.0},{"step":285,"nonzero":0.0},{"step":287,"nonzero":0.10416666666666667},{"step":289,"nonzero":0.0},{"step":291,"nonzero":0.08333333333333333},{"step":293,"nonzero":0.0},{"step":295,"nonzero":0.125},{"step":297,"nonzero":0.0},{"step":299,"nonzero":0.2916666666666667},{"step":301,"nonzero":0.0},{"step":303,"nonzero":0.041666666666666664},{"step":305,"nonzero":0.10416666666666667},{"step":307,"nonzero":0.0},{"step":309,"nonzero":0.0},{"step":311,"nonzero":0.041666666666666664},{"step":313,"nonzero":0.0},{"step":315,"nonzero":0.0},{"step":317,"nonzero":0.0},{"step":319,"nonzero":0.0},{"step":321,"nonzero":0.0},{"step":323,"nonzero":0.0},{"step":325,"nonzero":0.125},{"step":327,"nonzero":0.0},{"step":329,"nonzero":0.0},{"step":331,"nonzero":0.10416666666666667},{"step":333,"nonzero":0.0},{"step":335,"nonzero":0.0},{"step":337,"nonzero":0.0},{"step":339,"nonzero":0.0},{"step":341,"nonzero":0.125},{"step":343,"nonzero":0.0},{"step":345,"nonzero":0.10416666666666667},{"step":347,"nonzero":0.0},{"step":349,"nonzero":0.0},{"step":351,"nonzero":0.0},{"step":353,"nonzero":0.041666666666666664},{"step":355,"nonzero":0.0625},{"step":357,"nonzero":0.0},{"step":359,"nonzero":0.0},{"step":361,"nonzero":0.0},{"step":363,"nonzero":0.10416666666666667},{"step":365,"nonzero":0.0},{"step":367,"nonzero":0.0},{"step":369,"nonzero":0.125},{"step":371,"nonzero":0.0},{"step":373,"nonzero":0.0},{"step":375,"nonzero":0.14583333333333334},{"step":377,"nonzero":0.0},{"step":379,"nonzero":0.08333333333333333},{"step":381,"nonzero":0.08333333333333333},{"step":383,"nonzero":0.0},{"step":385,"nonzero":0.0},{"step":387,"nonzero":0.1875},{"step":389,"nonzero":0.08333333333333333},{"step":391,"nonzero":0.0},{"step":393,"nonzero":0.125},{"step":395,"nonzero":0.125},{"step":397,"nonzero":0.0},{"step":399,"nonzero":0.14583333333333334},{"step":401,"nonzero":0.0625},{"step":403,"nonzero":0.0},{"step":405,"nonzero":0.125},{"step":407,"nonzero":0.0},{"step":409,"nonzero":0.10416666666666667},{"step":411,"nonzero":0.0},{"step":413,"nonzero":0.3333333333333333},{"step":415,"nonzero":0.0},{"step":417,"nonzero":0.0},{"step":419,"nonzero":0.0},{"step":421,"nonzero":0.08333333333333333},{"step":423,"nonzero":0.0},{"step":425,"nonzero":0.08333333333333333},{"step":427,"nonzero":0.0}]},{"id":"b-nofloor","label":"No floor (first slice)","run_id":"1791130455","cpcv_floor":null,"train_steps":[{"step":1,"reward":-13.691131591796875,"core":-4.519927978515625,"cost":-0.08229321986436844,"kl":0.0,"entropy":0.46514444798231125},{"step":3,"reward":-5.732156276702881,"core":-1.8612264394760132,"cost":-0.008642680011689663,"kl":0.0007960743532748893,"entropy":0.4861450120806694},{"step":5,"reward":-17.342926025390625,"core":-5.711109638214111,"cost":-0.12990659475326538,"kl":0.0037555533926934004,"entropy":0.4398748651146889},{"step":7,"reward":-12.992416381835938,"core":-4.289566516876221,"cost":-0.07803010195493698,"kl":0.004631844465620816,"entropy":0.4698479175567627},{"step":9,"reward":-8.925400733947754,"core":-2.9104626178741455,"cost":-0.07186386734247208,"kl":0.004296157800126821,"entropy":0.4560704529285431},{"step":11,"reward":-7.450191497802734,"core":-2.428877830505371,"cost":-0.016624240204691887,"kl":0.007291797082871199,"entropy":0.42180417478084564},{"step":13,"reward":-11.026888847351074,"core":-3.623194456100464,"cost":-0.042639583349227905,"kl":0.007102656294591725,"entropy":0.4161031097173691},{"step":15,"reward":-11.341644287109375,"core":-3.728307008743286,"cost":-0.08703258633613586,"kl":0.00850693613756448,"entropy":0.4361537843942642},{"step":17,"reward":-7.5530524253845215,"core":-2.45560622215271,"cost":-0.04594432935118675,"kl":0.007382533163763583,"entropy":0.448491707444191},{"step":19,"reward":-8.5822114944458,"core":-2.8130741119384766,"cost":-0.047038350254297256,"kl":0.008652610937133431,"entropy":0.39086223393678665},{"step":21,"reward":-5.05176305770874,"core":-1.6419657468795776,"cost":-0.010446167550981045,"kl":0.015139983035624027,"entropy":0.47534219920635223},{"step":23,"reward":-11.705780029296875,"core":-3.85662841796875,"cost":-0.09535673260688782,"kl":0.016302991192787886,"entropy":0.4744702875614166},{"step":25,"reward":-10.09771728515625,"core":-3.275442123413086,"cost":-0.09592191129922867,"kl":0.017532648285850883,"entropy":0.45625410974025726},{"step":27,"reward":-8.421649932861328,"core":-2.7372047901153564,"cost":-0.04043639078736305,"kl":0.021124725695699453,"entropy":0.4479302763938904},{"step":29,"reward":-14.60486888885498,"core":-4.812819957733154,"cost":-0.10648778825998306,"kl":0.02545771235600114,"entropy":0.5137404352426529},{"step":31,"reward":-6.1559319496154785,"core":-1.9981664419174194,"cost":-0.012858600355684757,"kl":0.03113712603226304,"entropy":0.5609628856182098},{"step":33,"reward":-12.84476089477539,"core":-4.220502853393555,"cost":-0.0898505374789238,"kl":0.04107401939108968,"entropy":0.5024835616350174},{"step":35,"reward":-3.4668476581573486,"core":-1.0823062658309937,"cost":0.0,"kl":0.03839506581425667,"entropy":0.5193444043397903},{"step":37,"reward":-5.818578243255615,"core":-1.891891598701477,"cost":-0.00405177753418684,"kl":0.037898814771324396,"entropy":0.5669437348842621},{"step":39,"reward":-5.384666919708252,"core":-1.7514253854751587,"cost":0.0,"kl":0.036683617159724236,"entropy":0.5173253938555717},{"step":41,"reward":-5.066471576690674,"core":-1.6312371492385864,"cost":-0.0005721177440136671,"kl":0.08595569990575314,"entropy":0.5640025660395622},{"step":43,"reward":-4.294862270355225,"core":-1.3732045888900757,"cost":0.0,"kl":0.07404555939137936,"entropy":0.6439212262630463},{"step":45,"reward":-8.478031158447266,"core":-2.7869224548339844,"cost":-0.03336978331208229,"kl":0.10926069878041744,"entropy":0.5731787234544754},{"step":47,"reward":-7.439632415771484,"core":-2.442840337753296,"cost":-0.013531153090298176,"kl":0.11389736644923687,"entropy":0.5651534050703049},{"step":49,"reward":-3.3519599437713623,"core":-1.0687730312347412,"cost":0.0,"kl":0.130129748955369,"entropy":0.5788684338331223},{"step":51,"reward":-5.745908260345459,"core":-1.863326072692871,"cost":0.0,"kl":0.10577390156686306,"entropy":0.5499123781919479},{"step":53,"reward":-4.4984636306762695,"core":-1.4352527856826782,"cost":-0.010559548623859882,"kl":0.09495505876839161,"entropy":0.523258313536644},{"step":55,"reward":-4.391844272613525,"core":-1.4410187005996704,"cost":0.0,"kl":0.09407209791243076,"entropy":0.6230510324239731},{"step":57,"reward":-6.04340124130249,"core":-1.9277673959732056,"cost":-0.0311893317848444,"kl":0.06567546539008617,"entropy":0.4702662378549576},{"step":59,"reward":-4.611299991607666,"core":-1.4740978479385376,"cost":0.0,"kl":0.10156712681055069,"entropy":0.576071560382843},{"step":61,"reward":-2.9532830715179443,"core":-0.9125776290893555,"cost":0.0,"kl":0.10807236097753048,"entropy":0.5613571777939796},{"step":63,"reward":-4.647918224334717,"core":-1.4691721200942993,"cost":-0.04816541448235512,"kl":0.08273372612893581,"entropy":0.45967576652765274},{"step":65,"reward":-5.663008213043213,"core":-1.8262192010879517,"cost":-0.0027458108961582184,"kl":0.09851275570690632,"entropy":0.40451954305171967},{"step":67,"reward":-5.852113246917725,"core":-1.9049468040466309,"cost":0.0,"kl":0.08926944248378277,"entropy":0.4674045443534851},{"step":69,"reward":-5.514606475830078,"core":-1.7940720319747925,"cost":-0.003392001846805215,"kl":0.07659674808382988,"entropy":0.4463362544775009},{"step":71,"reward":-3.5871078968048096,"core":-1.1051554679870605,"cost":0.0,"kl":0.06745073106139898,"entropy":0.3604698181152344},{"step":73,"reward":-9.6743803024292,"core":-3.134214401245117,"cost":-0.05455264821648598,"kl":0.10672278329730034,"entropy":0.40220722556114197},{"step":75,"reward":-5.34996223449707,"core":-1.7343448400497437,"cost":0.0,"kl":0.1098023522645235,"entropy":0.5303622633218765},{"step":77,"reward":-7.532273769378662,"core":-2.4536426067352295,"cost":-0.03442602977156639,"kl":0.08631473779678345,"entropy":0.4667927399277687},{"step":79,"reward":-11.665484428405762,"core":-3.812822103500366,"cost":-0.10500118881464005,"kl":0.07773665059357882,"entropy":0.425196573138237},{"step":81,"reward":-3.553908586502075,"core":-1.105138897895813,"cost":0.0,"kl":0.06296532414853573,"entropy":0.4129450172185898},{"step":83,"reward":-6.30236291885376,"core":-2.0292041301727295,"cost":-0.012291529215872288,"kl":0.0980008477345109,"entropy":0.4012519493699074},{"step":85,"reward":-8.889148712158203,"core":-2.8961527347564697,"cost":-0.023565327748656273,"kl":0.06840791739523411,"entropy":0.37991274148225784},{"step":87,"reward":-10.292252540588379,"core":-3.347269058227539,"cost":-0.11075093597173691,"kl":0.07853751443326473,"entropy":0.4756437763571739},{"step":89,"reward":-3.5393950939178467,"core":-1.0946917533874512,"cost":-0.007681263145059347,"kl":0.09108961187303066,"entropy":0.45737510919570923},{"step":91,"reward":-8.808738708496094,"core":-2.865537643432617,"cost":-0.03829098492860794,"kl":0.16180330514907837,"entropy":0.47070398926734924},{"step":93,"reward":-3.5643012523651123,"core":-1.0776562690734863,"cost":0.0,"kl":0.07172174099832773,"entropy":0.4406670555472374},{"step":95,"reward":-3.1328632831573486,"core":-0.9780177474021912,"cost":0.0,"kl":0.06356058456003666,"entropy":0.4788781851530075},{"step":97,"reward":-6.624378681182861,"core":-2.1484885215759277,"cost":-0.03793087601661682,"kl":0.17434726655483246,"entropy":0.441969096660614},{"step":99,"reward":-9.396889686584473,"core":-3.0838935375213623,"cost":-0.033268410712480545,"kl":0.12193801812827587,"entropy":0.5460600405931473},{"step":101,"reward":-4.428060531616211,"core":-1.3856159448623657,"cost":-0.0015217057662084699,"kl":0.056794483214616776,"entropy":0.454096294939518},{"step":103,"reward":-4.294134616851807,"core":-1.3535418510437012,"cost":-0.001989176496863365,"kl":0.10570763982832432,"entropy":0.4146217852830887},{"step":105,"reward":-5.493550777435303,"core":-1.7731736898422241,"cost":0.0,"kl":0.10144956409931183,"entropy":0.588704064488411},{"step":107,"reward":-4.932629108428955,"core":-1.6109238862991333,"cost":-0.0033605676144361496,"kl":0.07617033272981644,"entropy":0.6052539050579071},{"step":109,"reward":-4.082801818847656,"core":-1.3082622289657593,"cost":-0.0038707267958670855,"kl":0.08081541676074266,"entropy":0.4609478861093521},{"step":111,"reward":-3.959380865097046,"core":-1.2551771402359009,"cost":0.0,"kl":0.10840246081352234,"entropy":0.6200617402791977},{"step":113,"reward":-4.670080661773682,"core":-1.484445571899414,"cost":-0.006647760514169931,"kl":0.1283598467707634,"entropy":0.600087508559227},{"step":115,"reward":-8.264958381652832,"core":-2.703035593032837,"cost":-0.012395850382745266,"kl":0.14012759551405907,"entropy":0.5770848244428635},{"step":117,"reward":-7.829753398895264,"core":-2.5525920391082764,"cost":-0.014279837720096111,"kl":0.0892914105206728,"entropy":0.5144894272089005},{"step":119,"reward":-3.7686767578125,"core":-1.2230015993118286,"cost":0.0,"kl":0.0927487201988697,"entropy":0.6351564973592758},{"step":121,"reward":-6.041282653808594,"core":-1.9563392400741577,"cost":-0.02505887858569622,"kl":0.0891171470284462,"entropy":0.5419510081410408},{"step":123,"reward":-5.573215961456299,"core":-1.8195791244506836,"cost":0.0,"kl":0.11861850321292877,"entropy":0.6628405153751373},{"step":125,"reward":-3.1737043857574463,"core":-1.0081983804702759,"cost":0.0,"kl":0.10189374908804893,"entropy":0.6010228246450424},{"step":127,"reward":-4.240710735321045,"core":-1.3476771116256714,"cost":-0.00628252699971199,"kl":0.10478288866579533,"entropy":0.575937494635582},{"step":129,"reward":-5.281949520111084,"core":-1.6804018020629883,"cost":-0.02952374517917633,"kl":0.10163287818431854,"entropy":0.5681206285953522},{"step":131,"reward":-5.537967681884766,"core":-1.7948657274246216,"cost":0.0,"kl":0.1159897968173027,"entropy":0.616536021232605},{"step":133,"reward":-3.421368360519409,"core":-1.0927789211273193,"cost":0.0,"kl":0.1185734961181879,"entropy":0.5165902972221375},{"step":135,"reward":-4.134572505950928,"core":-1.336051344871521,"cost":0.0,"kl":0.10635627619922161,"entropy":0.5821313560009003},{"step":137,"reward":-6.269645690917969,"core":-2.056833028793335,"cost":-0.007984673604369164,"kl":0.12574860267341137,"entropy":0.5035733506083488},{"step":139,"reward":-4.113572120666504,"core":-1.2900029420852661,"cost":0.0,"kl":0.1439765803515911,"entropy":0.4366750121116638},{"step":141,"reward":-4.381673812866211,"core":-1.4074896574020386,"cost":-0.004595211241394281,"kl":0.12675095908343792,"entropy":0.4611644074320793},{"step":143,"reward":-4.627291679382324,"core":-1.4801141023635864,"cost":-0.012566122226417065,"kl":0.12923779524862766,"entropy":0.5705694705247879},{"step":145,"reward":-4.250389575958252,"core":-1.3559560775756836,"cost":0.0,"kl":0.11326125264167786,"entropy":0.41952691972255707},{"step":147,"reward":-5.289644241333008,"core":-1.7137335538864136,"cost":0.0,"kl":0.19383852183818817,"entropy":0.5136421397328377},{"step":149,"reward":-5.367326259613037,"core":-1.7391916513442993,"cost":-0.006461640354245901,"kl":0.1734677143394947,"entropy":0.4543857127428055},{"step":151,"reward":-3.2094173431396484,"core":-1.009604811668396,"cost":0.0,"kl":0.14183637872338295,"entropy":0.5749179869890213},{"step":153,"reward":-4.315333843231201,"core":-1.3482986688613892,"cost":-0.00018335203640162945,"kl":0.15529362484812737,"entropy":0.4519305154681206},{"step":155,"reward":-3.2076759338378906,"core":-1.0032812356948853,"cost":0.0,"kl":0.11667872779071331,"entropy":0.47583089768886566},{"step":157,"reward":-5.202475070953369,"core":-1.6468764543533325,"cost":-0.03519882261753082,"kl":0.16838950663805008,"entropy":0.8525755181908607},{"step":159,"reward":-3.391024351119995,"core":-1.0421017408370972,"cost":0.0,"kl":0.15572469681501389,"entropy":0.46065567433834076},{"step":161,"reward":-2.503450393676758,"core":-0.772570788860321,"cost":0.0,"kl":0.09954820573329926,"entropy":0.45188601315021515},{"step":163,"reward":-4.714511394500732,"core":-1.529007911682129,"cost":0.0,"kl":0.2035370096564293,"entropy":0.5088722109794617},{"step":165,"reward":-2.8422329425811768,"core":-0.8559610247612,"cost":0.0,"kl":0.12667925842106342,"entropy":0.42210497707128525},{"step":167,"reward":-2.953782320022583,"core":-0.9548053741455078,"cost":0.0,"kl":0.1085127741098404,"entropy":0.47962717711925507},{"step":169,"reward":-3.0998332500457764,"core":-0.9614259600639343,"cost":0.0,"kl":0.09459807723760605,"entropy":0.45713401585817337},{"step":171,"reward":-5.321155548095703,"core":-1.717254638671875,"cost":0.0,"kl":0.181261345744133,"entropy":0.48861701786518097},{"step":173,"reward":-4.202297687530518,"core":-1.308042049407959,"cost":-0.010457918979227543,"kl":0.1631271243095398,"entropy":0.4412565976381302},{"step":175,"reward":-5.778200626373291,"core":-1.8617998361587524,"cost":-0.0018168095266446471,"kl":0.27627720311284065,"entropy":0.47351112216711044},{"step":177,"reward":-3.288726806640625,"core":-1.010125756263733,"cost":0.0,"kl":0.1478033810853958,"entropy":0.3858293518424034},{"step":179,"reward":-2.5499260425567627,"core":-0.7290029525756836,"cost":0.0,"kl":0.13346317410469055,"entropy":0.4604782238602638},{"step":181,"reward":-2.6678521633148193,"core":-0.7889730334281921,"cost":0.0,"kl":0.14023857191205025,"entropy":0.4277859926223755},{"step":183,"reward":-3.0307610034942627,"core":-0.9039625525474548,"cost":-0.0207060556858778,"kl":0.12427566573023796,"entropy":0.419192411005497},{"step":185,"reward":-2.3168790340423584,"core":-0.6663339734077454,"cost":0.0,"kl":0.5125833004713058,"entropy":0.4516187161207199},{"step":187,"reward":-4.855808258056641,"core":-1.5596729516983032,"cost":-0.005397200118750334,"kl":0.125916613265872,"entropy":0.559114471077919},{"step":189,"reward":-3.3981010913848877,"core":-1.0568342208862305,"cost":0.0,"kl":0.12254573218524456,"entropy":0.5146932229399681},{"step":191,"reward":-3.621901750564575,"core":-1.1124173402786255,"cost":-0.0037417991552501917,"kl":0.162723233923316,"entropy":0.4974535182118416},{"step":193,"reward":-3.116880416870117,"core":-0.948209285736084,"cost":0.0,"kl":0.15329121239483356,"entropy":0.5100002735853195},{"step":195,"reward":-4.192739009857178,"core":-1.3355275392532349,"cost":-0.007879451848566532,"kl":0.14607913419604301,"entropy":0.5645909607410431},{"step":197,"reward":-3.386354446411133,"core":-1.069508671760559,"cost":0.0,"kl":0.10565779730677605,"entropy":0.5336218327283859},{"step":199,"reward":-3.3358280658721924,"core":-1.0285590887069702,"cost":-0.002023357665166259,"kl":0.12597021646797657,"entropy":0.5807972252368927},{"step":201,"reward":-2.795961380004883,"core":-0.8744354248046875,"cost":0.0,"kl":0.08596374839544296,"entropy":0.5058134347200394},{"step":203,"reward":-3.466083288192749,"core":-1.0802416801452637,"cost":0.0,"kl":0.12971832044422626,"entropy":0.5543435662984848},{"step":205,"reward":-3.581470251083374,"core":-1.1219509840011597,"cost":0.0,"kl":0.12162984162569046,"entropy":0.5611819326877594},{"step":207,"reward":-3.129331588745117,"core":-0.9853660464286804,"cost":0.0,"kl":0.12513876892626286,"entropy":0.5385294109582901},{"step":209,"reward":-3.547485113143921,"core":-1.0763345956802368,"cost":-0.010175788775086403,"kl":0.16365494020283222,"entropy":0.5220752358436584},{"step":211,"reward":-4.422093868255615,"core":-1.3990103006362915,"cost":-0.00410706689581275,"kl":0.18715307116508484,"entropy":0.5192260891199112},{"step":213,"reward":-2.356416702270508,"core":-0.6732633709907532,"cost":0.0,"kl":0.2141805924475193,"entropy":0.5811828225851059},{"step":215,"reward":-4.996260643005371,"core":-1.5941672325134277,"cost":0.0,"kl":0.15716463327407837,"entropy":0.5847501158714294},{"step":217,"reward":-3.029014825820923,"core":-0.910452663898468,"cost":0.0,"kl":0.15125403925776482,"entropy":0.5244180560112},{"step":219,"reward":-5.038886070251465,"core":-1.5639100074768066,"cost":-0.039070818573236465,"kl":0.14291146770119667,"entropy":0.5408796221017838},{"step":221,"reward":-4.13827657699585,"core":-1.327284574508667,"cost":0.0,"kl":0.1174998190253973,"entropy":0.5950980633497238},{"step":223,"reward":-3.8165524005889893,"core":-1.187531590461731,"cost":-0.004231386352330446,"kl":0.10241363197565079,"entropy":0.4829782173037529},{"step":225,"reward":-4.561328887939453,"core":-1.4513880014419556,"cost":0.0,"kl":0.13444356061518192,"entropy":0.5890732109546661},{"step":227,"reward":-3.6805007457733154,"core":-1.1315279006958008,"cost":-0.0008433701586909592,"kl":0.18975958973169327,"entropy":0.5289323925971985},{"step":229,"reward":-3.514681100845337,"core":-1.100089430809021,"cost":0.0,"kl":0.21177712827920914,"entropy":0.5177140906453133},{"step":231,"reward":-2.283414840698242,"core":-0.6460677981376648,"cost":0.0,"kl":0.12003073841333389,"entropy":0.42997271567583084},{"step":233,"reward":-3.598313093185425,"core":-1.1421877145767212,"cost":-0.002976934425532818,"kl":0.1350468471646309,"entropy":0.6048940420150757},{"step":235,"reward":-3.7649710178375244,"core":-1.191823959350586,"cost":0.0,"kl":0.16186491772532463,"entropy":0.6397239416837692},{"step":237,"reward":-2.872217893600464,"core":-0.8913125991821289,"cost":0.0,"kl":0.11091727949678898,"entropy":0.50094785541296},{"step":239,"reward":-2.2346789836883545,"core":-0.6191962957382202,"cost":0.0,"kl":0.15820974111557007,"entropy":0.5279802605509758},{"step":241,"reward":-2.961385726928711,"core":-0.9203083515167236,"cost":0.0,"kl":0.1445877142250538,"entropy":0.6326352804899216},{"step":243,"reward":-3.593953847885132,"core":-1.1208301782608032,"cost":-0.016607506200671196,"kl":0.1113484837114811,"entropy":0.4871187284588814},{"step":245,"reward":-2.6999943256378174,"core":-0.8086660504341125,"cost":0.0,"kl":0.12678495980799198,"entropy":0.5050691068172455},{"step":247,"reward":-5.373782634735107,"core":-1.7385063171386719,"cost":-0.002256877487525344,"kl":0.18379667773842812,"entropy":0.5900310277938843},{"step":249,"reward":-2.4577949047088623,"core":-0.7324272990226746,"cost":0.0,"kl":0.12592888623476028,"entropy":0.5284513160586357},{"step":251,"reward":-2.175760269165039,"core":-0.5959024429321289,"cost":0.0,"kl":0.15106862783432007,"entropy":0.4286704882979393},{"step":253,"reward":-5.966336727142334,"core":-1.9276272058486938,"cost":-0.019865920767188072,"kl":0.15748582407832146,"entropy":0.546322837471962},{"step":255,"reward":-3.9749794006347656,"core":-1.2393372058868408,"cost":-0.005166204180568457,"kl":0.12835510447621346,"entropy":0.5253112241625786},{"step":257,"reward":-3.218867063522339,"core":-0.9619943499565125,"cost":0.0,"kl":0.14948073402047157,"entropy":0.4288966506719589},{"step":259,"reward":-2.6397557258605957,"core":-0.7977585792541504,"cost":0.0,"kl":0.12123287469148636,"entropy":0.4980962201952934},{"step":261,"reward":-5.392831802368164,"core":-1.7144273519515991,"cost":-0.007990043610334396,"kl":0.22107888758182526,"entropy":0.49723639339208603},{"step":263,"reward":-2.5871355533599854,"core":-0.7562205791473389,"cost":0.0,"kl":0.16921378672122955,"entropy":0.49488411098718643},{"step":265,"reward":-3.1565821170806885,"core":-0.9584792256355286,"cost":0.0,"kl":0.25029854476451874,"entropy":0.4586101099848747},{"step":267,"reward":-2.00492000579834,"core":-0.5245313048362732,"cost":-0.005219603423029184,"kl":0.23307305946946144,"entropy":0.3943193405866623},{"step":269,"reward":-2.2055015563964844,"core":-0.6285008788108826,"cost":0.0,"kl":0.14694293960928917,"entropy":0.4204959496855736},{"step":271,"reward":-2.039611577987671,"core":-0.5652225613594055,"cost":0.0,"kl":0.17967947758734226,"entropy":0.4522050619125366},{"step":273,"reward":-1.9315071105957031,"core":-0.5222013592720032,"cost":0.0,"kl":0.23319178819656372,"entropy":0.49629534035921097},{"step":275,"reward":-2.7586841583251953,"core":-0.8061063289642334,"cost":-0.0037350200582295656,"kl":0.15377642400562763,"entropy":0.4295286312699318},{"step":277,"reward":-2.11458420753479,"core":-0.6021355986595154,"cost":0.0,"kl":0.19727681949734688,"entropy":0.4479081705212593},{"step":279,"reward":-2.1014487743377686,"core":-0.5753441452980042,"cost":0.0,"kl":0.23646876960992813,"entropy":0.4591939225792885},{"step":281,"reward":-1.942482352256775,"core":-0.5107424855232239,"cost":-0.0011611813679337502,"kl":0.2062923163175583,"entropy":0.36611104011535645},{"step":283,"reward":-2.73041033744812,"core":-0.8019421100616455,"cost":0.0,"kl":0.2948356792330742,"entropy":0.4420301094651222},{"step":285,"reward":-2.4358198642730713,"core":-0.705357551574707,"cost":0.0,"kl":0.31428683549165726,"entropy":0.467367485165596},{"step":287,"reward":-1.847474455833435,"core":-0.5027986168861389,"cost":0.0,"kl":0.23676270991563797,"entropy":0.42192523926496506},{"step":289,"reward":-1.3628917932510376,"core":-0.3024781048297882,"cost":-1.589455314388033e-05,"kl":0.14378297701478004,"entropy":0.37299326062202454},{"step":291,"reward":-1.7367562055587769,"core":-0.47080501914024353,"cost":0.0,"kl":0.20565691217780113,"entropy":0.46207285672426224},{"step":293,"reward":-3.576292037963867,"core":-1.1023004055023193,"cost":-0.007480463478714228,"kl":0.20016571879386902,"entropy":0.4646188095211983},{"step":295,"reward":-3.9184505939483643,"core":-1.164175033569336,"cost":-0.0749431923031807,"kl":0.21699536219239235,"entropy":0.4469423070549965},{"step":297,"reward":-2.3959128856658936,"core":-0.7008941769599915,"cost":0.0,"kl":0.16280462592840195,"entropy":0.46658509224653244},{"step":299,"reward":-2.0508930683135986,"core":-0.577202558517456,"cost":0.0,"kl":0.26824042946100235,"entropy":0.48250506073236465},{"step":301,"reward":-1.5586767196655273,"core":-0.39036011695861816,"cost":0.0,"kl":0.21389777585864067,"entropy":0.4717889279127121},{"step":303,"reward":-2.4871323108673096,"core":-0.7369594573974609,"cost":0.0,"kl":0.15305810049176216,"entropy":0.5072871968150139},{"step":305,"reward":-2.8755252361297607,"core":-0.8465674519538879,"cost":-0.02222415991127491,"kl":0.19895531237125397,"entropy":0.48510660976171494},{"step":307,"reward":-3.664306640625,"core":-1.0971946716308594,"cost":-0.05017745494842529,"kl":0.18802328407764435,"entropy":0.5308956727385521},{"step":309,"reward":-3.251477003097534,"core":-0.9858188629150391,"cost":-0.015205630101263523,"kl":0.18443627282977104,"entropy":0.4832543134689331},{"step":311,"reward":-1.0710954666137695,"core":-0.2062007039785385,"cost":0.0,"kl":0.22507360577583313,"entropy":0.48195669800043106},{"step":313,"reward":-2.01777720451355,"core":-0.5625050067901611,"cost":0.0,"kl":0.4208686463534832,"entropy":0.5310280472040176},{"step":315,"reward":-2.5727550983428955,"core":-0.7272865176200867,"cost":-9.481570305069908e-05,"kl":0.3741315118968487,"entropy":0.4505457356572151},{"step":317,"reward":-1.572591781616211,"core":-0.3993200361728668,"cost":0.0,"kl":1.9140001200139523,"entropy":0.4039339944720268},{"step":319,"reward":-2.1380631923675537,"core":-0.5743297934532166,"cost":0.0,"kl":0.19449864327907562,"entropy":0.43370775133371353},{"step":321,"reward":-2.357891798019409,"core":-0.637185275554657,"cost":-0.03512440249323845,"kl":0.13792624697089195,"entropy":0.41349611431360245},{"step":323,"reward":-2.711416244506836,"core":-0.7762323021888733,"cost":-0.01575920172035694,"kl":0.17071079462766647,"entropy":0.40777410566806793},{"step":325,"reward":-1.2904937267303467,"core":-0.3006869852542877,"cost":0.0,"kl":0.2130541428923607,"entropy":0.4697642996907234},{"step":327,"reward":-2.3601319789886475,"core":-0.675358235836029,"cost":0.0,"kl":0.1818910911679268,"entropy":0.4282787963747978},{"step":329,"reward":-3.7217319011688232,"core":-1.1374074220657349,"cost":-0.006814062595367432,"kl":0.21651280671358109,"entropy":0.5392488986253738},{"step":331,"reward":-4.928098201751709,"core":-1.5256096124649048,"cost":-0.05027000606060028,"kl":0.3145880661904812,"entropy":0.4881933405995369},{"step":333,"reward":-3.398317337036133,"core":-1.0271941423416138,"cost":-0.008695852942764759,"kl":0.1675296053290367,"entropy":0.48718906939029694},{"step":335,"reward":-1.9524809122085571,"core":-0.5601845383644104,"cost":0.0,"kl":0.17558636143803596,"entropy":0.4604537636041641},{"step":337,"reward":-3.5407583713531494,"core":-1.0933820009231567,"cost":0.0,"kl":0.20913665369153023,"entropy":0.5487489998340607},{"step":339,"reward":-2.2021849155426025,"core":-0.6149479746818542,"cost":0.0,"kl":0.2284924052655697,"entropy":0.4924168065190315}],"nonzero_advantage":[{"step":1,"nonzero":1.0},{"step":3,"nonzero":1.0},{"step":5,"nonzero":1.0},{"step":7,"nonzero":1.0},{"step":9,"nonzero":1.0},{"step":11,"nonzero":1.0},{"step":13,"nonzero":1.0},{"step":15,"nonzero":1.0},{"step":17,"nonzero":1.0},{"step":19,"nonzero":1.0},{"step":21,"nonzero":1.0},{"step":23,"nonzero":1.0},{"step":25,"nonzero":1.0},{"step":27,"nonzero":1.0},{"step":29,"nonzero":1.0},{"step":31,"nonzero":1.0},{"step":33,"nonzero":1.0},{"step":35,"nonzero":1.0},{"step":37,"nonzero":1.0},{"step":39,"nonzero":1.0},{"step":41,"nonzero":1.0},{"step":43,"nonzero":1.0},{"step":45,"nonzero":1.0},{"step":47,"nonzero":1.0},{"step":49,"nonzero":1.0},{"step":51,"nonzero":1.0},{"step":53,"nonzero":1.0},{"step":55,"nonzero":1.0},{"step":57,"nonzero":1.0},{"step":59,"nonzero":1.0},{"step":61,"nonzero":1.0},{"step":63,"nonzero":1.0},{"step":65,"nonzero":1.0},{"step":67,"nonzero":1.0},{"step":69,"nonzero":1.0},{"step":71,"nonzero":1.0},{"step":73,"nonzero":1.0},{"step":75,"nonzero":0.8125},{"step":77,"nonzero":1.0},{"step":79,"nonzero":1.0},{"step":81,"nonzero":1.0},{"step":83,"nonzero":1.0},{"step":85,"nonzero":1.0},{"step":87,"nonzero":1.0},{"step":89,"nonzero":1.0},{"step":91,"nonzero":1.0},{"step":93,"nonzero":1.0},{"step":95,"nonzero":1.0},{"step":97,"nonzero":1.0},{"step":99,"nonzero":1.0},{"step":101,"nonzero":1.0},{"step":103,"nonzero":1.0},{"step":105,"nonzero":1.0},{"step":107,"nonzero":1.0},{"step":109,"nonzero":1.0},{"step":111,"nonzero":1.0},{"step":113,"nonzero":1.0},{"step":115,"nonzero":1.0},{"step":117,"nonzero":1.0},{"step":119,"nonzero":1.0},{"step":121,"nonzero":1.0},{"step":123,"nonzero":1.0},{"step":125,"nonzero":1.0},{"step":127,"nonzero":1.0},{"step":129,"nonzero":1.0},{"step":131,"nonzero":1.0},{"step":133,"nonzero":1.0},{"step":135,"nonzero":1.0},{"step":137,"nonzero":1.0},{"step":139,"nonzero":1.0},{"step":141,"nonzero":1.0},{"step":143,"nonzero":1.0},{"step":145,"nonzero":1.0},{"step":147,"nonzero":1.0},{"step":149,"nonzero":1.0},{"step":151,"nonzero":1.0},{"step":153,"nonzero":1.0},{"step":155,"nonzero":1.0},{"step":157,"nonzero":1.0},{"step":159,"nonzero":1.0},{"step":161,"nonzero":1.0},{"step":163,"nonzero":1.0},{"step":165,"nonzero":1.0},{"step":167,"nonzero":1.0},{"step":169,"nonzero":1.0},{"step":171,"nonzero":1.0},{"step":173,"nonzero":1.0},{"step":175,"nonzero":1.0},{"step":177,"nonzero":1.0},{"step":179,"nonzero":1.0},{"step":181,"nonzero":1.0},{"step":183,"nonzero":1.0},{"step":185,"nonzero":1.0},{"step":187,"nonzero":1.0},{"step":189,"nonzero":1.0},{"step":191,"nonzero":1.0},{"step":193,"nonzero":1.0},{"step":195,"nonzero":1.0},{"step":197,"nonzero":1.0},{"step":199,"nonzero":1.0},{"step":201,"nonzero":1.0},{"step":203,"nonzero":1.0},{"step":205,"nonzero":1.0},{"step":207,"nonzero":1.0},{"step":209,"nonzero":1.0},{"step":211,"nonzero":1.0},{"step":213,"nonzero":1.0},{"step":215,"nonzero":1.0},{"step":217,"nonzero":1.0},{"step":219,"nonzero":1.0},{"step":221,"nonzero":1.0},{"step":223,"nonzero":1.0},{"step":225,"nonzero":1.0},{"step":227,"nonzero":1.0},{"step":229,"nonzero":1.0},{"step":231,"nonzero":1.0},{"step":233,"nonzero":1.0},{"step":235,"nonzero":1.0},{"step":237,"nonzero":1.0},{"step":239,"nonzero":1.0},{"step":241,"nonzero":1.0},{"step":243,"nonzero":1.0},{"step":245,"nonzero":1.0},{"step":247,"nonzero":1.0},{"step":249,"nonzero":1.0},{"step":251,"nonzero":1.0},{"step":253,"nonzero":1.0},{"step":255,"nonzero":1.0},{"step":257,"nonzero":1.0},{"step":259,"nonzero":1.0},{"step":261,"nonzero":1.0},{"step":263,"nonzero":1.0},{"step":265,"nonzero":1.0},{"step":267,"nonzero":1.0},{"step":269,"nonzero":1.0},{"step":271,"nonzero":1.0},{"step":273,"nonzero":1.0},{"step":275,"nonzero":1.0},{"step":277,"nonzero":1.0},{"step":279,"nonzero":1.0},{"step":281,"nonzero":1.0},{"step":283,"nonzero":1.0},{"step":285,"nonzero":1.0},{"step":287,"nonzero":1.0},{"step":289,"nonzero":1.0},{"step":291,"nonzero":1.0},{"step":293,"nonzero":1.0},{"step":295,"nonzero":1.0},{"step":297,"nonzero":1.0},{"step":299,"nonzero":1.0},{"step":301,"nonzero":1.0},{"step":303,"nonzero":1.0},{"step":305,"nonzero":1.0},{"step":307,"nonzero":1.0},{"step":309,"nonzero":1.0},{"step":311,"nonzero":1.0},{"step":313,"nonzero":1.0},{"step":315,"nonzero":1.0},{"step":317,"nonzero":1.0},{"step":319,"nonzero":1.0},{"step":321,"nonzero":1.0},{"step":323,"nonzero":1.0},{"step":325,"nonzero":1.0},{"step":327,"nonzero":1.0},{"step":329,"nonzero":1.0},{"step":331,"nonzero":1.0},{"step":333,"nonzero":1.0},{"step":335,"nonzero":1.0},{"step":337,"nonzero":1.0},{"step":339,"nonzero":1.0},{"step":341,"nonzero":1.0}]},{"id":"b2-floor0","label":"Advantage floor at 0 (second slice)","run_id":"1791044621","cpcv_floor":0.0,"train_steps":[{"step":1,"reward":-14.943755149841309,"core":-4.9268927574157715,"cost":-0.14593690633773804,"kl":0.0,"entropy":0.4622746407985687},{"step":3,"reward":-13.21225357055664,"core":-4.327184200286865,"cost":-0.16050690412521362,"kl":0.0,"entropy":0.5798681378364563},{"step":5,"reward":-10.832158088684082,"core":-3.572863817214966,"cost":-0.05809831991791725,"kl":0.0,"entropy":0.5268194377422333},{"step":7,"reward":-12.872424125671387,"core":-4.231039524078369,"cost":-0.13003195822238922,"kl":0.0007139052904676646,"entropy":0.49950408190488815},{"step":9,"reward":-15.913569450378418,"core":-5.239191055297852,"cost":-0.16018226742744446,"kl":0.0009470054937992245,"entropy":0.46414392441511154},{"step":11,"reward":-23.680742263793945,"core":-7.7284111976623535,"cost":-0.4011879861354828,"kl":0.001900292350910604,"entropy":0.4866461604833603},{"step":13,"reward":-6.037933349609375,"core":-1.9534845352172852,"cost":-0.02699357271194458,"kl":0.002315357531188056,"entropy":0.4651305824518204},{"step":15,"reward":-16.78317642211914,"core":-5.530848979949951,"cost":-0.17294317483901978,"kl":0.004045686102472246,"entropy":0.5548868030309677},{"step":17,"reward":-11.370951652526855,"core":-3.716710329055786,"cost":-0.11882229894399643,"kl":0.003164278867188841,"entropy":0.5006880611181259},{"step":19,"reward":-18.533790588378906,"core":-6.108579158782959,"cost":-0.18616247177124023,"kl":0.004872061545029283,"entropy":0.4728669300675392},{"step":21,"reward":-6.732044219970703,"core":-2.1841652393341064,"cost":-0.02653045952320099,"kl":0.004927098285406828,"entropy":0.5154439061880112},{"step":23,"reward":-18.783973693847656,"core":-6.164483547210693,"cost":-0.27526867389678955,"kl":0.007118872134014964,"entropy":0.5876353159546852},{"step":25,"reward":-9.044463157653809,"core":-2.970317840576172,"cost":-0.03691685572266579,"kl":0.00677091593388468,"entropy":0.55001050978899},{"step":27,"reward":-7.3169121742248535,"core":-2.3929941654205322,"cost":-0.06121736392378807,"kl":0.006434695329517126,"entropy":0.5099567323923111},{"step":29,"reward":-8.359814643859863,"core":-2.718048334121704,"cost":-0.07493697851896286,"kl":0.007201551576144993,"entropy":0.540796272456646},{"step":31,"reward":-8.808212280273438,"core":-2.8749678134918213,"cost":-0.04465152323246002,"kl":0.006959199323318899,"entropy":0.46242018789052963},{"step":33,"reward":-11.567612648010254,"core":-3.812178373336792,"cost":-0.05787898972630501,"kl":0.00681571732275188,"entropy":0.4542586952447891},{"step":35,"reward":-10.88616943359375,"core":-3.5830228328704834,"cost":-0.049481216818094254,"kl":0.008453044691123068,"entropy":0.4634878560900688},{"step":37,"reward":-9.481022834777832,"core":-3.0443737506866455,"cost":-0.11856696754693985,"kl":0.01036111987195909,"entropy":0.4623263254761696},{"step":39,"reward":-5.825588226318359,"core":-1.9029617309570312,"cost":0.0,"kl":0.015270449919626117,"entropy":0.5234012752771378},{"step":41,"reward":-9.244956970214844,"core":-3.0481555461883545,"cost":-0.03237748518586159,"kl":0.013125683180987835,"entropy":0.4817906320095062},{"step":43,"reward":-6.01544189453125,"core":-1.9776684045791626,"cost":0.0,"kl":0.007602025754749775,"entropy":0.4402220696210861},{"step":45,"reward":-15.307441711425781,"core":-5.030157566070557,"cost":-0.16036652028560638,"kl":0.008033623453229666,"entropy":0.44647373259067535},{"step":47,"reward":-23.80280113220215,"core":-7.793446063995361,"cost":-0.362090140581131,"kl":0.006585349328815937,"entropy":0.42072203010320663},{"step":49,"reward":-16.882143020629883,"core":-5.551673412322998,"cost":-0.20766060054302216,"kl":0.0085789900040254,"entropy":0.46978482604026794},{"step":51,"reward":-12.578783988952637,"core":-4.130108833312988,"cost":-0.12381241470575333,"kl":0.007160327513702214,"entropy":0.46730076521635056},{"step":53,"reward":-7.512742519378662,"core":-2.458174228668213,"cost":-0.05500007048249245,"kl":0.0062639444367960095,"entropy":0.5113096758723259},{"step":55,"reward":-8.180870056152344,"core":-2.6465249061584473,"cost":-0.06279660016298294,"kl":0.006029057549312711,"entropy":0.4440867677330971},{"step":57,"reward":-21.772470474243164,"core":-7.149621963500977,"cost":-0.2860388159751892,"kl":0.005024026380851865,"entropy":0.5358620882034302},{"step":59,"reward":-10.114715576171875,"core":-3.3251094818115234,"cost":-0.06942377239465714,"kl":0.0053648113971576095,"entropy":0.46419695764780045},{"step":61,"reward":-17.13424301147461,"core":-5.634990692138672,"cost":-0.19613833725452423,"kl":0.007108310121111572,"entropy":0.5674596428871155},{"step":63,"reward":-7.775886058807373,"core":-2.539445638656616,"cost":-0.044898081570863724,"kl":0.007525066030211747,"entropy":0.5252073556184769},{"step":65,"reward":-8.40935230255127,"core":-2.760732889175415,"cost":-0.026595130562782288,"kl":0.0045919716358184814,"entropy":0.5190817564725876},{"step":67,"reward":-14.59883975982666,"core":-4.781158924102783,"cost":-0.19147074222564697,"kl":0.005418541259132326,"entropy":0.4751771464943886},{"step":69,"reward":-7.086606502532959,"core":-2.2763988971710205,"cost":-0.05625121667981148,"kl":0.0054344980744645,"entropy":0.5437989383935928},{"step":71,"reward":-16.069120407104492,"core":-5.270776271820068,"cost":-0.19386553764343262,"kl":0.009347188053652644,"entropy":0.5243657305836678},{"step":73,"reward":-7.363794326782227,"core":-2.4082605838775635,"cost":-0.038984548300504684,"kl":0.0072341435588896275,"entropy":0.4998975619673729},{"step":75,"reward":-7.540634632110596,"core":-2.453860282897949,"cost":-0.04828420281410217,"kl":0.007369630271568894,"entropy":0.5294522494077682},{"step":77,"reward":-14.080516815185547,"core":-4.620802402496338,"cost":-0.1179063692688942,"kl":0.005502408952452242,"entropy":0.4646354392170906},{"step":79,"reward":-12.853096008300781,"core":-4.21891450881958,"cost":-0.1205548346042633,"kl":0.009672887157648802,"entropy":0.5356946662068367},{"step":81,"reward":-11.763233184814453,"core":-3.891904830932617,"cost":-0.04282433167099953,"kl":0.00818752613849938,"entropy":0.5216161236166954},{"step":83,"reward":-18.764240264892578,"core":-6.183929443359375,"cost":-0.19394469261169434,"kl":0.011156300781294703,"entropy":0.5300933793187141},{"step":85,"reward":-15.260640144348145,"core":-5.024933815002441,"cost":-0.1175522431731224,"kl":0.008852686965838075,"entropy":0.5148391425609589},{"step":87,"reward":-19.208913803100586,"core":-6.325791835784912,"cost":-0.23638100922107697,"kl":0.007512929267250001,"entropy":0.49835921078920364},{"step":89,"reward":-9.993511199951172,"core":-3.288722276687622,"cost":-0.01860762946307659,"kl":0.013443870935589075,"entropy":0.5831181108951569},{"step":91,"reward":-21.26683235168457,"core":-6.974857807159424,"cost":-0.3017466366291046,"kl":0.015513915801420808,"entropy":0.6462136954069138},{"step":93,"reward":-17.755126953125,"core":-5.796222686767578,"cost":-0.2747774124145508,"kl":0.009501679101958871,"entropy":0.5599678307771683},{"step":95,"reward":-7.834500789642334,"core":-2.5445263385772705,"cost":-0.06876180320978165,"kl":0.00677243759855628,"entropy":0.52988600730896},{"step":97,"reward":-17.556406021118164,"core":-5.7659125328063965,"cost":-0.2097409963607788,"kl":0.007622051169164479,"entropy":0.5626519024372101},{"step":99,"reward":-9.769702911376953,"core":-3.1991615295410156,"cost":-0.05841366574168205,"kl":0.012260960647836328,"entropy":0.5177645459771156},{"step":101,"reward":-12.594428062438965,"core":-4.120762348175049,"cost":-0.11798223108053207,"kl":0.010565583361312747,"entropy":0.511131264269352},{"step":103,"reward":-11.977717399597168,"core":-3.9542362689971924,"cost":-0.05642767623066902,"kl":0.004543884831946343,"entropy":0.5010288879275322},{"step":105,"reward":-7.929159164428711,"core":-2.595501184463501,"cost":-0.025538912042975426,"kl":0.00985111529007554,"entropy":0.5553024709224701},{"step":107,"reward":-16.830297470092773,"core":-5.48504638671875,"cost":-0.2476237267255783,"kl":0.006282282876782119,"entropy":0.48632264137268066},{"step":109,"reward":-9.713714599609375,"core":-3.1473629474639893,"cost":-0.11442742496728897,"kl":0.009497596882283688,"entropy":0.485389307141304},{"step":111,"reward":-19.262731552124023,"core":-6.295257091522217,"cost":-0.3128690719604492,"kl":0.008555720211006701,"entropy":0.4977342486381531},{"step":113,"reward":-8.280847549438477,"core":-2.699465751647949,"cost":-0.050390854477882385,"kl":0.008295407402329147,"entropy":0.542208731174469},{"step":115,"reward":-14.63634204864502,"core":-4.8233513832092285,"cost":-0.10180312395095825,"kl":0.006519275135360658,"entropy":0.5326095074415207},{"step":117,"reward":-10.711197853088379,"core":-3.499887466430664,"cost":-0.08538863807916641,"kl":0.007415887084789574,"entropy":0.5571765303611755},{"step":119,"reward":-15.971789360046387,"core":-5.255194664001465,"cost":-0.1696932166814804,"kl":0.006085887900553644,"entropy":0.5459609404206276},{"step":121,"reward":-23.222068786621094,"core":-7.6118083000183105,"cost":-0.30756399035453796,"kl":0.00412524922285229,"entropy":0.5146852731704712},{"step":123,"reward":-12.34485149383545,"core":-4.001092910766602,"cost":-0.1629709154367447,"kl":0.004117857315577567,"entropy":0.528481736779213},{"step":125,"reward":-11.523655891418457,"core":-3.778573989868164,"cost":-0.1156594380736351,"kl":0.006480699288658798,"entropy":0.6126985847949982},{"step":127,"reward":-10.050945281982422,"core":-3.2874019145965576,"cost":-0.045710254460573196,"kl":0.004440701974090189,"entropy":0.4553700014948845},{"step":129,"reward":-12.714027404785156,"core":-4.178859710693359,"cost":-0.13184987008571625,"kl":0.004681365680880845,"entropy":0.5477416291832924},{"step":131,"reward":-9.920166969299316,"core":-3.2497596740722656,"cost":-0.07456188648939133,"kl":0.0039054593071341515,"entropy":0.5210763961076736},{"step":133,"reward":-10.219569206237793,"core":-3.378430128097534,"cost":-0.00996104534715414,"kl":0.003910782455932349,"entropy":0.5003820806741714},{"step":135,"reward":-10.776298522949219,"core":-3.528789758682251,"cost":-0.11449820548295975,"kl":0.005565974395722151,"entropy":0.5635088384151459},{"step":137,"reward":-8.89536190032959,"core":-2.943727493286133,"cost":-0.026359722018241882,"kl":0.003738577477633953,"entropy":0.4952864423394203},{"step":139,"reward":-10.227334022521973,"core":-3.3368804454803467,"cost":-0.08339991420507431,"kl":0.003936818626243621,"entropy":0.4441210478544235},{"step":141,"reward":-10.712346076965332,"core":-3.494680404663086,"cost":-0.12583059072494507,"kl":0.0037837550044059753,"entropy":0.521411269903183},{"step":143,"reward":-7.357595443725586,"core":-2.4120962619781494,"cost":-0.028868623077869415,"kl":0.0062455442966893315,"entropy":0.5389194041490555},{"step":145,"reward":-11.630961418151855,"core":-3.8074328899383545,"cost":-0.11978738754987717,"kl":0.003902779775671661,"entropy":0.4334762915968895},{"step":147,"reward":-9.43864917755127,"core":-3.0992114543914795,"cost":-0.06693639606237411,"kl":0.005545514635741711,"entropy":0.4723982885479927},{"step":149,"reward":-19.96630859375,"core":-6.530202865600586,"cost":-0.29302605986595154,"kl":0.005451636854559183,"entropy":0.5238796398043633},{"step":151,"reward":-8.767284393310547,"core":-2.8739328384399414,"cost":-0.07104656100273132,"kl":0.005145220668055117,"entropy":0.43955331295728683},{"step":153,"reward":-13.13927936553955,"core":-4.322536468505859,"cost":-0.1287192702293396,"kl":0.010325256502255797,"entropy":0.5185573920607567},{"step":155,"reward":-9.549609184265137,"core":-3.1251754760742188,"cost":-0.06431111693382263,"kl":0.013716083951294422,"entropy":0.5368073806166649},{"step":157,"reward":-15.134033203125,"core":-4.991363525390625,"cost":-0.08020754903554916,"kl":0.009679856477305293,"entropy":0.4462946131825447},{"step":159,"reward":-13.774303436279297,"core":-4.518781661987305,"cost":-0.13464775681495667,"kl":0.01001063920557499,"entropy":0.4280528128147125},{"step":161,"reward":-18.141828536987305,"core":-5.917600154876709,"cost":-0.2623147666454315,"kl":0.014628402888774872,"entropy":0.5067131295800209},{"step":163,"reward":-10.187270164489746,"core":-3.339864730834961,"cost":-0.08005864173173904,"kl":0.016460638726130128,"entropy":0.4938148632645607},{"step":165,"reward":-10.49988079071045,"core":-3.463367223739624,"cost":-0.044593241065740585,"kl":0.011304190382361412,"entropy":0.4276318848133087},{"step":167,"reward":-15.047574043273926,"core":-4.9570488929748535,"cost":-0.17145021259784698,"kl":0.013690871885046363,"entropy":0.47991521656513214},{"step":169,"reward":-10.150216102600098,"core":-3.3576242923736572,"cost":-0.0518900491297245,"kl":0.019591443706303835,"entropy":0.48414820432662964},{"step":171,"reward":-9.121506690979004,"core":-2.980468988418579,"cost":-0.07957123965024948,"kl":0.013753354782238603,"entropy":0.49955782294273376},{"step":173,"reward":-13.569694519042969,"core":-4.424753189086914,"cost":-0.14321579039096832,"kl":0.013321720529347658,"entropy":0.41193918883800507},{"step":175,"reward":-12.973401069641113,"core":-4.262553691864014,"cost":-0.1009841337800026,"kl":0.009795452817343175,"entropy":0.4514433518052101},{"step":177,"reward":-8.292550086975098,"core":-2.6876449584960938,"cost":-0.08059043437242508,"kl":0.01590820890851319,"entropy":0.49074751138687134},{"step":179,"reward":-18.05740737915039,"core":-5.908088684082031,"cost":-0.25403276085853577,"kl":0.01011837087571621,"entropy":0.480074867606163},{"step":181,"reward":-8.237650871276855,"core":-2.7130908966064453,"cost":-0.04615844413638115,"kl":0.009743617149069905,"entropy":0.5285781472921371},{"step":183,"reward":-15.809300422668457,"core":-5.18400764465332,"cost":-0.20012760162353516,"kl":0.0112443957477808,"entropy":0.6148865967988968},{"step":185,"reward":-11.267233848571777,"core":-3.6907851696014404,"cost":-0.12726972997188568,"kl":0.010998022858984768,"entropy":0.47918181121349335},{"step":187,"reward":-12.832562446594238,"core":-4.226076602935791,"cost":-0.0793004259467125,"kl":0.014448428526520729,"entropy":0.5700435638427734},{"step":189,"reward":-11.519740104675293,"core":-3.7612457275390625,"cost":-0.1359727382659912,"kl":0.008718295022845268,"entropy":0.5459738522768021},{"step":191,"reward":-12.862828254699707,"core":-4.223361492156982,"cost":-0.11946304887533188,"kl":0.010176721261814237,"entropy":0.5398225039243698},{"step":193,"reward":-9.768607139587402,"core":-3.2079622745513916,"cost":-0.0627564787864685,"kl":0.008754047099500895,"entropy":0.538612850010395},{"step":195,"reward":-18.565034866333008,"core":-6.102715969085693,"cost":-0.22777025401592255,"kl":0.009205162059515715,"entropy":0.5263935923576355},{"step":197,"reward":-9.887996673583984,"core":-3.254023313522339,"cost":-0.07408961653709412,"kl":0.01294925669208169,"entropy":0.5504345297813416},{"step":199,"reward":-7.767147064208984,"core":-2.5488157272338867,"cost":-0.022088496014475822,"kl":0.013158971210941672,"entropy":0.5212404876947403},{"step":201,"reward":-10.51849365234375,"core":-3.47088623046875,"cost":-0.04752147197723389,"kl":0.01430554874241352,"entropy":0.5118624866008759},{"step":203,"reward":-12.910164833068848,"core":-4.239391803741455,"cost":-0.12826642394065857,"kl":0.017256133491173387,"entropy":0.4603162854909897},{"step":205,"reward":-8.992977142333984,"core":-2.938214063644409,"cost":-0.05104302242398262,"kl":0.023232088424265385,"entropy":0.5360008925199509},{"step":207,"reward":-9.873982429504395,"core":-3.220749616622925,"cost":-0.049958735704422,"kl":0.02881190413609147,"entropy":0.6039669066667557},{"step":209,"reward":-9.099514961242676,"core":-2.9837722778320312,"cost":-0.03789670392870903,"kl":0.021477652247995138,"entropy":0.44199638813734055},{"step":211,"reward":-12.412938117980957,"core":-4.114211082458496,"cost":-0.05231207236647606,"kl":0.0303554218262434,"entropy":0.5182302668690681},{"step":213,"reward":-8.393760681152344,"core":-2.702960252761841,"cost":-0.11063242703676224,"kl":0.03837180929258466,"entropy":0.5723302215337753},{"step":215,"reward":-10.190878868103027,"core":-3.3424434661865234,"cost":-0.10230153799057007,"kl":0.021336837206035852,"entropy":0.3919755592942238},{"step":217,"reward":-13.358123779296875,"core":-4.364878177642822,"cost":-0.13032172620296478,"kl":0.0296051730401814,"entropy":0.49918343871831894},{"step":219,"reward":-4.8958587646484375,"core":-1.5548925399780273,"cost":-0.00334542035125196,"kl":0.02933308994397521,"entropy":0.4368187040090561},{"step":221,"reward":-13.698987007141113,"core":-4.508419513702393,"cost":-0.10880997031927109,"kl":0.02115978905931115,"entropy":0.5344072431325912},{"step":223,"reward":-8.54837703704834,"core":-2.786090612411499,"cost":-0.06145830824971199,"kl":0.03242670185863972,"entropy":0.5430159866809845},{"step":225,"reward":-11.806556701660156,"core":-3.879796266555786,"cost":-0.11855094879865646,"kl":0.032720651011914015,"entropy":0.6036143749952316},{"step":227,"reward":-10.518752098083496,"core":-3.461059331893921,"cost":-0.042445093393325806,"kl":0.03145836992189288,"entropy":0.5063014775514603},{"step":229,"reward":-9.543172836303711,"core":-3.1103391647338867,"cost":-0.09400475770235062,"kl":0.022004384081810713,"entropy":0.48638108372688293},{"step":231,"reward":-27.01768684387207,"core":-8.869457244873047,"cost":-0.40808096528053284,"kl":0.010837451322004199,"entropy":0.46294592320919037},{"step":233,"reward":-15.773594856262207,"core":-5.159839630126953,"cost":-0.2023339718580246,"kl":0.016883887350559235,"entropy":0.5011584982275963},{"step":235,"reward":-10.54106616973877,"core":-3.4740800857543945,"cost":-0.052554886788129807,"kl":0.027480059303343296,"entropy":0.5275929868221283},{"step":237,"reward":-8.32901668548584,"core":-2.716217279434204,"cost":-0.05858040973544121,"kl":0.0201008771546185,"entropy":0.4393726587295532},{"step":239,"reward":-12.550525665283203,"core":-4.116066932678223,"cost":-0.13644813001155853,"kl":0.02086499333381653,"entropy":0.5294757336378098},{"step":241,"reward":-5.0558342933654785,"core":-1.6176543235778809,"cost":-0.016116363927721977,"kl":0.015742533607408404,"entropy":0.4659617841243744},{"step":243,"reward":-14.20513916015625,"core":-4.687485218048096,"cost":-0.11482745409011841,"kl":0.021136560244485736,"entropy":0.6275429427623749},{"step":245,"reward":-9.432528495788574,"core":-3.0952250957489014,"cost":-0.051888152956962585,"kl":0.02061357954517007,"entropy":0.5339055731892586},{"step":247,"reward":-14.343274116516113,"core":-4.703146457672119,"cost":-0.19633495807647705,"kl":0.025640266481786966,"entropy":0.5061681568622589},{"step":249,"reward":-6.945835113525391,"core":-2.258295774459839,"cost":-0.047987889498472214,"kl":0.025433044880628586,"entropy":0.5177524387836456},{"step":251,"reward":-8.115649223327637,"core":-2.657243490219116,"cost":-0.04284091666340828,"kl":0.019861504435539246,"entropy":0.5089970380067825},{"step":253,"reward":-10.217384338378906,"core":-3.369816780090332,"cost":-0.05991721525788307,"kl":0.025313066318631172,"entropy":0.46111272275447845},{"step":255,"reward":-5.627060413360596,"core":-1.8169854879379272,"cost":0.0,"kl":0.01424933597445488,"entropy":0.579750269651413},{"step":257,"reward":-9.981301307678223,"core":-3.287435293197632,"cost":-0.06092936173081398,"kl":0.02256077667698264,"entropy":0.4987323060631752},{"step":259,"reward":-11.518264770507812,"core":-3.7835094928741455,"cost":-0.04454381391406059,"kl":0.017849911469966173,"entropy":0.5447148978710175},{"step":261,"reward":-13.67641544342041,"core":-4.487107753753662,"cost":-0.1429593712091446,"kl":0.0173882266972214,"entropy":0.5123205184936523},{"step":263,"reward":-14.145133972167969,"core":-4.64971399307251,"cost":-0.08284822851419449,"kl":0.007992055150680244,"entropy":0.5294717699289322},{"step":265,"reward":-9.808170318603516,"core":-3.2157394886016846,"cost":-0.0865154042840004,"kl":0.011588356457650661,"entropy":0.5028635784983635},{"step":267,"reward":-8.879673957824707,"core":-2.8905270099639893,"cost":-0.054234579205513,"kl":0.021886996924877167,"entropy":0.5115016475319862},{"step":269,"reward":-7.171396732330322,"core":-2.3654472827911377,"cost":-0.003987560514360666,"kl":0.02839750424027443,"entropy":0.5544620156288147},{"step":271,"reward":-7.895923614501953,"core":-2.5717289447784424,"cost":-0.06904250383377075,"kl":0.020692999940365553,"entropy":0.5402009040117264},{"step":273,"reward":-6.141524791717529,"core":-1.9684633016586304,"cost":-0.044110506772994995,"kl":0.012537846807390451,"entropy":0.4584871083498001},{"step":275,"reward":-18.795005798339844,"core":-6.192409515380859,"cost":-0.20369212329387665,"kl":0.016358648659661412,"entropy":0.4767470508813858},{"step":277,"reward":-5.133007049560547,"core":-1.6618942022323608,"cost":-0.0064477031119167805,"kl":0.014184285886585712,"entropy":0.516801580786705},{"step":279,"reward":-10.313136100769043,"core":-3.3657655715942383,"cost":-0.11163409799337387,"kl":0.0194746651686728,"entropy":0.5004502907395363},{"step":281,"reward":-14.381367683410645,"core":-4.707425594329834,"cost":-0.17816714942455292,"kl":0.018744315952062607,"entropy":0.5456597357988358},{"step":283,"reward":-6.401283264160156,"core":-2.079334020614624,"cost":-0.05152612552046776,"kl":0.016104345675557852,"entropy":0.5464725941419601},{"step":285,"reward":-15.528823852539062,"core":-5.07607889175415,"cost":-0.19497446715831757,"kl":0.00926815951243043,"entropy":0.4755127653479576},{"step":287,"reward":-11.41671085357666,"core":-3.790045976638794,"cost":-0.03399975225329399,"kl":0.013796093175187707,"entropy":0.6037440448999405},{"step":289,"reward":-15.92176342010498,"core":-5.194521427154541,"cost":-0.23311753571033478,"kl":0.019253374310210347,"entropy":0.5465015396475792},{"step":291,"reward":-17.018125534057617,"core":-5.567188739776611,"cost":-0.22660352289676666,"kl":0.009280511410906911,"entropy":0.5137267559766769},{"step":293,"reward":-8.6776762008667,"core":-2.8281753063201904,"cost":-0.08482792228460312,"kl":0.012847515055909753,"entropy":0.5664772540330887},{"step":295,"reward":-5.9905877113342285,"core":-1.9468082189559937,"cost":-0.01704290881752968,"kl":0.019591689808294177,"entropy":0.5617358833551407},{"step":297,"reward":-5.852162837982178,"core":-1.9000266790390015,"cost":-0.031748346984386444,"kl":0.017209808574989438,"entropy":0.6060368567705154},{"step":299,"reward":-10.5609769821167,"core":-3.469182252883911,"cost":-0.09498242288827896,"kl":0.01947742816992104,"entropy":0.5190985798835754},{"step":301,"reward":-8.738869667053223,"core":-2.8382060527801514,"cost":-0.08282633125782013,"kl":0.009890531888231635,"entropy":0.5198365151882172},{"step":303,"reward":-11.374740600585938,"core":-3.726600408554077,"cost":-0.08493226766586304,"kl":0.009484190493822098,"entropy":0.5321483463048935},{"step":305,"reward":-10.391274452209473,"core":-3.4002997875213623,"cost":-0.08473234623670578,"kl":0.021315118297934532,"entropy":0.54887904971838},{"step":307,"reward":-14.957402229309082,"core":-4.884034633636475,"cost":-0.21720945835113525,"kl":0.020983899012207985,"entropy":0.5204496011137962},{"step":309,"reward":-9.595603942871094,"core":-3.134749174118042,"cost":-0.0776124820113182,"kl":0.018903160002082586,"entropy":0.48347049951553345},{"step":311,"reward":-10.355636596679688,"core":-3.4027316570281982,"cost":-0.07414200901985168,"kl":0.016936689615249634,"entropy":0.5108914747834206},{"step":313,"reward":-16.221433639526367,"core":-5.285274982452393,"cost":-0.26422229409217834,"kl":0.0173710398375988,"entropy":0.5926690250635147},{"step":315,"reward":-9.911384582519531,"core":-3.260303497314453,"cost":-0.0350349061191082,"kl":0.015308103756979108,"entropy":0.46611789613962173},{"step":317,"reward":-16.089311599731445,"core":-5.295327663421631,"cost":-0.18010805547237396,"kl":0.026868475368246436,"entropy":0.5094800963997841},{"step":319,"reward":-12.024086952209473,"core":-3.959627866744995,"cost":-0.10292018204927444,"kl":0.01591547904536128,"entropy":0.5273362100124359},{"step":321,"reward":-11.358447074890137,"core":-3.728874444961548,"cost":-0.0726696327328682,"kl":0.01460655895061791,"entropy":0.46985702961683273},{"step":323,"reward":-13.206920623779297,"core":-4.322205066680908,"cost":-0.15084116160869598,"kl":0.011196302715688944,"entropy":0.5334618464112282},{"step":325,"reward":-7.521919250488281,"core":-2.4395134449005127,"cost":-0.033771347254514694,"kl":0.041907033417373896,"entropy":0.4804714247584343},{"step":327,"reward":-15.624606132507324,"core":-5.175960063934326,"cost":-0.07864762097597122,"kl":0.01764809782616794,"entropy":0.5616945922374725},{"step":329,"reward":-7.773281574249268,"core":-2.5533030033111572,"cost":-0.032238587737083435,"kl":0.013499509543180466,"entropy":0.5687799900770187},{"step":331,"reward":-10.526294708251953,"core":-3.4519996643066406,"cost":-0.10669640451669693,"kl":0.021583233028650284,"entropy":0.5927147567272186},{"step":333,"reward":-13.05860424041748,"core":-4.290857791900635,"cost":-0.14543800055980682,"kl":0.013465412193909287,"entropy":0.5968281179666519},{"step":335,"reward":-11.20288372039795,"core":-3.677387237548828,"cost":-0.08727981895208359,"kl":0.017653871793299913,"entropy":0.4967569038271904},{"step":337,"reward":-9.685723304748535,"core":-3.163437604904175,"cost":-0.0701620876789093,"kl":0.014662222703918815,"entropy":0.5605244785547256},{"step":339,"reward":-15.005355834960938,"core":-4.9050068855285645,"cost":-0.19854414463043213,"kl":0.022043507313355803,"entropy":0.48732513189315796},{"step":341,"reward":-6.841663360595703,"core":-2.2380921840667725,"cost":-0.02756122499704361,"kl":0.032014778815209866,"entropy":0.5268120765686035},{"step":343,"reward":-13.902377128601074,"core":-4.547571182250977,"cost":-0.17182254791259766,"kl":0.02374913590028882,"entropy":0.5685146749019623},{"step":345,"reward":-4.3864850997924805,"core":-1.4009097814559937,"cost":-0.013228666968643665,"kl":0.013115198584273458,"entropy":0.5560465827584267},{"step":347,"reward":-9.901206970214844,"core":-3.2396209239959717,"cost":-0.04834166541695595,"kl":0.005916665948461741,"entropy":0.487393356859684},{"step":349,"reward":-10.215153694152832,"core":-3.3672640323638916,"cost":-0.0416857935488224,"kl":0.013788730488158762,"entropy":0.5096940472722054},{"step":351,"reward":-16.598875045776367,"core":-5.437798023223877,"cost":-0.18101315200328827,"kl":0.011386414524167776,"entropy":0.5357311442494392},{"step":353,"reward":-7.059184551239014,"core":-2.2724955081939697,"cost":-0.042992010712623596,"kl":0.020791388116776943,"entropy":0.49485645443201065},{"step":355,"reward":-8.40558910369873,"core":-2.7464733123779297,"cost":-0.03244559466838837,"kl":0.011509503470733762,"entropy":0.49147599190473557},{"step":357,"reward":-12.082915306091309,"core":-3.9479598999023438,"cost":-0.11661545187234879,"kl":0.013770770281553268,"entropy":0.5067304894328117},{"step":359,"reward":-6.7169623374938965,"core":-2.179004430770874,"cost":-0.06459110230207443,"kl":0.043663824908435345,"entropy":0.4841361865401268},{"step":361,"reward":-8.182098388671875,"core":-2.6835882663726807,"cost":-0.023853948339819908,"kl":0.013576501980423927,"entropy":0.5158685445785522},{"step":363,"reward":-11.468836784362793,"core":-3.7793967723846436,"cost":-0.08972674608230591,"kl":0.00952410267200321,"entropy":0.5681114494800568},{"step":365,"reward":-10.705897331237793,"core":-3.522871732711792,"cost":-0.05917331576347351,"kl":0.014295356348156929,"entropy":0.4464244917035103},{"step":367,"reward":-9.00489330291748,"core":-2.9626331329345703,"cost":-0.03628241643309593,"kl":0.02450030343607068,"entropy":0.5483388602733612},{"step":369,"reward":-8.5415620803833,"core":-2.8142216205596924,"cost":-0.02450476586818695,"kl":0.014713420066982508,"entropy":0.5739676505327225},{"step":371,"reward":-12.156121253967285,"core":-3.977832078933716,"cost":-0.11963006108999252,"kl":0.020987405441701412,"entropy":0.4898766055703163},{"step":373,"reward":-9.564300537109375,"core":-3.1359565258026123,"cost":-0.05176648870110512,"kl":0.013006569119170308,"entropy":0.5965258926153183},{"step":375,"reward":-10.29148006439209,"core":-3.4060306549072266,"cost":-0.02332608960568905,"kl":0.009226397378370166,"entropy":0.4978052079677582},{"step":377,"reward":-11.93049144744873,"core":-3.900369644165039,"cost":-0.1089160218834877,"kl":0.012248497689142823,"entropy":0.5097445622086525},{"step":379,"reward":-15.12364673614502,"core":-4.963296890258789,"cost":-0.19040073454380035,"kl":0.018352016573771834,"entropy":0.49486320465803146},{"step":381,"reward":-6.597657680511475,"core":-2.1208415031433105,"cost":-0.03316672518849373,"kl":0.020202654413878918,"entropy":0.5170570984482765},{"step":383,"reward":-17.338834762573242,"core":-5.704652309417725,"cost":-0.2070191353559494,"kl":0.01991824759170413,"entropy":0.5822277963161469},{"step":385,"reward":-8.809046745300293,"core":-2.8889834880828857,"cost":-0.041885942220687866,"kl":0.012241480522789061,"entropy":0.45904290676116943},{"step":387,"reward":-14.998534202575684,"core":-4.9385552406311035,"cost":-0.12124192714691162,"kl":0.007481004344299436,"entropy":0.511277548968792},{"step":389,"reward":-21.194345474243164,"core":-6.989589214324951,"cost":-0.2057403177022934,"kl":0.007660707226023078,"entropy":0.4702383801341057},{"step":391,"reward":-10.209675788879395,"core":-3.305316925048828,"cost":-0.08405569195747375,"kl":0.006888504372909665,"entropy":0.48299068212509155},{"step":393,"reward":-8.653779029846191,"core":-2.8315181732177734,"cost":-0.06241793930530548,"kl":0.012320303823798895,"entropy":0.46464818716049194},{"step":395,"reward":-12.97044849395752,"core":-4.2344207763671875,"cost":-0.13547158241271973,"kl":0.007485249196179211,"entropy":0.4820922687649727},{"step":397,"reward":-8.806207656860352,"core":-2.876957654953003,"cost":-0.06537988036870956,"kl":0.012592100538313389,"entropy":0.5417289286851883},{"step":399,"reward":-14.84449291229248,"core":-4.852519989013672,"cost":-0.19466710090637207,"kl":0.007759009022265673,"entropy":0.5703863054513931},{"step":401,"reward":-9.592093467712402,"core":-3.145578145980835,"cost":-0.0736655592918396,"kl":0.013420750619843602,"entropy":0.5297279506921768},{"step":403,"reward":-7.581618785858154,"core":-2.477287769317627,"cost":-0.001116194878704846,"kl":0.011226815870031714,"entropy":0.5824639052152634},{"step":405,"reward":-5.290136814117432,"core":-1.7054208517074585,"cost":-0.020733356475830078,"kl":0.01600916823372245,"entropy":0.6392431855201721},{"step":407,"reward":-9.066292762756348,"core":-2.960880994796753,"cost":-0.032935500144958496,"kl":0.0074201253009960055,"entropy":0.4733533710241318},{"step":409,"reward":-8.202555656433105,"core":-2.6919357776641846,"cost":-0.0185076966881752,"kl":0.009798162151128054,"entropy":0.5373276770114899},{"step":411,"reward":-11.991607666015625,"core":-3.922016143798828,"cost":-0.1274118423461914,"kl":0.028154734522104263,"entropy":0.4891091287136078},{"step":413,"reward":-11.520233154296875,"core":-3.786816358566284,"cost":-0.09783357381820679,"kl":0.010645734844729304,"entropy":0.5363897308707237},{"step":415,"reward":-9.900908470153809,"core":-3.274258852005005,"cost":-0.012103029526770115,"kl":0.01437675254419446,"entropy":0.4887971729040146},{"step":417,"reward":-15.414176940917969,"core":-5.043155670166016,"cost":-0.1555406153202057,"kl":0.008452646667137742,"entropy":0.4111175611615181},{"step":419,"reward":-14.309794425964355,"core":-4.692892551422119,"cost":-0.17440833151340485,"kl":0.008378339000046253,"entropy":0.5386754870414734},{"step":421,"reward":-10.11715030670166,"core":-3.3200302124023438,"cost":-0.054649438709020615,"kl":0.009141416288912296,"entropy":0.5116661041975021},{"step":423,"reward":-11.795985221862793,"core":-3.889021635055542,"cost":-0.07037758827209473,"kl":0.00900999957229942,"entropy":0.5862306132912636},{"step":425,"reward":-7.660301208496094,"core":-2.5210821628570557,"cost":-0.042809486389160156,"kl":0.01930486224591732,"entropy":0.5639926120638847},{"step":427,"reward":-22.064233779907227,"core":-7.22091817855835,"cost":-0.3341456949710846,"kl":0.00976989429909736,"entropy":0.5014162436127663},{"step":429,"reward":-13.051276206970215,"core":-4.260395050048828,"cost":-0.1811848282814026,"kl":0.011471323436126113,"entropy":0.5259917303919792},{"step":431,"reward":-12.403111457824707,"core":-4.084300518035889,"cost":-0.10861744731664658,"kl":0.011549837072379887,"entropy":0.47495755553245544},{"step":433,"reward":-17.28386116027832,"core":-5.684206485748291,"cost":-0.16333220899105072,"kl":0.011424128315411508,"entropy":0.5219195485115051},{"step":435,"reward":-8.369912147521973,"core":-2.7426414489746094,"cost":-0.07305865734815598,"kl":0.012403146829456091,"entropy":0.5624955222010612},{"step":437,"reward":-12.73153305053711,"core":-4.189355373382568,"cost":-0.10511871427297592,"kl":0.006969744572415948,"entropy":0.46774104982614517},{"step":439,"reward":-12.845130920410156,"core":-4.212449550628662,"cost":-0.10056338459253311,"kl":0.008302788715809584,"entropy":0.50775246322155},{"step":441,"reward":-9.884658813476562,"core":-3.2314364910125732,"cost":-0.06894508749246597,"kl":0.014420020044781268,"entropy":0.4842904210090637},{"step":443,"reward":-15.280108451843262,"core":-5.062379837036133,"cost":-0.08978796750307083,"kl":0.007533548632636666,"entropy":0.5293328687548637},{"step":445,"reward":-10.178820610046387,"core":-3.3467376232147217,"cost":-0.0536109060049057,"kl":0.018663462484255433,"entropy":0.47233039140701294},{"step":447,"reward":-6.238815784454346,"core":-2.056098699569702,"cost":-0.023678245022892952,"kl":0.012023151270113885,"entropy":0.5145761296153069},{"step":449,"reward":-8.672260284423828,"core":-2.826662063598633,"cost":-0.08255469053983688,"kl":0.00711675186175853,"entropy":0.45804788172245026}],"nonzero_advantage":[{"step":1,"nonzero":0.0},{"step":3,"nonzero":0.0},{"step":5,"nonzero":0.125},{"step":7,"nonzero":0.0},{"step":9,"nonzero":0.0},{"step":11,"nonzero":0.08333333333333333},{"step":13,"nonzero":0.0},{"step":15,"nonzero":0.041666666666666664},{"step":17,"nonzero":0.0},{"step":19,"nonzero":0.0},{"step":21,"nonzero":0.0},{"step":23,"nonzero":0.0},{"step":25,"nonzero":0.0},{"step":27,"nonzero":0.0},{"step":29,"nonzero":0.041666666666666664},{"step":31,"nonzero":0.0},{"step":33,"nonzero":0.0},{"step":35,"nonzero":0.0},{"step":37,"nonzero":0.0},{"step":39,"nonzero":0.0},{"step":41,"nonzero":0.0},{"step":43,"nonzero":0.0},{"step":45,"nonzero":0.10416666666666667},{"step":47,"nonzero":0.0},{"step":49,"nonzero":0.0},{"step":51,"nonzero":0.0},{"step":53,"nonzero":0.0},{"step":55,"nonzero":0.0},{"step":57,"nonzero":0.0},{"step":59,"nonzero":0.0},{"step":61,"nonzero":0.0},{"step":63,"nonzero":0.0},{"step":65,"nonzero":0.125},{"step":67,"nonzero":0.0},{"step":69,"nonzero":0.0},{"step":71,"nonzero":0.0},{"step":73,"nonzero":0.0},{"step":75,"nonzero":0.0},{"step":77,"nonzero":0.0},{"step":79,"nonzero":0.125},{"step":81,"nonzero":0.0},{"step":83,"nonzero":0.0},{"step":85,"nonzero":0.0},{"step":87,"nonzero":0.0},{"step":89,"nonzero":0.0},{"step":91,"nonzero":0.0},{"step":93,"nonzero":0.0},{"step":95,"nonzero":0.0625},{"step":97,"nonzero":0.0},{"step":99,"nonzero":0.0},{"step":101,"nonzero":0.0},{"step":103,"nonzero":0.0},{"step":105,"nonzero":0.0},{"step":107,"nonzero":0.0},{"step":109,"nonzero":0.0},{"step":111,"nonzero":0.0},{"step":113,"nonzero":0.0},{"step":115,"nonzero":0.0},{"step":117,"nonzero":0.0},{"step":119,"nonzero":0.0},{"step":121,"nonzero":0.0},{"step":123,"nonzero":0.0},{"step":125,"nonzero":0.0},{"step":127,"nonzero":0.0},{"step":129,"nonzero":0.0},{"step":131,"nonzero":0.0},{"step":133,"nonzero":0.0625},{"step":135,"nonzero":0.0},{"step":137,"nonzero":0.0},{"step":139,"nonzero":0.0},{"step":141,"nonzero":0.0},{"step":143,"nonzero":0.14583333333333334},{"step":145,"nonzero":0.0},{"step":147,"nonzero":0.10416666666666667},{"step":149,"nonzero":0.0},{"step":151,"nonzero":0.0},{"step":153,"nonzero":0.08333333333333333},{"step":155,"nonzero":0.16666666666666666},{"step":157,"nonzero":0.0},{"step":159,"nonzero":0.0},{"step":161,"nonzero":0.0},{"step":163,"nonzero":0.0},{"step":165,"nonzero":0.0},{"step":167,"nonzero":0.10416666666666667},{"step":169,"nonzero":0.0625},{"step":171,"nonzero":0.0},{"step":173,"nonzero":0.0},{"step":175,"nonzero":0.0},{"step":177,"nonzero":0.0625},{"step":179,"nonzero":0.0},{"step":181,"nonzero":0.14583333333333334},{"step":183,"nonzero":0.0},{"step":185,"nonzero":0.0},{"step":187,"nonzero":0.0},{"step":189,"nonzero":0.041666666666666664},{"step":191,"nonzero":0.0},{"step":193,"nonzero":0.0625},{"step":195,"nonzero":0.0},{"step":197,"nonzero":0.041666666666666664},{"step":199,"nonzero":0.16666666666666666},{"step":201,"nonzero":0.0},{"step":203,"nonzero":0.0},{"step":205,"nonzero":0.14583333333333334},{"step":207,"nonzero":0.0625},{"step":209,"nonzero":0.0},{"step":211,"nonzero":0.0},{"step":213,"nonzero":0.5208333333333334},{"step":215,"nonzero":0.22916666666666666},{"step":217,"nonzero":0.0},{"step":219,"nonzero":0.0},{"step":221,"nonzero":0.10416666666666667},{"step":223,"nonzero":0.0},{"step":225,"nonzero":0.22916666666666666},{"step":227,"nonzero":0.0},{"step":229,"nonzero":0.10416666666666667},{"step":231,"nonzero":0.0},{"step":233,"nonzero":0.14583333333333334},{"step":235,"nonzero":0.0},{"step":237,"nonzero":0.0},{"step":239,"nonzero":0.16666666666666666},{"step":241,"nonzero":0.14583333333333334},{"step":243,"nonzero":0.0},{"step":245,"nonzero":0.10416666666666667},{"step":247,"nonzero":0.0},{"step":249,"nonzero":0.0},{"step":251,"nonzero":0.0625},{"step":253,"nonzero":0.0},{"step":255,"nonzero":0.0},{"step":257,"nonzero":0.0},{"step":259,"nonzero":0.0},{"step":261,"nonzero":0.0},{"step":263,"nonzero":0.0},{"step":265,"nonzero":0.0},{"step":267,"nonzero":0.0},{"step":269,"nonzero":0.14583333333333334},{"step":271,"nonzero":0.0},{"step":273,"nonzero":0.0},{"step":275,"nonzero":0.14583333333333334},{"step":277,"nonzero":0.3541666666666667},{"step":279,"nonzero":0.1875},{"step":281,"nonzero":0.0},{"step":283,"nonzero":0.0},{"step":285,"nonzero":0.0},{"step":287,"nonzero":0.0},{"step":289,"nonzero":0.10416666666666667},{"step":291,"nonzero":0.0},{"step":293,"nonzero":0.08333333333333333},{"step":295,"nonzero":0.0},{"step":297,"nonzero":0.125},{"step":299,"nonzero":0.08333333333333333},{"step":301,"nonzero":0.0},{"step":303,"nonzero":0.0},{"step":305,"nonzero":0.0},{"step":307,"nonzero":0.20833333333333334},{"step":309,"nonzero":0.0},{"step":311,"nonzero":0.0},{"step":313,"nonzero":0.0},{"step":315,"nonzero":0.125},{"step":317,"nonzero":0.0},{"step":319,"nonzero":0.0},{"step":321,"nonzero":0.0},{"step":323,"nonzero":0.10416666666666667},{"step":325,"nonzero":0.14583333333333334},{"step":327,"nonzero":0.0},{"step":329,"nonzero":0.0},{"step":331,"nonzero":0.22916666666666666},{"step":333,"nonzero":0.0},{"step":335,"nonzero":0.0},{"step":337,"nonzero":0.0},{"step":339,"nonzero":0.0},{"step":341,"nonzero":0.0},{"step":343,"nonzero":0.16666666666666666},{"step":345,"nonzero":0.0},{"step":347,"nonzero":0.0},{"step":349,"nonzero":0.08333333333333333},{"step":351,"nonzero":0.0},{"step":353,"nonzero":0.0},{"step":355,"nonzero":0.0},{"step":357,"nonzero":0.0},{"step":359,"nonzero":0.3541666666666667},{"step":361,"nonzero":0.08333333333333333},{"step":363,"nonzero":0.0},{"step":365,"nonzero":0.0},{"step":367,"nonzero":0.0625},{"step":369,"nonzero":0.0},{"step":371,"nonzero":0.0},{"step":373,"nonzero":0.0},{"step":375,"nonzero":0.0},{"step":377,"nonzero":0.0},{"step":379,"nonzero":0.10416666666666667},{"step":381,"nonzero":0.0},{"step":383,"nonzero":0.0},{"step":385,"nonzero":0.0},{"step":387,"nonzero":0.0},{"step":389,"nonzero":0.0},{"step":391,"nonzero":0.0},{"step":393,"nonzero":0.1875},{"step":395,"nonzero":0.0},{"step":397,"nonzero":0.0},{"step":399,"nonzero":0.0},{"step":401,"nonzero":0.3125},{"step":403,"nonzero":0.0},{"step":405,"nonzero":0.0},{"step":407,"nonzero":0.0},{"step":409,"nonzero":0.0},{"step":411,"nonzero":0.16666666666666666},{"step":413,"nonzero":0.0},{"step":415,"nonzero":0.0},{"step":417,"nonzero":0.0},{"step":419,"nonzero":0.0},{"step":421,"nonzero":0.0},{"step":423,"nonzero":0.0},{"step":425,"nonzero":0.0},{"step":427,"nonzero":0.0},{"step":429,"nonzero":0.0},{"step":431,"nonzero":0.0},{"step":433,"nonzero":0.0},{"step":435,"nonzero":0.0},{"step":437,"nonzero":0.0},{"step":439,"nonzero":0.0},{"step":441,"nonzero":0.0},{"step":443,"nonzero":0.0},{"step":445,"nonzero":0.0},{"step":447,"nonzero":0.16666666666666666},{"step":449,"nonzero":0.08333333333333333}]},{"id":"b2-nofloor","label":"No floor (second slice)","run_id":"1791065555","cpcv_floor":null,"train_steps":[{"step":1,"reward":-15.024345397949219,"core":-4.9500555992126465,"cost":-0.14259950816631317,"kl":0.0,"entropy":0.462548591196537},{"step":3,"reward":-10.082160949707031,"core":-3.3086729049682617,"cost":-0.03883307799696922,"kl":0.0009237638878403232,"entropy":0.5168384984135628},{"step":5,"reward":-10.771801948547363,"core":-3.5568017959594727,"cost":-0.041870877146720886,"kl":0.003357605019118637,"entropy":0.520971342921257},{"step":7,"reward":-13.310376167297363,"core":-4.3911004066467285,"cost":-0.09479834884405136,"kl":0.010004305746406317,"entropy":0.41163191199302673},{"step":9,"reward":-10.399726867675781,"core":-3.411717653274536,"cost":-0.05417333170771599,"kl":0.019577782601118088,"entropy":0.35852622240781784},{"step":11,"reward":-13.287567138671875,"core":-4.272876262664795,"cost":-0.21317291259765625,"kl":0.029823944438248873,"entropy":0.3183613047003746},{"step":13,"reward":-4.079253673553467,"core":-1.2851415872573853,"cost":0.0,"kl":0.0434271190315485,"entropy":0.3496931791305542},{"step":15,"reward":-8.611893653869629,"core":-2.809643507003784,"cost":-0.06147712469100952,"kl":0.058590625412762165,"entropy":0.40827639400959015},{"step":17,"reward":-7.135105133056641,"core":-2.3137176036834717,"cost":-0.037482667714357376,"kl":0.057552938349545,"entropy":0.4531506672501564},{"step":19,"reward":-9.453614234924316,"core":-3.094193696975708,"cost":-0.05341751500964165,"kl":0.06340472307056189,"entropy":0.4891667440533638},{"step":21,"reward":-4.3868937492370605,"core":-1.352211356163025,"cost":-0.03142793849110603,"kl":0.149184362962842,"entropy":0.5398175418376923},{"step":23,"reward":-9.72822093963623,"core":-3.1971731185913086,"cost":-0.08296694606542587,"kl":0.0517151765525341,"entropy":0.5132918581366539},{"step":25,"reward":-7.345247268676758,"core":-2.3782811164855957,"cost":-0.012610815465450287,"kl":0.04675349313765764,"entropy":0.5822728127241135},{"step":27,"reward":-4.53830099105835,"core":-1.429815411567688,"cost":-0.04562872275710106,"kl":0.048846544697880745,"entropy":0.5979006290435791},{"step":29,"reward":-6.752233028411865,"core":-2.1902973651885986,"cost":-0.029943762347102165,"kl":0.0774042159318924,"entropy":0.5998656898736954},{"step":31,"reward":-5.744497776031494,"core":-1.8235162496566772,"cost":-0.026325998827815056,"kl":0.05455074738711119,"entropy":0.5006675943732262},{"step":33,"reward":-6.915596008300781,"core":-2.2659943103790283,"cost":-0.002300956752151251,"kl":0.05594681669026613,"entropy":0.5409032702445984},{"step":35,"reward":-7.872831344604492,"core":-2.579514265060425,"cost":-0.018801650032401085,"kl":0.06093264650553465,"entropy":0.5630257874727249},{"step":37,"reward":-4.173734664916992,"core":-1.3177011013031006,"cost":0.0,"kl":0.05356743931770325,"entropy":0.561082199215889},{"step":39,"reward":-3.4722983837127686,"core":-1.0755974054336548,"cost":-0.0067526535131037235,"kl":0.06294527370482683,"entropy":0.5441936627030373},{"step":41,"reward":-4.504718780517578,"core":-1.4424875974655151,"cost":-0.0044022067449986935,"kl":0.07470085565000772,"entropy":0.5716954618692398},{"step":43,"reward":-3.9064712524414062,"core":-1.250733733177185,"cost":-0.0001770463422872126,"kl":0.09314458072185516,"entropy":0.5105762407183647},{"step":45,"reward":-16.881452560424805,"core":-5.518682479858398,"cost":-0.20120316743850708,"kl":0.07391565665602684,"entropy":0.49326879531145096},{"step":47,"reward":-6.937291622161865,"core":-2.226731061935425,"cost":-0.01832391321659088,"kl":0.08152334485203028,"entropy":0.4503880962729454},{"step":49,"reward":-8.105411529541016,"core":-2.6639373302459717,"cost":-0.010717305354773998,"kl":0.09715083800256252,"entropy":0.5305699110031128},{"step":51,"reward":-5.266147136688232,"core":-1.6953204870224,"cost":-0.002100455341860652,"kl":0.1223644521087408,"entropy":0.51634581387043},{"step":53,"reward":-2.955937623977661,"core":-0.90516597032547,"cost":0.0,"kl":0.13243345357477665,"entropy":0.4851000905036926},{"step":55,"reward":-3.3166306018829346,"core":-0.9694793820381165,"cost":-0.01731223426759243,"kl":0.09631652943789959,"entropy":0.43147800117731094},{"step":57,"reward":-8.050178527832031,"core":-2.6299989223480225,"cost":-0.008898870088160038,"kl":0.14615920931100845,"entropy":0.46016761660575867},{"step":59,"reward":-3.2069594860076904,"core":-0.9909799695014954,"cost":-0.00054197694407776,"kl":0.14338131994009018,"entropy":0.4534284472465515},{"step":61,"reward":-4.931126117706299,"core":-1.5603183507919312,"cost":-0.008368206210434437,"kl":0.14869708009064198,"entropy":0.55441515147686},{"step":63,"reward":-1.7874633073806763,"core":-0.48759713768959045,"cost":0.0,"kl":0.14560735411942005,"entropy":0.6146889328956604},{"step":65,"reward":-2.1797866821289062,"core":-0.6368228793144226,"cost":0.0,"kl":0.11842760443687439,"entropy":0.5234540849924088},{"step":67,"reward":-3.4637796878814697,"core":-1.0596339702606201,"cost":0.0,"kl":0.15160860493779182,"entropy":0.4761442542076111},{"step":69,"reward":-2.281294107437134,"core":-0.6539795994758606,"cost":0.0,"kl":0.21616299450397491,"entropy":0.5458009988069534},{"step":71,"reward":-5.3815131187438965,"core":-1.768389105796814,"cost":0.0,"kl":0.13786941953003407,"entropy":0.5689463019371033},{"step":73,"reward":-3.3832738399505615,"core":-1.0399998426437378,"cost":-0.005667660851031542,"kl":0.14536568336188793,"entropy":0.506914883852005},{"step":75,"reward":-2.7950541973114014,"core":-0.8349296450614929,"cost":0.0,"kl":0.16448241099715233,"entropy":0.6143966466188431},{"step":77,"reward":-3.5553932189941406,"core":-1.1023911237716675,"cost":-0.003407903714105487,"kl":0.2688793055713177,"entropy":0.5164211690425873},{"step":79,"reward":-3.0580339431762695,"core":-0.927415668964386,"cost":0.0,"kl":0.19909843988716602,"entropy":0.5418968200683594},{"step":81,"reward":-6.748785495758057,"core":-2.162184715270996,"cost":-0.07452260702848434,"kl":0.11767086572945118,"entropy":0.5573064833879471},{"step":83,"reward":-6.429607391357422,"core":-2.0805141925811768,"cost":-0.04734804853796959,"kl":0.19723543152213097,"entropy":0.6056821346282959},{"step":85,"reward":-8.06812858581543,"core":-2.6046626567840576,"cost":-0.0700676441192627,"kl":0.18065627291798592,"entropy":0.5562285333871841},{"step":87,"reward":-5.364265441894531,"core":-1.74125337600708,"cost":-0.0008052322664298117,"kl":0.12661584094166756,"entropy":0.49854961037635803},{"step":89,"reward":-2.872058153152466,"core":-0.859294593334198,"cost":0.0,"kl":0.13929017633199692,"entropy":0.6436971426010132},{"step":91,"reward":-7.335352420806885,"core":-2.382929563522339,"cost":-0.039809904992580414,"kl":0.16527151316404343,"entropy":0.7142766565084457},{"step":93,"reward":-4.102288246154785,"core":-1.2625738382339478,"cost":-0.012374510057270527,"kl":0.14698223397135735,"entropy":0.5831062197685242},{"step":95,"reward":-3.075692892074585,"core":-0.9361693263053894,"cost":0.0,"kl":0.13852408155798912,"entropy":0.6389679312705994},{"step":97,"reward":-4.247234344482422,"core":-1.3374263048171997,"cost":0.0,"kl":0.19131019711494446,"entropy":0.5796929597854614},{"step":99,"reward":-3.136855125427246,"core":-0.9780805706977844,"cost":0.0,"kl":0.13018841855227947,"entropy":0.5400039851665497},{"step":101,"reward":-4.845845699310303,"core":-1.538785457611084,"cost":-0.0029633529484272003,"kl":0.21537430584430695,"entropy":0.6004531979560852},{"step":103,"reward":-6.79464864730835,"core":-2.2194106578826904,"cost":-0.019363392144441605,"kl":0.1706443876028061,"entropy":0.5354473441839218},{"step":105,"reward":-2.859095573425293,"core":-0.8449538350105286,"cost":0.0,"kl":0.1495663058012724,"entropy":0.5498783215880394},{"step":107,"reward":-3.416825294494629,"core":-1.0071064233779907,"cost":-0.015851881355047226,"kl":0.1558969970792532,"entropy":0.45844000577926636},{"step":109,"reward":-2.762709379196167,"core":-0.8012452721595764,"cost":-0.009294404648244381,"kl":0.16717228293418884,"entropy":0.49540556967258453},{"step":111,"reward":-3.1623451709747314,"core":-0.9549462199211121,"cost":0.0,"kl":0.20365794375538826,"entropy":0.5108658969402313},{"step":113,"reward":-4.508808612823486,"core":-1.406760573387146,"cost":-0.00744912214577198,"kl":0.16362872906029224,"entropy":0.49006712436676025},{"step":115,"reward":-2.4733834266662598,"core":-0.704683780670166,"cost":0.0,"kl":0.2825525887310505,"entropy":0.3956702649593353},{"step":117,"reward":-3.4804744720458984,"core":-1.069244146347046,"cost":-0.0034179093781858683,"kl":0.28678473830223083,"entropy":0.4868672117590904},{"step":119,"reward":-5.442718505859375,"core":-1.739534854888916,"cost":-0.0036267880350351334,"kl":0.2365097440779209,"entropy":0.45723532140254974},{"step":121,"reward":-6.0890350341796875,"core":-1.921571135520935,"cost":-0.03347530588507652,"kl":0.27646925300359726,"entropy":0.49435242265462875},{"step":123,"reward":-2.69632887840271,"core":-0.7750956416130066,"cost":0.0,"kl":0.2359454520046711,"entropy":0.44815972447395325},{"step":125,"reward":-3.4100182056427,"core":-1.0410081148147583,"cost":0.0,"kl":0.21478386968374252,"entropy":0.4995637759566307},{"step":127,"reward":-2.900733232498169,"core":-0.8590931296348572,"cost":0.0,"kl":0.23660288006067276,"entropy":0.43094299733638763},{"step":129,"reward":-2.6544952392578125,"core":-0.8035556674003601,"cost":0.0,"kl":0.17799190059304237,"entropy":0.47708556801080704},{"step":131,"reward":-2.4487810134887695,"core":-0.7096132636070251,"cost":0.0,"kl":0.20786266401410103,"entropy":0.5075960755348206},{"step":133,"reward":-2.7432897090911865,"core":-0.7994491457939148,"cost":0.0,"kl":0.25564994663000107,"entropy":0.45480602234601974},{"step":135,"reward":-3.8732330799102783,"core":-1.2226024866104126,"cost":0.0,"kl":0.17868687584996223,"entropy":0.5803728997707367},{"step":137,"reward":-2.766284704208374,"core":-0.7992978096008301,"cost":-0.00334945903159678,"kl":0.17708908393979073,"entropy":0.5038105770945549},{"step":139,"reward":-2.546306848526001,"core":-0.7308276295661926,"cost":0.0,"kl":0.12067942880094051,"entropy":0.44896651804447174},{"step":141,"reward":-1.8911418914794922,"core":-0.5266970992088318,"cost":0.0,"kl":0.15725227072834969,"entropy":0.5584332942962646},{"step":143,"reward":-2.27231502532959,"core":-0.6660676002502441,"cost":0.0,"kl":0.13610462844371796,"entropy":0.5549067258834839},{"step":145,"reward":-3.556549072265625,"core":-1.1010608673095703,"cost":0.0,"kl":0.12164358049631119,"entropy":0.5503848940134048},{"step":147,"reward":-2.722820997238159,"core":-0.8091116547584534,"cost":0.0,"kl":0.12581760808825493,"entropy":0.5106368809938431},{"step":149,"reward":-8.531631469726562,"core":-2.7865216732025146,"cost":-0.057634491473436356,"kl":0.19942434132099152,"entropy":0.6292093247175217},{"step":151,"reward":-3.871805191040039,"core":-1.1985979080200195,"cost":-0.017383836209774017,"kl":0.1384180672466755,"entropy":0.5194528177380562},{"step":153,"reward":-3.9749209880828857,"core":-1.2484040260314941,"cost":-0.0043599228374660015,"kl":0.16843635216355324,"entropy":0.5552470982074738},{"step":155,"reward":-4.304203033447266,"core":-1.373232364654541,"cost":-0.0028392523527145386,"kl":0.1822838969528675,"entropy":0.6665670871734619},{"step":157,"reward":-3.607147455215454,"core":-1.0969873666763306,"cost":-0.0070173610001802444,"kl":0.14761137962341309,"entropy":0.49289269745349884},{"step":159,"reward":-2.4163730144500732,"core":-0.6825471520423889,"cost":0.0,"kl":0.13518363051116467,"entropy":0.4615084305405617},{"step":161,"reward":-3.6917226314544678,"core":-1.1653151512145996,"cost":0.0,"kl":0.10780007764697075,"entropy":0.6552509516477585}],"nonzero_advantage":[{"step":1,"nonzero":1.0},{"step":3,"nonzero":1.0},{"step":5,"nonzero":1.0},{"step":7,"nonzero":1.0},{"step":9,"nonzero":1.0},{"step":11,"nonzero":1.0},{"step":13,"nonzero":1.0},{"step":15,"nonzero":1.0},{"step":17,"nonzero":1.0},{"step":19,"nonzero":1.0},{"step":21,"nonzero":1.0},{"step":23,"nonzero":1.0},{"step":25,"nonzero":1.0},{"step":27,"nonzero":1.0},{"step":29,"nonzero":1.0},{"step":31,"nonzero":1.0},{"step":33,"nonzero":1.0},{"step":35,"nonzero":1.0},{"step":37,"nonzero":1.0},{"step":39,"nonzero":1.0},{"step":41,"nonzero":1.0},{"step":43,"nonzero":1.0},{"step":45,"nonzero":1.0},{"step":47,"nonzero":1.0},{"step":49,"nonzero":1.0},{"step":51,"nonzero":1.0},{"step":53,"nonzero":1.0},{"step":55,"nonzero":1.0},{"step":57,"nonzero":1.0},{"step":59,"nonzero":1.0},{"step":61,"nonzero":1.0},{"step":63,"nonzero":1.0},{"step":65,"nonzero":1.0},{"step":67,"nonzero":1.0},{"step":69,"nonzero":1.0},{"step":71,"nonzero":1.0},{"step":73,"nonzero":1.0},{"step":75,"nonzero":1.0},{"step":77,"nonzero":1.0},{"step":79,"nonzero":1.0},{"step":81,"nonzero":1.0},{"step":83,"nonzero":1.0},{"step":85,"nonzero":1.0},{"step":87,"nonzero":1.0},{"step":89,"nonzero":1.0},{"step":91,"nonzero":1.0},{"step":93,"nonzero":1.0},{"step":95,"nonzero":1.0},{"step":97,"nonzero":1.0},{"step":99,"nonzero":1.0},{"step":101,"nonzero":1.0},{"step":103,"nonzero":1.0},{"step":105,"nonzero":1.0},{"step":107,"nonzero":1.0},{"step":109,"nonzero":1.0},{"step":111,"nonzero":1.0},{"step":113,"nonzero":1.0},{"step":115,"nonzero":1.0},{"step":117,"nonzero":1.0},{"step":119,"nonzero":1.0},{"step":121,"nonzero":1.0},{"step":123,"nonzero":1.0},{"step":125,"nonzero":1.0},{"step":127,"nonzero":1.0},{"step":129,"nonzero":1.0},{"step":131,"nonzero":1.0},{"step":133,"nonzero":1.0},{"step":135,"nonzero":1.0},{"step":137,"nonzero":1.0},{"step":139,"nonzero":1.0},{"step":141,"nonzero":1.0},{"step":143,"nonzero":1.0},{"step":145,"nonzero":1.0},{"step":147,"nonzero":1.0},{"step":149,"nonzero":1.0},{"step":151,"nonzero":1.0},{"step":153,"nonzero":1.0},{"step":155,"nonzero":1.0},{"step":157,"nonzero":1.0},{"step":159,"nonzero":1.0},{"step":161,"nonzero":0.8333333333333334}]},{"id":"b3-floor0","label":"Advantage floor at 0 (third slice)","run_id":"1791044830","cpcv_floor":0.0,"train_steps":[{"step":1,"reward":-19.380556106567383,"core":-6.356939792633057,"cost":-0.2515563368797302,"kl":0.0,"entropy":0.4994571954011917},{"step":3,"reward":-11.108418464660645,"core":-3.641251802444458,"cost":-0.0856427326798439,"kl":0.0,"entropy":0.41777072101831436},{"step":5,"reward":-12.132441520690918,"core":-4.002344608306885,"cost":-0.06646369397640228,"kl":0.0,"entropy":0.48753925412893295},{"step":7,"reward":-14.499547004699707,"core":-4.773775577545166,"cost":-0.1261947602033615,"kl":0.0,"entropy":0.5145643576979637},{"step":9,"reward":-15.127578735351562,"core":-4.957881450653076,"cost":-0.18434065580368042,"kl":0.0,"entropy":0.4446423500776291},{"step":11,"reward":-12.522533416748047,"core":-4.107020378112793,"cost":-0.11521514505147934,"kl":0.0006035741535015404,"entropy":0.4987601786851883},{"step":13,"reward":-20.162595748901367,"core":-6.650894641876221,"cost":-0.18383245170116425,"kl":0.0007156547362683341,"entropy":0.5299279242753983},{"step":15,"reward":-20.839344024658203,"core":-6.837017059326172,"cost":-0.28517886996269226,"kl":0.0009291542373830453,"entropy":0.45610614120960236},{"step":17,"reward":-7.709020137786865,"core":-2.514478921890259,"cost":-0.08312220126390457,"kl":0.0013863496133126318,"entropy":0.5247602388262749},{"step":19,"reward":-11.677531242370605,"core":-3.8134632110595703,"cost":-0.14727981388568878,"kl":0.0017578919942025095,"entropy":0.5276640877127647},{"step":21,"reward":-14.1929292678833,"core":-4.632421493530273,"cost":-0.19608540832996368,"kl":0.0018962854810524732,"entropy":0.45349739491939545},{"step":23,"reward":-9.11485767364502,"core":-2.995413064956665,"cost":-0.02977406419813633,"kl":0.004907752154394984,"entropy":0.5859395861625671},{"step":25,"reward":-8.804177284240723,"core":-2.889744758605957,"cost":-0.030023222789168358,"kl":0.007853951887227595,"entropy":0.5543298050761223},{"step":27,"reward":-10.486407279968262,"core":-3.444143295288086,"cost":-0.09935811161994934,"kl":0.014418790116906166,"entropy":0.5553930997848511},{"step":29,"reward":-14.86445140838623,"core":-4.890725612640381,"cost":-0.13973870873451233,"kl":0.019908753922209144,"entropy":0.5522127151489258},{"step":31,"reward":-13.332966804504395,"core":-4.3714470863342285,"cost":-0.10786878317594528,"kl":0.016964562237262726,"entropy":0.49109987169504166},{"step":33,"reward":-9.701483726501465,"core":-3.19621205329895,"cost":-0.05435662344098091,"kl":0.01625952823087573,"entropy":0.5297179371118546},{"step":35,"reward":-23.29063606262207,"core":-7.625314712524414,"cost":-0.3634345531463623,"kl":0.011947403196245432,"entropy":0.4469980075955391},{"step":37,"reward":-10.60155200958252,"core":-3.450955629348755,"cost":-0.13949355483055115,"kl":0.020746879745274782,"entropy":0.5013721510767937},{"step":39,"reward":-7.666414737701416,"core":-2.524674415588379,"cost":-0.013913396745920181,"kl":0.021008880343288183,"entropy":0.5539582520723343},{"step":41,"reward":-7.000782489776611,"core":-2.278059720993042,"cost":-0.03577820584177971,"kl":0.02052603242918849,"entropy":0.4922083392739296},{"step":43,"reward":-7.750625133514404,"core":-2.5476629734039307,"cost":-0.042419806122779846,"kl":0.03882446652278304,"entropy":0.5488234013319016},{"step":45,"reward":-6.9084296226501465,"core":-2.2346389293670654,"cost":-0.017290210351347923,"kl":0.02037537842988968,"entropy":0.4290938749909401},{"step":47,"reward":-15.02894115447998,"core":-4.9488749504089355,"cost":-0.13919329643249512,"kl":0.02423831494525075,"entropy":0.4544745236635208},{"step":49,"reward":-8.434115409851074,"core":-2.751707077026367,"cost":-0.04382442310452461,"kl":0.018408820033073425,"entropy":0.4047381356358528},{"step":51,"reward":-8.394732475280762,"core":-2.728670835494995,"cost":-0.07078532129526138,"kl":0.022128676995635033,"entropy":0.43642572313547134},{"step":53,"reward":-7.120141506195068,"core":-2.3303518295288086,"cost":-0.03069145791232586,"kl":0.01884002727456391,"entropy":0.41381191462278366},{"step":55,"reward":-6.472951889038086,"core":-2.1278305053710938,"cost":-0.004238566383719444,"kl":0.01709903869777918,"entropy":0.48851896822452545},{"step":57,"reward":-9.030782699584961,"core":-2.9384567737579346,"cost":-0.074598029255867,"kl":0.020707099232822657,"entropy":0.47483063489198685},{"step":59,"reward":-14.248531341552734,"core":-4.676026344299316,"cost":-0.10233805328607559,"kl":0.015479732304811478,"entropy":0.5238659232854843},{"step":61,"reward":-13.94771957397461,"core":-4.550625324249268,"cost":-0.16096042096614838,"kl":0.013404862023890018,"entropy":0.4853162094950676},{"step":63,"reward":-10.491534233093262,"core":-3.4132003784179688,"cost":-0.11881721019744873,"kl":0.016082247020676732,"entropy":0.4393179714679718},{"step":65,"reward":-11.5027494430542,"core":-3.807429075241089,"cost":-0.06176574155688286,"kl":0.022586172446608543,"entropy":0.554929256439209},{"step":67,"reward":-13.586235046386719,"core":-4.47393798828125,"cost":-0.1035231351852417,"kl":0.01999138086102903,"entropy":0.4900866895914078},{"step":69,"reward":-6.9124298095703125,"core":-2.253364324569702,"cost":-0.015123228542506695,"kl":0.014783072285354137,"entropy":0.5550553649663925},{"step":71,"reward":-7.5219879150390625,"core":-2.4769694805145264,"cost":-0.02808801271021366,"kl":0.027176120784133673,"entropy":0.5945251882076263},{"step":73,"reward":-14.4603271484375,"core":-4.751654148101807,"cost":-0.16734285652637482,"kl":0.017917553894221783,"entropy":0.49222785234451294},{"step":75,"reward":-9.43139934539795,"core":-3.1231377124786377,"cost":-0.05291527882218361,"kl":0.019714479334652424,"entropy":0.5158459320664406},{"step":77,"reward":-21.355140686035156,"core":-6.97943115234375,"cost":-0.38659486174583435,"kl":0.016712085343897343,"entropy":0.4685230925679207},{"step":79,"reward":-8.059856414794922,"core":-2.642766237258911,"cost":-0.007900032214820385,"kl":0.018603578442707658,"entropy":0.46669113636016846},{"step":81,"reward":-12.912245750427246,"core":-4.266438961029053,"cost":-0.08250153809785843,"kl":0.023926662746816874,"entropy":0.5337513610720634},{"step":83,"reward":-15.5645170211792,"core":-5.113128185272217,"cost":-0.1722242385149002,"kl":0.019687980879098177,"entropy":0.4354344680905342},{"step":85,"reward":-19.9027156829834,"core":-6.559770107269287,"cost":-0.18932612240314484,"kl":0.012800663243979216,"entropy":0.4408257380127907},{"step":87,"reward":-6.999582290649414,"core":-2.291297197341919,"cost":-0.01786077953875065,"kl":0.018672720529139042,"entropy":0.5062825977802277},{"step":89,"reward":-7.618081569671631,"core":-2.5008771419525146,"cost":-0.03265269473195076,"kl":0.018436400685459375,"entropy":0.4517866000533104},{"step":91,"reward":-9.922442436218262,"core":-3.26324462890625,"cost":-0.05500897392630577,"kl":0.018568480852991343,"entropy":0.4693353995680809},{"step":93,"reward":-10.53515625,"core":-3.44336199760437,"cost":-0.08849690109491348,"kl":0.013781461399048567,"entropy":0.4841958060860634},{"step":95,"reward":-15.49454402923584,"core":-5.054958820343018,"cost":-0.21636606752872467,"kl":0.01211547083221376,"entropy":0.39231862872838974},{"step":97,"reward":-6.848781585693359,"core":-2.2578861713409424,"cost":-0.01410458143800497,"kl":0.016952684614807367,"entropy":0.49202343821525574},{"step":99,"reward":-8.010689735412598,"core":-2.592005729675293,"cost":-0.039779797196388245,"kl":0.010909767937846482,"entropy":0.5018869340419769},{"step":101,"reward":-12.1948881149292,"core":-4.00372838973999,"cost":-0.10850118845701218,"kl":0.013193734339438379,"entropy":0.4777904152870178},{"step":103,"reward":-12.67969036102295,"core":-4.14380407333374,"cost":-0.13089174032211304,"kl":0.014894961612299085,"entropy":0.47276586294174194},{"step":105,"reward":-16.006986618041992,"core":-5.256621360778809,"cost":-0.15978571772575378,"kl":0.012853797292336822,"entropy":0.47506020218133926},{"step":107,"reward":-7.075090408325195,"core":-2.2934646606445312,"cost":-0.05315201357007027,"kl":0.013350655557587743,"entropy":0.5236743539571762},{"step":109,"reward":-9.448193550109863,"core":-3.0778322219848633,"cost":-0.06107354536652565,"kl":0.013416775967925787,"entropy":0.41840454190969467},{"step":111,"reward":-13.848591804504395,"core":-4.5342116355896,"cost":-0.15004374086856842,"kl":0.015134139452129602,"entropy":0.4944343566894531},{"step":113,"reward":-14.189301490783691,"core":-4.67015266418457,"cost":-0.13071615993976593,"kl":0.01151907816529274,"entropy":0.45604363828897476},{"step":115,"reward":-18.190603256225586,"core":-5.959238052368164,"cost":-0.2458512932062149,"kl":0.01564799644984305,"entropy":0.4856045991182327},{"step":117,"reward":-13.910140037536621,"core":-4.583883285522461,"cost":-0.12889231741428375,"kl":0.010918847285211086,"entropy":0.4805697724223137},{"step":119,"reward":-13.067164421081543,"core":-4.255532741546631,"cost":-0.210530623793602,"kl":0.018471014918759465,"entropy":0.44583428651094437},{"step":121,"reward":-7.485210418701172,"core":-2.452962636947632,"cost":-0.05877425894141197,"kl":0.01731432299129665,"entropy":0.5231717973947525},{"step":123,"reward":-16.835844039916992,"core":-5.523408889770508,"cost":-0.1991475373506546,"kl":0.024128074175678194,"entropy":0.4803832024335861},{"step":125,"reward":-8.512553215026855,"core":-2.7673308849334717,"cost":-0.034931208938360214,"kl":0.010641333414241672,"entropy":0.4057620018720627},{"step":127,"reward":-10.70927906036377,"core":-3.5398871898651123,"cost":-0.04595935344696045,"kl":0.017602316569536924,"entropy":0.4932870641350746},{"step":129,"reward":-18.61650848388672,"core":-6.075850009918213,"cost":-0.32121679186820984,"kl":0.008336080703884363,"entropy":0.45114464312791824},{"step":131,"reward":-13.21602725982666,"core":-4.340992450714111,"cost":-0.13962553441524506,"kl":0.014274878427386284,"entropy":0.4946080297231674},{"step":133,"reward":-8.02479362487793,"core":-2.5996673107147217,"cost":-0.0777583047747612,"kl":0.009179509710520506,"entropy":0.42864735424518585},{"step":135,"reward":-9.469071388244629,"core":-3.1163971424102783,"cost":-0.05207354947924614,"kl":0.009748317766934633,"entropy":0.5393391251564026},{"step":137,"reward":-9.356158256530762,"core":-3.0789058208465576,"cost":-0.003946175333112478,"kl":0.00578083039727062,"entropy":0.39877641946077347},{"step":139,"reward":-17.048913955688477,"core":-5.624517917633057,"cost":-0.11595586687326431,"kl":0.012682976899668574,"entropy":0.4955347180366516},{"step":141,"reward":-12.490254402160645,"core":-4.102151870727539,"cost":-0.1400911957025528,"kl":0.009997370652854443,"entropy":0.4257754608988762},{"step":143,"reward":-19.331151962280273,"core":-6.317506790161133,"cost":-0.283406525850296,"kl":0.008605397189967334,"entropy":0.48196519911289215}],"nonzero_advantage":[{"step":1,"nonzero":0.0},{"step":3,"nonzero":0.0},{"step":5,"nonzero":0.0},{"step":7,"nonzero":0.0},{"step":9,"nonzero":0.0625},{"step":11,"nonzero":0.0},{"step":13,"nonzero":0.0},{"step":15,"nonzero":0.0},{"step":17,"nonzero":0.0},{"step":19,"nonzero":0.20833333333333334},{"step":21,"nonzero":0.0},{"step":23,"nonzero":0.0},{"step":25,"nonzero":0.0},{"step":27,"nonzero":0.10416666666666667},{"step":29,"nonzero":0.0},{"step":31,"nonzero":0.0},{"step":33,"nonzero":0.14583333333333334},{"step":35,"nonzero":0.0},{"step":37,"nonzero":0.125},{"step":39,"nonzero":0.10416666666666667},{"step":41,"nonzero":0.0},{"step":43,"nonzero":0.0},{"step":45,"nonzero":0.0},{"step":47,"nonzero":0.0},{"step":49,"nonzero":0.0},{"step":51,"nonzero":0.0},{"step":53,"nonzero":0.14583333333333334},{"step":55,"nonzero":0.0},{"step":57,"nonzero":0.041666666666666664},{"step":59,"nonzero":0.10416666666666667},{"step":61,"nonzero":0.0},{"step":63,"nonzero":0.0},{"step":65,"nonzero":0.0},{"step":67,"nonzero":0.0625},{"step":69,"nonzero":0.0},{"step":71,"nonzero":0.125},{"step":73,"nonzero":0.0},{"step":75,"nonzero":0.10416666666666667},{"step":77,"nonzero":0.10416666666666667},{"step":79,"nonzero":0.0},{"step":81,"nonzero":0.0},{"step":83,"nonzero":0.0},{"step":85,"nonzero":0.0},{"step":87,"nonzero":0.0},{"step":89,"nonzero":0.0},{"step":91,"nonzero":0.08333333333333333},{"step":93,"nonzero":0.0},{"step":95,"nonzero":0.25},{"step":97,"nonzero":0.0},{"step":99,"nonzero":0.0},{"step":101,"nonzero":0.0},{"step":103,"nonzero":0.0},{"step":105,"nonzero":0.0},{"step":107,"nonzero":0.0},{"step":109,"nonzero":0.0},{"step":111,"nonzero":0.0},{"step":113,"nonzero":0.041666666666666664},{"step":115,"nonzero":0.14583333333333334},{"step":117,"nonzero":0.0},{"step":119,"nonzero":0.0},{"step":121,"nonzero":0.125},{"step":123,"nonzero":0.0},{"step":125,"nonzero":0.0},{"step":127,"nonzero":0.041666666666666664},{"step":129,"nonzero":0.0},{"step":131,"nonzero":0.0},{"step":133,"nonzero":0.10416666666666667},{"step":135,"nonzero":0.0},{"step":137,"nonzero":0.0},{"step":139,"nonzero":0.0},{"step":141,"nonzero":0.0},{"step":143,"nonzero":0.0}]},{"id":"b3-nofloor","label":"No floor (third slice)","run_id":"1791130204","cpcv_floor":null,"train_steps":[{"step":1,"reward":-19.009735107421875,"core":-6.242278575897217,"cost":-0.2255139797925949,"kl":0.0,"entropy":0.49098174273967743},{"step":3,"reward":-9.944573402404785,"core":-3.2745754718780518,"cost":-0.030204951763153076,"kl":0.0005765757232438773,"entropy":0.4292754828929901},{"step":5,"reward":-10.303011894226074,"core":-3.4005959033966064,"cost":-0.04101278632879257,"kl":0.002329066541278735,"entropy":0.46478271484375},{"step":7,"reward":-12.095707893371582,"core":-3.942817449569702,"cost":-0.14172464609146118,"kl":0.00662712671328336,"entropy":0.5613181442022324},{"step":9,"reward":-12.777016639709473,"core":-4.160802364349365,"cost":-0.18609733879566193,"kl":0.00756690907292068,"entropy":0.45637837797403336},{"step":11,"reward":-12.898650169372559,"core":-4.245652198791504,"cost":-0.09507611393928528,"kl":0.011840706458315253,"entropy":0.4935869798064232},{"step":13,"reward":-10.25499439239502,"core":-3.369959831237793,"cost":-0.0703398808836937,"kl":0.0141780327539891,"entropy":0.4937457740306854},{"step":15,"reward":-12.59280014038086,"core":-4.12632942199707,"cost":-0.12636004388332367,"kl":0.020828555803745985,"entropy":0.49372030049562454},{"step":17,"reward":-6.708986759185791,"core":-2.194997549057007,"cost":-0.04232390224933624,"kl":0.017673942260444164,"entropy":0.5338631644845009},{"step":19,"reward":-4.429197311401367,"core":-1.4270490407943726,"cost":-0.012604796327650547,"kl":0.029546687845140696,"entropy":0.5241913348436356},{"step":21,"reward":-8.294654846191406,"core":-2.7268104553222656,"cost":-0.009193436242640018,"kl":0.04971966054290533,"entropy":0.4387289583683014},{"step":23,"reward":-5.073052406311035,"core":-1.6497598886489868,"cost":-0.019518887624144554,"kl":0.043806662783026695,"entropy":0.5657379925251007},{"step":25,"reward":-5.378979206085205,"core":-1.7569347620010376,"cost":0.0,"kl":0.040004558861255646,"entropy":0.4419279024004936},{"step":27,"reward":-4.291808605194092,"core":-1.3974164724349976,"cost":0.0,"kl":0.04806906636804342,"entropy":0.42248905450105667},{"step":29,"reward":-6.97482442855835,"core":-2.2710084915161133,"cost":-0.019804252311587334,"kl":0.10159038286656141,"entropy":0.38651297241449356},{"step":31,"reward":-6.869300842285156,"core":-2.228931427001953,"cost":-0.01777549646794796,"kl":0.07037136144936085,"entropy":0.3642829656600952},{"step":33,"reward":-14.580299377441406,"core":-4.793844699859619,"cost":-0.12494120746850967,"kl":0.0532494829967618,"entropy":0.40169938653707504},{"step":35,"reward":-15.508769989013672,"core":-5.056297302246094,"cost":-0.24529874324798584,"kl":0.05719861667603254,"entropy":0.34745901823043823},{"step":37,"reward":-4.233029842376709,"core":-1.3670138120651245,"cost":0.0,"kl":0.08141520526260138,"entropy":0.3668093830347061},{"step":39,"reward":-6.383880615234375,"core":-2.0827176570892334,"cost":-0.0008057521190494299,"kl":0.09096011333167553,"entropy":0.3914167881011963},{"step":41,"reward":-4.925972938537598,"core":-1.6067590713500977,"cost":0.0,"kl":0.07515102531760931,"entropy":0.3712315112352371},{"step":43,"reward":-3.9801807403564453,"core":-1.2660313844680786,"cost":0.0,"kl":0.11527538672089577,"entropy":0.3995494470000267},{"step":45,"reward":-8.382372856140137,"core":-2.7503716945648193,"cost":-0.02801288478076458,"kl":0.058372754603624344,"entropy":0.3768581598997116},{"step":47,"reward":-8.562376976013184,"core":-2.841458320617676,"cost":0.0,"kl":0.06673111487179995,"entropy":0.39184989035129547},{"step":49,"reward":-4.208565711975098,"core":-1.350895881652832,"cost":0.0,"kl":0.04859730787575245,"entropy":0.350110188126564},{"step":51,"reward":-5.266636371612549,"core":-1.7039939165115356,"cost":0.0,"kl":0.050188589841127396,"entropy":0.40979618579149246},{"step":53,"reward":-4.526350498199463,"core":-1.4531623125076294,"cost":0.0,"kl":0.054649162106215954,"entropy":0.4218897521495819},{"step":55,"reward":-5.556384563446045,"core":-1.7975431680679321,"cost":-0.003901328193023801,"kl":0.06876618880778551,"entropy":0.4812583550810814},{"step":57,"reward":-3.903620958328247,"core":-1.2382867336273193,"cost":-0.008579653687775135,"kl":0.06114924978464842,"entropy":0.48585059493780136},{"step":59,"reward":-7.6593451499938965,"core":-2.5115158557891846,"cost":-0.04693913087248802,"kl":0.06864143256098032,"entropy":0.5785015821456909},{"step":61,"reward":-4.702303886413574,"core":-1.5179600715637207,"cost":-0.004028990399092436,"kl":0.05777769070118666,"entropy":0.4541207179427147},{"step":63,"reward":-5.076469421386719,"core":-1.6029821634292603,"cost":-0.034929320216178894,"kl":0.05246087908744812,"entropy":0.435103140771389},{"step":65,"reward":-8.163283348083496,"core":-2.6782567501068115,"cost":-0.04825327917933464,"kl":0.08431461453437805,"entropy":0.5351532548666},{"step":67,"reward":-8.902278900146484,"core":-2.9285666942596436,"cost":-0.025758346542716026,"kl":0.09386199340224266,"entropy":0.4894018843770027},{"step":69,"reward":-4.557988166809082,"core":-1.4463599920272827,"cost":0.0,"kl":0.06383980438113213,"entropy":0.5027018934488297},{"step":71,"reward":-4.783263683319092,"core":-1.5364937782287598,"cost":-0.011884982697665691,"kl":0.08396179787814617,"entropy":0.5400129109621048},{"step":73,"reward":-5.306196689605713,"core":-1.715113639831543,"cost":0.0,"kl":0.0831704456359148,"entropy":0.4894970655441284},{"step":75,"reward":-4.112757205963135,"core":-1.3234652280807495,"cost":-2.4066415790002793e-05,"kl":0.0705598033964634,"entropy":0.47287166118621826},{"step":77,"reward":-12.967547416687012,"core":-4.236320495605469,"cost":-0.17555516958236694,"kl":0.09349325299263,"entropy":0.4615880474448204},{"step":79,"reward":-3.9381649494171143,"core":-1.21530282497406,"cost":-0.007472885772585869,"kl":0.08685537427663803,"entropy":0.5138002634048462},{"step":81,"reward":-5.0980939865112305,"core":-1.6651304960250854,"cost":0.0,"kl":0.09402503445744514,"entropy":0.5628683120012283},{"step":83,"reward":-5.463571071624756,"core":-1.8023643493652344,"cost":0.0,"kl":0.09226253814995289,"entropy":0.46717900037765503},{"step":85,"reward":-5.1699018478393555,"core":-1.6669002771377563,"cost":0.0,"kl":0.11557978577911854,"entropy":0.4844798445701599},{"step":87,"reward":-4.016908645629883,"core":-1.2918903827667236,"cost":0.0,"kl":0.12491845339536667,"entropy":0.5079388692975044},{"step":89,"reward":-5.0163350105285645,"core":-1.6174249649047852,"cost":-0.0348326675593853,"kl":0.12454798072576523,"entropy":0.46295342594385147},{"step":91,"reward":-5.869584560394287,"core":-1.8641785383224487,"cost":-0.08959835767745972,"kl":0.12228264473378658,"entropy":0.4875984713435173},{"step":93,"reward":-3.332026243209839,"core":-1.0387005805969238,"cost":0.0,"kl":0.10421585664153099,"entropy":0.46139009296894073},{"step":95,"reward":-5.786241054534912,"core":-1.8688968420028687,"cost":0.0,"kl":0.10331694222986698,"entropy":0.3841555118560791},{"step":97,"reward":-2.7757070064544678,"core":-0.8649862408638,"cost":0.0,"kl":0.09641245938837528,"entropy":0.5220653563737869},{"step":99,"reward":-5.673206806182861,"core":-1.8339190483093262,"cost":0.0,"kl":0.17108382284641266,"entropy":0.521229162812233},{"step":101,"reward":-6.544993877410889,"core":-2.108532190322876,"cost":-0.022212596610188484,"kl":0.17013663426041603,"entropy":0.5267152413725853},{"step":103,"reward":-2.8753011226654053,"core":-0.8445504307746887,"cost":0.0,"kl":0.12612201645970345,"entropy":0.4760717898607254},{"step":105,"reward":-4.583471775054932,"core":-1.4541397094726562,"cost":0.0,"kl":0.13358145393431187,"entropy":0.5096238255500793},{"step":107,"reward":-3.765733003616333,"core":-1.1774487495422363,"cost":0.0,"kl":0.13582479767501354,"entropy":0.48881160467863083},{"step":109,"reward":-3.8508079051971436,"core":-1.2059526443481445,"cost":0.0,"kl":0.2017505131661892,"entropy":0.5006129071116447},{"step":111,"reward":-4.728569984436035,"core":-1.46006441116333,"cost":-0.037928059697151184,"kl":0.14521546848118305,"entropy":0.507273904979229},{"step":113,"reward":-4.042181491851807,"core":-1.2721449136734009,"cost":-0.011662141419947147,"kl":0.14745193719863892,"entropy":0.4777228906750679},{"step":115,"reward":-5.286823749542236,"core":-1.700879693031311,"cost":-0.0027421351987868547,"kl":0.18395421653985977,"entropy":0.482380636036396},{"step":117,"reward":-4.790900230407715,"core":-1.5548324584960938,"cost":-0.005165900103747845,"kl":0.13759745471179485,"entropy":0.5187589824199677},{"step":119,"reward":-4.17254638671875,"core":-1.3340641260147095,"cost":0.0,"kl":0.09440874867141247,"entropy":0.44911667704582214},{"step":121,"reward":-3.07414174079895,"core":-0.9800145626068115,"cost":0.0,"kl":0.07997506950050592,"entropy":0.4928441569209099},{"step":123,"reward":-5.338275909423828,"core":-1.7235665321350098,"cost":-0.012724866159260273,"kl":0.1394710037857294,"entropy":0.5247234627604485},{"step":125,"reward":-4.6733198165893555,"core":-1.4602384567260742,"cost":-0.029322883114218712,"kl":0.12943747825920582,"entropy":0.4457559362053871},{"step":127,"reward":-4.432997226715088,"core":-1.4324384927749634,"cost":-0.022435924038290977,"kl":0.10576417483389378,"entropy":0.5148757174611092},{"step":129,"reward":-8.489529609680176,"core":-2.7770745754241943,"cost":-0.06385328620672226,"kl":0.09388891607522964,"entropy":0.5602174401283264},{"step":131,"reward":-4.499170780181885,"core":-1.4820451736450195,"cost":0.0,"kl":0.10349415987730026,"entropy":0.5149178206920624},{"step":133,"reward":-3.653697967529297,"core":-1.1326795816421509,"cost":0.0,"kl":0.09635326080024242,"entropy":0.46035970002412796},{"step":135,"reward":-3.9253692626953125,"core":-1.2799655199050903,"cost":0.0,"kl":0.11101790703833103,"entropy":0.5598675459623337},{"step":137,"reward":-5.704345703125,"core":-1.838301658630371,"cost":0.0,"kl":0.12609275989234447,"entropy":0.42492395639419556},{"step":139,"reward":-4.132491588592529,"core":-1.3203420639038086,"cost":0.0,"kl":0.13290071487426758,"entropy":0.5671236217021942},{"step":141,"reward":-3.7785251140594482,"core":-1.221341848373413,"cost":-0.008579653687775135,"kl":0.13662544079124928,"entropy":0.430658258497715},{"step":143,"reward":-4.7045578956604,"core":-1.5044751167297363,"cost":0.0,"kl":0.15495476126670837,"entropy":0.5024386346340179},{"step":145,"reward":-3.143756628036499,"core":-0.9904013276100159,"cost":0.0,"kl":0.12217587232589722,"entropy":0.5235531032085419},{"step":147,"reward":-4.426514148712158,"core":-1.435948371887207,"cost":0.0,"kl":0.12089831195771694,"entropy":0.5298959165811539},{"step":149,"reward":-3.541936159133911,"core":-1.1274179220199585,"cost":0.0,"kl":0.10277114808559418,"entropy":0.5316848084330559},{"step":151,"reward":-4.380416393280029,"core":-1.3995152711868286,"cost":0.0,"kl":0.1871374435722828,"entropy":0.5263749361038208},{"step":153,"reward":-5.2470903396606445,"core":-1.7209609746932983,"cost":-0.008194207213819027,"kl":0.11321512795984745,"entropy":0.48757505416870117},{"step":155,"reward":-2.8682172298431396,"core":-0.8591310977935791,"cost":0.0,"kl":0.1360226757824421,"entropy":0.40464604645967484},{"step":157,"reward":-4.6503682136535645,"core":-1.446554183959961,"cost":-0.04619959369301796,"kl":0.12094242125749588,"entropy":0.4687194228172302},{"step":159,"reward":-2.951939344406128,"core":-0.9168345332145691,"cost":0.0,"kl":0.15605835244059563,"entropy":0.42569591104984283},{"step":161,"reward":-3.2425458431243896,"core":-0.989279568195343,"cost":0.0,"kl":0.1352570466697216,"entropy":0.45315249264240265},{"step":163,"reward":-3.8879024982452393,"core":-1.215722680091858,"cost":-0.007132656406611204,"kl":0.1848572939634323,"entropy":0.5305487513542175},{"step":165,"reward":-4.514284610748291,"core":-1.4353584051132202,"cost":-0.003969153854995966,"kl":0.1563205122947693,"entropy":0.47219057381153107},{"step":167,"reward":-3.153759002685547,"core":-0.9930620193481445,"cost":0.0,"kl":0.1827676836401224,"entropy":0.5135424360632896},{"step":169,"reward":-4.598021984100342,"core":-1.4749280214309692,"cost":-0.023512855172157288,"kl":0.10963889583945274,"entropy":0.4296794682741165},{"step":171,"reward":-2.969012975692749,"core":-0.9278416037559509,"cost":0.0,"kl":0.1377528253942728,"entropy":0.6031644344329834},{"step":173,"reward":-4.162088394165039,"core":-1.3256795406341553,"cost":0.0,"kl":0.22763027623295784,"entropy":0.5559469759464264},{"step":175,"reward":-4.082291126251221,"core":-1.2998485565185547,"cost":0.0,"kl":0.14725057035684586,"entropy":0.46718190610408783},{"step":177,"reward":-5.296925067901611,"core":-1.6942591667175293,"cost":-0.035029202699661255,"kl":0.13804993405938148,"entropy":0.5065094828605652},{"step":179,"reward":-3.7194318771362305,"core":-1.1796321868896484,"cost":-0.004457823466509581,"kl":0.16047240793704987,"entropy":0.5189974755048752},{"step":181,"reward":-7.238447666168213,"core":-2.3891685009002686,"cost":-0.004127867985516787,"kl":0.22352969646453857,"entropy":0.49157436192035675},{"step":183,"reward":-3.71872615814209,"core":-1.1878936290740967,"cost":-0.013142205774784088,"kl":0.10708940215408802,"entropy":0.502140037715435},{"step":185,"reward":-3.6149513721466064,"core":-1.1291695833206177,"cost":0.0,"kl":0.18690454587340355,"entropy":0.5401958301663399},{"step":187,"reward":-3.857567548751831,"core":-1.2097386121749878,"cost":0.0,"kl":0.20556919649243355,"entropy":0.5256688520312309},{"step":189,"reward":-3.052790403366089,"core":-0.9438865780830383,"cost":0.0,"kl":0.15123333781957626,"entropy":0.5045753419399261},{"step":191,"reward":-3.450352430343628,"core":-1.0907260179519653,"cost":-0.004509864840656519,"kl":0.14348164200782776,"entropy":0.5421527475118637},{"step":193,"reward":-3.6056156158447266,"core":-1.1515870094299316,"cost":-0.006095258519053459,"kl":0.17940444312989712,"entropy":0.5794317722320557},{"step":195,"reward":-4.186895847320557,"core":-1.3220826387405396,"cost":0.0,"kl":0.1805449202656746,"entropy":0.44260791689157486},{"step":197,"reward":-2.398179531097412,"core":-0.7202653884887695,"cost":0.0,"kl":0.215712558478117,"entropy":0.5823928564786911},{"step":199,"reward":-3.450086832046509,"core":-1.066685676574707,"cost":0.0,"kl":0.1842175666242838,"entropy":0.5093915536999702},{"step":201,"reward":-5.308324337005615,"core":-1.7105735540390015,"cost":-0.013767249882221222,"kl":0.16555680707097054,"entropy":0.5604763031005859},{"step":203,"reward":-4.295825958251953,"core":-1.3774409294128418,"cost":0.0,"kl":0.22555385157465935,"entropy":0.5775810182094574},{"step":205,"reward":-6.632605075836182,"core":-2.1421220302581787,"cost":-0.05090281367301941,"kl":0.1417649481445551,"entropy":0.5185812562704086},{"step":207,"reward":-4.466914653778076,"core":-1.427587866783142,"cost":-0.006026083137840033,"kl":0.19613755494356155,"entropy":0.5869265645742416},{"step":209,"reward":-3.6594297885894775,"core":-1.1244878768920898,"cost":-0.01833508163690567,"kl":0.12774950824677944,"entropy":0.4898689538240433},{"step":211,"reward":-4.957226276397705,"core":-1.6130290031433105,"cost":0.0,"kl":0.13206574320793152,"entropy":0.5403095483779907},{"step":213,"reward":-3.4377803802490234,"core":-1.06385338306427,"cost":0.0,"kl":0.1579700168222189,"entropy":0.54795703291893},{"step":215,"reward":-4.068769931793213,"core":-1.2824441194534302,"cost":0.0,"kl":0.1481310874223709,"entropy":0.5667134523391724},{"step":217,"reward":-4.1265692710876465,"core":-1.3269298076629639,"cost":0.0,"kl":0.13296234048902988,"entropy":0.5075536593794823},{"step":219,"reward":-3.3357181549072266,"core":-1.0476875305175781,"cost":0.0,"kl":0.15813402645289898,"entropy":0.5247476473450661},{"step":221,"reward":-3.9631147384643555,"core":-1.2458009719848633,"cost":0.0,"kl":0.20738059096038342,"entropy":0.5416112542152405},{"step":223,"reward":-3.9063186645507812,"core":-1.2530626058578491,"cost":0.0,"kl":0.1486414335668087,"entropy":0.543528601527214},{"step":225,"reward":-3.500696897506714,"core":-1.0826066732406616,"cost":-0.0021108738146722317,"kl":0.16605102270841599,"entropy":0.46263908594846725},{"step":227,"reward":-5.152892589569092,"core":-1.6632000207901,"cost":-0.001033932901918888,"kl":0.13694093748927116,"entropy":0.48911163955926895},{"step":229,"reward":-3.6968324184417725,"core":-1.18722665309906,"cost":0.0,"kl":0.14369171112775803,"entropy":0.5143350213766098},{"step":231,"reward":-3.162057876586914,"core":-0.9884018898010254,"cost":0.0,"kl":0.13312761671841145,"entropy":0.5724841207265854},{"step":233,"reward":-3.3056633472442627,"core":-1.0539264678955078,"cost":0.0,"kl":0.12193361669778824,"entropy":0.46675361692905426},{"step":235,"reward":-2.8616275787353516,"core":-0.852627694606781,"cost":0.0,"kl":0.12803591042757034,"entropy":0.4561779946088791},{"step":237,"reward":-3.3555870056152344,"core":-1.0692256689071655,"cost":0.0,"kl":0.16248127073049545,"entropy":0.589288204908371},{"step":239,"reward":-11.259392738342285,"core":-3.6840317249298096,"cost":-0.07600713521242142,"kl":0.13577447086572647,"entropy":0.4994807615876198},{"step":241,"reward":-4.164712905883789,"core":-1.318678855895996,"cost":0.0,"kl":0.12236974947154522,"entropy":0.4649309813976288},{"step":243,"reward":-7.19495153427124,"core":-2.347003698348999,"cost":-0.025619884952902794,"kl":0.14661646634340286,"entropy":0.48849088698625565},{"step":245,"reward":-6.1934332847595215,"core":-2.0204577445983887,"cost":-0.018253957852721214,"kl":0.161116361618042,"entropy":0.5109238624572754},{"step":247,"reward":-2.559738874435425,"core":-0.7990642189979553,"cost":0.0,"kl":0.10441724956035614,"entropy":0.5314967110753059},{"step":249,"reward":-2.28098464012146,"core":-0.637080192565918,"cost":0.0,"kl":0.1365208923816681,"entropy":0.42451024055480957},{"step":251,"reward":-10.662322044372559,"core":-3.5012025833129883,"cost":-0.06532881408929825,"kl":0.13196329027414322,"entropy":0.5699133425951004},{"step":253,"reward":-5.103646755218506,"core":-1.668204426765442,"cost":-0.0013212895719334483,"kl":0.09630473703145981,"entropy":0.4751432165503502},{"step":255,"reward":-2.8684089183807373,"core":-0.9257887005805969,"cost":0.0,"kl":0.0791297908872366,"entropy":0.4046919718384743},{"step":257,"reward":-4.100041389465332,"core":-1.3003886938095093,"cost":-0.004908813629299402,"kl":0.1470446977764368,"entropy":0.4362534135580063},{"step":259,"reward":-5.354959964752197,"core":-1.652811050415039,"cost":-0.09279900789260864,"kl":0.09208343550562859,"entropy":0.4061049595475197},{"step":261,"reward":-3.651252031326294,"core":-1.1542695760726929,"cost":0.0,"kl":0.12909035384655,"entropy":0.44324128329753876},{"step":263,"reward":-4.513845443725586,"core":-1.4473527669906616,"cost":-0.011320839636027813,"kl":0.10535715520381927,"entropy":0.4393647462129593},{"step":265,"reward":-4.298539638519287,"core":-1.3652534484863281,"cost":-0.010046376846730709,"kl":0.11493301577866077,"entropy":0.40467893332242966},{"step":267,"reward":-4.647175312042236,"core":-1.495165467262268,"cost":-0.0016757334815338254,"kl":0.15375614166259766,"entropy":0.4512200728058815},{"step":269,"reward":-4.216683864593506,"core":-1.325419306755066,"cost":-0.04746810719370842,"kl":0.11641089245676994,"entropy":0.4547291696071625},{"step":271,"reward":-3.741790533065796,"core":-1.1953567266464233,"cost":0.0,"kl":0.13376039825379848,"entropy":0.47073039412498474},{"step":273,"reward":-4.162484645843506,"core":-1.3369067907333374,"cost":0.0,"kl":0.09896594472229481,"entropy":0.34486766904592514},{"step":275,"reward":-4.073892593383789,"core":-1.2882044315338135,"cost":0.0,"kl":0.16437574103474617,"entropy":0.5315835326910019},{"step":277,"reward":-5.01806640625,"core":-1.5678504705429077,"cost":-0.0067170350812375546,"kl":0.1709233857691288,"entropy":0.36289453506469727},{"step":279,"reward":-2.695253610610962,"core":-0.8160171508789062,"cost":0.0,"kl":0.12076223827898502,"entropy":0.51982781291008},{"step":281,"reward":-4.140861988067627,"core":-1.3160423040390015,"cost":0.0,"kl":0.1358135212212801,"entropy":0.41873180866241455},{"step":283,"reward":-4.660648345947266,"core":-1.5018348693847656,"cost":-0.0044793332926929,"kl":0.12167908623814583,"entropy":0.5356532782316208},{"step":285,"reward":-5.14744234085083,"core":-1.6623950004577637,"cost":0.0,"kl":0.15161282941699028,"entropy":0.545945554971695},{"step":287,"reward":-3.744133710861206,"core":-1.169409990310669,"cost":0.0,"kl":0.08951353654265404,"entropy":0.42195013910532},{"step":289,"reward":-4.656365871429443,"core":-1.5038785934448242,"cost":0.0,"kl":0.06692749541252851,"entropy":0.48009417951107025},{"step":291,"reward":-3.145615577697754,"core":-0.9406207203865051,"cost":0.0,"kl":0.08533878158777952,"entropy":0.445286400616169},{"step":293,"reward":-4.168957710266113,"core":-1.322996735572815,"cost":-0.004821529146283865,"kl":0.13507906533777714,"entropy":0.4804179146885872},{"step":295,"reward":-3.434616804122925,"core":-1.0466302633285522,"cost":0.0,"kl":0.10206322744488716,"entropy":0.45351215451955795},{"step":297,"reward":-2.3971288204193115,"core":-0.7012585997581482,"cost":0.0,"kl":0.14844956062734127,"entropy":0.3851080536842346},{"step":299,"reward":-3.032700777053833,"core":-0.9413235783576965,"cost":0.0,"kl":0.132567647844553,"entropy":0.4664331451058388},{"step":301,"reward":-4.364119052886963,"core":-1.368296504020691,"cost":-0.04536209627985954,"kl":0.10537982732057571,"entropy":0.467002309858799},{"step":303,"reward":-2.9415085315704346,"core":-0.8925309181213379,"cost":0.0,"kl":0.1266784481704235,"entropy":0.4741973206400871},{"step":305,"reward":-3.712998151779175,"core":-1.1471418142318726,"cost":-0.008032386191189289,"kl":0.12144341133534908,"entropy":0.42688919603824615},{"step":307,"reward":-8.675613403320312,"core":-2.800596237182617,"cost":-0.07521254569292068,"kl":0.1357093621045351,"entropy":0.4528474807739258},{"step":309,"reward":-4.369534492492676,"core":-1.4132747650146484,"cost":0.0,"kl":0.15004924312233925,"entropy":0.5318684428930283},{"step":311,"reward":-6.823635578155518,"core":-2.214801073074341,"cost":-0.018108809366822243,"kl":0.10909606702625751,"entropy":0.4638414829969406},{"step":313,"reward":-3.2854995727539062,"core":-1.0176507234573364,"cost":0.0,"kl":0.1317148245871067,"entropy":0.5144106447696686},{"step":315,"reward":-3.787583589553833,"core":-1.1972752809524536,"cost":0.0,"kl":0.1033769715577364,"entropy":0.5545191019773483},{"step":317,"reward":-4.069660186767578,"core":-1.2849581241607666,"cost":-0.021444046869874,"kl":0.12655351124703884,"entropy":0.531373530626297},{"step":319,"reward":-3.9869749546051025,"core":-1.2617197036743164,"cost":-0.00013380304153542966,"kl":0.1011599563062191,"entropy":0.5372210666537285},{"step":321,"reward":-2.1564176082611084,"core":-0.5979853272438049,"cost":0.0,"kl":0.09683484770357609,"entropy":0.45371202379465103},{"step":323,"reward":-5.560640811920166,"core":-1.7807308435440063,"cost":-0.02712494321167469,"kl":0.09239618759602308,"entropy":0.4504055231809616},{"step":325,"reward":-5.166891098022461,"core":-1.625623345375061,"cost":-0.013714884407818317,"kl":0.14900379069149494,"entropy":0.5347094684839249},{"step":327,"reward":-5.100793838500977,"core":-1.6005946397781372,"cost":-0.020377283915877342,"kl":0.12726862356066704,"entropy":0.446344830095768},{"step":329,"reward":-4.039286136627197,"core":-1.2734349966049194,"cost":0.0,"kl":0.1707929354161024,"entropy":0.5256704241037369},{"step":331,"reward":-4.42653226852417,"core":-1.4074796438217163,"cost":0.0,"kl":0.16388297453522682,"entropy":0.41043471544981003},{"step":333,"reward":-4.807687759399414,"core":-1.5381288528442383,"cost":0.0,"kl":0.09993047825992107,"entropy":0.43672560155391693},{"step":335,"reward":-3.65468692779541,"core":-1.1249842643737793,"cost":0.0,"kl":0.10190101526677608,"entropy":0.46224359422922134},{"step":337,"reward":-5.345150470733643,"core":-1.6949386596679688,"cost":-0.04072194918990135,"kl":0.12996101193130016,"entropy":0.4737463518977165},{"step":339,"reward":-5.84208345413208,"core":-1.9226042032241821,"cost":0.0,"kl":0.09750201366841793,"entropy":0.40867117047309875},{"step":341,"reward":-2.9408462047576904,"core":-0.8956401944160461,"cost":0.0,"kl":0.09390005376189947,"entropy":0.5383158698678017},{"step":343,"reward":-6.847693920135498,"core":-2.237589120864868,"cost":-0.008136157877743244,"kl":0.16009268164634705,"entropy":0.44782527536153793},{"step":345,"reward":-3.3960704803466797,"core":-1.0631237030029297,"cost":0.0,"kl":0.10984143801033497,"entropy":0.47670669108629227},{"step":347,"reward":-3.366225481033325,"core":-1.0708905458450317,"cost":0.0,"kl":0.10439466126263142,"entropy":0.457955501973629},{"step":349,"reward":-1.822731375694275,"core":-0.5294597148895264,"cost":0.0,"kl":0.0767862293869257,"entropy":0.3576837480068207},{"step":351,"reward":-4.860652446746826,"core":-1.5507348775863647,"cost":0.0,"kl":0.1474248431622982,"entropy":0.44792475551366806},{"step":353,"reward":-2.8288631439208984,"core":-0.8477373123168945,"cost":0.0,"kl":0.12288128957152367,"entropy":0.42456386238336563},{"step":355,"reward":-4.0311126708984375,"core":-1.2394331693649292,"cost":-0.03519139811396599,"kl":0.07971011474728584,"entropy":0.35086920112371445},{"step":357,"reward":-3.1394383907318115,"core":-0.9523867964744568,"cost":0.0,"kl":0.12225254438817501,"entropy":0.38711272180080414},{"step":359,"reward":-2.789576292037964,"core":-0.8329618573188782,"cost":0.0,"kl":0.18142477795481682,"entropy":0.5044523105025291},{"step":361,"reward":-2.910665512084961,"core":-0.873691737651825,"cost":0.0,"kl":0.16997617855668068,"entropy":0.4477898105978966},{"step":363,"reward":-3.7000181674957275,"core":-1.1752119064331055,"cost":0.0,"kl":0.15880265831947327,"entropy":0.48088187724351883},{"step":365,"reward":-4.462372779846191,"core":-1.4184430837631226,"cost":-0.009389171376824379,"kl":0.16699950024485588,"entropy":0.49246325343847275},{"step":367,"reward":-1.7630304098129272,"core":-0.4670655429363251,"cost":0.0,"kl":0.15058620274066925,"entropy":0.49085673689842224},{"step":369,"reward":-4.145650386810303,"core":-1.3323057889938354,"cost":-0.002827538177371025,"kl":0.15476292371749878,"entropy":0.506449781358242}],"nonzero_advantage":[{"step":1,"nonzero":1.0},{"step":3,"nonzero":1.0},{"step":5,"nonzero":1.0},{"step":7,"nonzero":1.0},{"step":9,"nonzero":1.0},{"step":11,"nonzero":1.0},{"step":13,"nonzero":1.0},{"step":15,"nonzero":1.0},{"step":17,"nonzero":1.0},{"step":19,"nonzero":1.0},{"step":21,"nonzero":1.0},{"step":23,"nonzero":1.0},{"step":25,"nonzero":1.0},{"step":27,"nonzero":1.0},{"step":29,"nonzero":1.0},{"step":31,"nonzero":1.0},{"step":33,"nonzero":1.0},{"step":35,"nonzero":1.0},{"step":37,"nonzero":1.0},{"step":39,"nonzero":1.0},{"step":41,"nonzero":1.0},{"step":43,"nonzero":1.0},{"step":45,"nonzero":1.0},{"step":47,"nonzero":1.0},{"step":49,"nonzero":1.0},{"step":51,"nonzero":1.0},{"step":53,"nonzero":1.0},{"step":55,"nonzero":1.0},{"step":57,"nonzero":1.0},{"step":59,"nonzero":1.0},{"step":61,"nonzero":1.0},{"step":63,"nonzero":1.0},{"step":65,"nonzero":1.0},{"step":67,"nonzero":1.0},{"step":69,"nonzero":1.0},{"step":71,"nonzero":1.0},{"step":73,"nonzero":1.0},{"step":75,"nonzero":1.0},{"step":77,"nonzero":1.0},{"step":79,"nonzero":1.0},{"step":81,"nonzero":1.0},{"step":83,"nonzero":1.0},{"step":85,"nonzero":1.0},{"step":87,"nonzero":1.0},{"step":89,"nonzero":1.0},{"step":91,"nonzero":1.0},{"step":93,"nonzero":1.0},{"step":95,"nonzero":1.0},{"step":97,"nonzero":1.0},{"step":99,"nonzero":1.0},{"step":101,"nonzero":1.0},{"step":103,"nonzero":1.0},{"step":105,"nonzero":1.0},{"step":107,"nonzero":1.0},{"step":109,"nonzero":1.0},{"step":111,"nonzero":1.0},{"step":113,"nonzero":1.0},{"step":115,"nonzero":1.0},{"step":117,"nonzero":1.0},{"step":119,"nonzero":1.0},{"step":121,"nonzero":1.0},{"step":123,"nonzero":1.0},{"step":125,"nonzero":1.0},{"step":127,"nonzero":1.0},{"step":129,"nonzero":1.0},{"step":131,"nonzero":1.0},{"step":133,"nonzero":1.0},{"step":135,"nonzero":1.0},{"step":137,"nonzero":1.0},{"step":139,"nonzero":1.0},{"step":141,"nonzero":1.0},{"step":143,"nonzero":1.0},{"step":145,"nonzero":1.0},{"step":147,"nonzero":1.0},{"step":149,"nonzero":1.0},{"step":151,"nonzero":1.0},{"step":153,"nonzero":1.0},{"step":155,"nonzero":1.0},{"step":157,"nonzero":1.0},{"step":159,"nonzero":1.0},{"step":161,"nonzero":1.0},{"step":163,"nonzero":1.0},{"step":165,"nonzero":1.0},{"step":167,"nonzero":1.0},{"step":169,"nonzero":1.0},{"step":171,"nonzero":1.0},{"step":173,"nonzero":1.0},{"step":175,"nonzero":1.0},{"step":177,"nonzero":1.0},{"step":179,"nonzero":1.0},{"step":181,"nonzero":1.0},{"step":183,"nonzero":0.7916666666666666},{"step":185,"nonzero":1.0},{"step":187,"nonzero":1.0},{"step":189,"nonzero":1.0},{"step":191,"nonzero":1.0},{"step":193,"nonzero":1.0},{"step":195,"nonzero":1.0},{"step":197,"nonzero":1.0},{"step":199,"nonzero":1.0},{"step":201,"nonzero":1.0},{"step":203,"nonzero":1.0},{"step":205,"nonzero":1.0},{"step":207,"nonzero":1.0},{"step":209,"nonzero":1.0},{"step":211,"nonzero":1.0},{"step":213,"nonzero":1.0},{"step":215,"nonzero":1.0},{"step":217,"nonzero":1.0},{"step":219,"nonzero":1.0},{"step":221,"nonzero":1.0},{"step":223,"nonzero":1.0},{"step":225,"nonzero":1.0},{"step":227,"nonzero":1.0},{"step":229,"nonzero":1.0},{"step":231,"nonzero":1.0},{"step":233,"nonzero":1.0},{"step":235,"nonzero":1.0},{"step":237,"nonzero":1.0},{"step":239,"nonzero":1.0},{"step":241,"nonzero":1.0},{"step":243,"nonzero":1.0},{"step":245,"nonzero":1.0},{"step":247,"nonzero":1.0},{"step":249,"nonzero":1.0},{"step":251,"nonzero":1.0},{"step":253,"nonzero":1.0},{"step":255,"nonzero":1.0},{"step":257,"nonzero":1.0},{"step":259,"nonzero":1.0},{"step":261,"nonzero":1.0},{"step":263,"nonzero":1.0},{"step":265,"nonzero":1.0},{"step":267,"nonzero":1.0},{"step":269,"nonzero":1.0},{"step":271,"nonzero":1.0},{"step":273,"nonzero":1.0},{"step":275,"nonzero":1.0},{"step":277,"nonzero":1.0},{"step":279,"nonzero":1.0},{"step":281,"nonzero":1.0},{"step":283,"nonzero":1.0},{"step":285,"nonzero":1.0},{"step":287,"nonzero":1.0},{"step":289,"nonzero":1.0},{"step":291,"nonzero":1.0},{"step":293,"nonzero":1.0},{"step":295,"nonzero":1.0},{"step":297,"nonzero":1.0},{"step":299,"nonzero":1.0},{"step":301,"nonzero":1.0},{"step":303,"nonzero":1.0},{"step":305,"nonzero":1.0},{"step":307,"nonzero":1.0},{"step":309,"nonzero":1.0},{"step":311,"nonzero":1.0},{"step":313,"nonzero":1.0},{"step":315,"nonzero":1.0},{"step":317,"nonzero":1.0},{"step":319,"nonzero":1.0},{"step":321,"nonzero":1.0},{"step":323,"nonzero":1.0},{"step":325,"nonzero":1.0},{"step":327,"nonzero":1.0},{"step":329,"nonzero":1.0},{"step":331,"nonzero":1.0},{"step":333,"nonzero":1.0},{"step":335,"nonzero":1.0},{"step":337,"nonzero":1.0},{"step":339,"nonzero":1.0},{"step":341,"nonzero":1.0},{"step":343,"nonzero":1.0},{"step":345,"nonzero":1.0},{"step":347,"nonzero":1.0},{"step":349,"nonzero":1.0},{"step":351,"nonzero":1.0},{"step":353,"nonzero":1.0},{"step":355,"nonzero":1.0},{"step":357,"nonzero":1.0},{"step":359,"nonzero":1.0},{"step":361,"nonzero":1.0},{"step":363,"nonzero":1.0},{"step":365,"nonzero":1.0},{"step":367,"nonzero":1.0},{"step":369,"nonzero":1.0}]},{"id":"b4-nofloor","label":"No floor (fourth slice)","run_id":"1791130610","cpcv_floor":null,"train_steps":[{"step":1,"reward":-14.877108573913574,"core":-4.8620476722717285,"cost":-0.18359072506427765,"kl":0.0,"entropy":0.49185793101787567},{"step":3,"reward":-10.82125473022461,"core":-3.5323288440704346,"cost":-0.09821552038192749,"kl":0.0009753682388691232,"entropy":0.6035551875829697},{"step":5,"reward":-6.803366184234619,"core":-2.1708590984344482,"cost":-0.03183769807219505,"kl":0.0016296389803756028,"entropy":0.5701221227645874},{"step":7,"reward":-10.291155815124512,"core":-3.332240343093872,"cost":-0.07771527022123337,"kl":0.00547688000369817,"entropy":0.5030030086636543},{"step":9,"reward":-10.064247131347656,"core":-3.274799346923828,"cost":-0.07808791100978851,"kl":0.011808298528194427,"entropy":0.5006609559059143},{"step":11,"reward":-9.886168479919434,"core":-3.221968412399292,"cost":-0.06334645301103592,"kl":0.020418659085407853,"entropy":0.4548523426055908},{"step":13,"reward":-9.1611328125,"core":-2.943023920059204,"cost":-0.1019383892416954,"kl":0.025245825294405222,"entropy":0.4362412840127945},{"step":15,"reward":-6.913618564605713,"core":-2.1896169185638428,"cost":-0.049770891666412354,"kl":0.04612576402723789,"entropy":0.4404192417860031},{"step":17,"reward":-2.5712339878082275,"core":-0.7891069054603577,"cost":0.0,"kl":0.03830957738682628,"entropy":0.4468081146478653},{"step":19,"reward":-10.938018798828125,"core":-3.594888687133789,"cost":-0.028850823640823364,"kl":0.052242455538362265,"entropy":0.45509567856788635},{"step":21,"reward":-4.919596195220947,"core":-1.5374473333358765,"cost":-0.019465623423457146,"kl":0.049405419267714024,"entropy":0.4444841966032982},{"step":23,"reward":-1.6586517095565796,"core":-0.38047730922698975,"cost":0.0,"kl":0.058569833636283875,"entropy":0.5187619999051094},{"step":25,"reward":-5.0380120277404785,"core":-1.6076469421386719,"cost":-0.007202065084129572,"kl":0.076247688382864,"entropy":0.5064432099461555},{"step":27,"reward":-6.5684967041015625,"core":-2.0572354793548584,"cost":-0.04647050425410271,"kl":0.05994971934705973,"entropy":0.6042719930410385},{"step":29,"reward":-2.255293607711792,"core":-0.58888179063797,"cost":0.0,"kl":0.05633103009313345,"entropy":0.4556780457496643},{"step":31,"reward":-6.418521881103516,"core":-2.0206527709960938,"cost":-0.0785868838429451,"kl":0.07701404765248299,"entropy":0.4929543361067772},{"step":33,"reward":-3.7153632640838623,"core":-1.12877357006073,"cost":0.0,"kl":0.07557082362473011,"entropy":0.5772439539432526},{"step":35,"reward":-6.943714141845703,"core":-2.2776081562042236,"cost":-0.03695978596806526,"kl":0.09772546589374542,"entropy":0.5783729553222656},{"step":37,"reward":-4.272907733917236,"core":-1.3197158575057983,"cost":-0.032010260969400406,"kl":0.07994672656059265,"entropy":0.5296378582715988},{"step":39,"reward":-1.7728983163833618,"core":-0.5073109865188599,"cost":0.0,"kl":0.10103357024490833,"entropy":0.5668975114822388},{"step":41,"reward":-2.5949437618255615,"core":-0.6824505925178528,"cost":0.0,"kl":0.11824370361864567,"entropy":0.4951799660921097},{"step":43,"reward":-4.619626045227051,"core":-1.4154154062271118,"cost":-0.031844694167375565,"kl":0.17872394993901253,"entropy":0.5508186370134354},{"step":45,"reward":-4.155986309051514,"core":-1.315592646598816,"cost":-0.00724723981693387,"kl":0.09313033893704414,"entropy":0.5164899975061417},{"step":47,"reward":-8.0279541015625,"core":-2.6171469688415527,"cost":-0.05757850408554077,"kl":0.06860806327313185,"entropy":0.4464063122868538},{"step":49,"reward":-2.4423635005950928,"core":-0.6207382678985596,"cost":-0.028233563527464867,"kl":0.08035931456834078,"entropy":0.42107339948415756},{"step":51,"reward":-10.7837495803833,"core":-3.514693021774292,"cost":-0.07294436544179916,"kl":0.10088846832513809,"entropy":0.5108584016561508},{"step":53,"reward":-4.304376125335693,"core":-1.2862008810043335,"cost":-0.016686981543898582,"kl":0.11381157673895359,"entropy":0.5168530121445656},{"step":55,"reward":-6.0310235023498535,"core":-1.9112211465835571,"cost":-0.010045775212347507,"kl":0.0824087094515562,"entropy":0.5244887843728065},{"step":57,"reward":-3.2932188510894775,"core":-0.9458810687065125,"cost":0.0,"kl":0.17722578905522823,"entropy":0.4841505214571953},{"step":59,"reward":-6.169613361358643,"core":-1.9651306867599487,"cost":-0.02372126467525959,"kl":0.11598117835819721,"entropy":0.5786947160959244},{"step":61,"reward":-5.038392543792725,"core":-1.5280259847640991,"cost":-0.030368534848093987,"kl":0.08979202061891556,"entropy":0.4742618501186371},{"step":63,"reward":-3.7065770626068115,"core":-1.1685339212417603,"cost":0.0,"kl":0.11952264606952667,"entropy":0.6107527315616608},{"step":65,"reward":-6.046070098876953,"core":-1.867182731628418,"cost":-0.024483567103743553,"kl":0.09311037696897984,"entropy":0.5142047330737114},{"step":67,"reward":-3.9996631145477295,"core":-1.2492791414260864,"cost":0.0,"kl":0.13635339587926865,"entropy":0.6283989101648331},{"step":69,"reward":-5.181606292724609,"core":-1.6646827459335327,"cost":-0.001923876698128879,"kl":0.1113781426101923,"entropy":0.5582340508699417},{"step":71,"reward":-2.9266748428344727,"core":-0.8399285674095154,"cost":0.0,"kl":0.09307015873491764,"entropy":0.6329682618379593},{"step":73,"reward":-2.3658087253570557,"core":-0.6572180390357971,"cost":0.0,"kl":0.105223273858428,"entropy":0.7069412469863892},{"step":75,"reward":-4.551165580749512,"core":-1.4548193216323853,"cost":0.0,"kl":0.09996155463159084,"entropy":0.5546401590108871},{"step":77,"reward":-4.6488356590271,"core":-1.4174805879592896,"cost":-0.028247922658920288,"kl":0.10728921182453632,"entropy":0.6149940639734268},{"step":79,"reward":-7.756839752197266,"core":-2.485084056854248,"cost":-0.05925128981471062,"kl":0.10470853187143803,"entropy":0.5973149538040161},{"step":81,"reward":-4.887008190155029,"core":-1.5663741827011108,"cost":-0.0009060203447006643,"kl":0.1418596375733614,"entropy":0.7078354507684708},{"step":83,"reward":-3.236773729324341,"core":-0.9582175612449646,"cost":-0.006805096287280321,"kl":0.09830354526638985,"entropy":0.5816888809204102},{"step":85,"reward":-3.6204330921173096,"core":-1.0706340074539185,"cost":-0.01986626349389553,"kl":0.14631505869328976,"entropy":0.6409860104322433},{"step":87,"reward":-4.906202793121338,"core":-1.551884651184082,"cost":-0.04579916596412659,"kl":0.1352770496159792,"entropy":0.6729817092418671},{"step":89,"reward":-2.1917665004730225,"core":-0.584917426109314,"cost":-0.0068951803259551525,"kl":0.11815940029919147,"entropy":0.4926261529326439},{"step":91,"reward":-3.381458282470703,"core":-1.0421032905578613,"cost":0.0,"kl":0.11124034598469734,"entropy":0.6001852750778198},{"step":93,"reward":-2.4948484897613525,"core":-0.6927034258842468,"cost":0.0,"kl":0.08165989071130753,"entropy":0.5642378032207489},{"step":95,"reward":-4.996362209320068,"core":-1.5450776815414429,"cost":0.0,"kl":0.13937117159366608,"entropy":0.6226602643728256},{"step":97,"reward":-4.464446544647217,"core":-1.3804291486740112,"cost":-0.004886580631136894,"kl":0.13506091572344303,"entropy":0.6401385962963104},{"step":99,"reward":-2.7744367122650146,"core":-0.8021958470344543,"cost":0.0,"kl":0.21736840903759003,"entropy":0.6070083677768707},{"step":101,"reward":-3.4675979614257812,"core":-1.0860323905944824,"cost":0.0,"kl":0.10547764040529728,"entropy":0.5447793453931808},{"step":103,"reward":-2.8547003269195557,"core":-0.8142905235290527,"cost":0.0,"kl":0.13423479907214642,"entropy":0.5379788354039192},{"step":105,"reward":-2.7659504413604736,"core":-0.8488125801086426,"cost":0.0,"kl":0.09761868230998516,"entropy":0.4896600842475891},{"step":107,"reward":-2.762695550918579,"core":-0.7586884498596191,"cost":0.0,"kl":0.12868965417146683,"entropy":0.4649636223912239},{"step":109,"reward":-2.174238443374634,"core":-0.5351836085319519,"cost":0.0,"kl":0.14795712009072304,"entropy":0.47060687094926834},{"step":111,"reward":-2.1892759799957275,"core":-0.5201313495635986,"cost":0.0,"kl":0.15082450956106186,"entropy":0.5158900320529938},{"step":113,"reward":-2.7438137531280518,"core":-0.8043741583824158,"cost":0.0,"kl":0.11971657536923885,"entropy":0.5420118868350983},{"step":115,"reward":-1.953577995300293,"core":-0.4863353669643402,"cost":0.0,"kl":0.12748480588197708,"entropy":0.4924478083848953},{"step":117,"reward":-3.2154781818389893,"core":-1.0254005193710327,"cost":0.0,"kl":0.1187813002616167,"entropy":0.5486257374286652},{"step":119,"reward":-2.9893484115600586,"core":-0.8736945986747742,"cost":0.0,"kl":0.15348385646939278,"entropy":0.5266085416078568},{"step":121,"reward":-2.8671934604644775,"core":-0.8323304653167725,"cost":0.0,"kl":0.1307186260819435,"entropy":0.5248879343271255},{"step":123,"reward":-4.7560553550720215,"core":-1.4988688230514526,"cost":-0.005359657574445009,"kl":0.13605093210935593,"entropy":0.6353387236595154},{"step":125,"reward":-2.914222002029419,"core":-0.8142557740211487,"cost":0.0,"kl":0.10642198473215103,"entropy":0.4699053317308426},{"step":127,"reward":-2.1509134769439697,"core":-0.5592977404594421,"cost":0.0,"kl":0.1065977867692709,"entropy":0.5155993103981018},{"step":129,"reward":-5.8582763671875,"core":-1.9024401903152466,"cost":-0.00934835895895958,"kl":0.12670256942510605,"entropy":0.6041177958250046},{"step":131,"reward":-2.651780605316162,"core":-0.7759056091308594,"cost":0.0,"kl":0.13116651587188244,"entropy":0.6835011541843414},{"step":133,"reward":-2.6708450317382812,"core":-0.8249511122703552,"cost":0.0,"kl":0.08375500328838825,"entropy":0.6273809969425201},{"step":135,"reward":-4.417050838470459,"core":-1.330810308456421,"cost":-0.03747320547699928,"kl":0.13268548250198364,"entropy":0.5966563075780869},{"step":137,"reward":-3.446133852005005,"core":-0.96042799949646,"cost":-0.015229739248752594,"kl":0.13274156861007214,"entropy":0.5776915699243546},{"step":139,"reward":-2.1982338428497314,"core":-0.578382670879364,"cost":0.0,"kl":0.14162560552358627,"entropy":0.6177410930395126},{"step":141,"reward":-5.123955249786377,"core":-1.652934193611145,"cost":0.0,"kl":0.12607391737401485,"entropy":0.6613598614931107},{"step":143,"reward":-3.1606006622314453,"core":-0.8875865936279297,"cost":0.0,"kl":0.11762930266559124,"entropy":0.5097120255231857},{"step":145,"reward":-4.335569858551025,"core":-1.359084129333496,"cost":-0.005916109774261713,"kl":0.1432630829513073,"entropy":0.6181366890668869},{"step":147,"reward":-2.650602340698242,"core":-0.785500705242157,"cost":-0.006233902648091316,"kl":0.13468231447041035,"entropy":0.532190591096878},{"step":149,"reward":-2.440248727798462,"core":-0.6468074917793274,"cost":0.0,"kl":0.09878179989755154,"entropy":0.5540356040000916},{"step":151,"reward":-3.535961389541626,"core":-1.0722488164901733,"cost":0.0,"kl":0.1917107254266739,"entropy":0.5869264155626297},{"step":153,"reward":-4.441526889801025,"core":-1.3503354787826538,"cost":-0.014472395181655884,"kl":0.13313444145023823,"entropy":0.5345429033041},{"step":155,"reward":-2.7858760356903076,"core":-0.7724000811576843,"cost":-0.010180897079408169,"kl":0.16777943447232246,"entropy":0.5654255598783493},{"step":157,"reward":-3.2660071849823,"core":-0.9263496398925781,"cost":-0.006619317922741175,"kl":0.1171595249325037,"entropy":0.5230711698532104},{"step":159,"reward":-3.2144596576690674,"core":-0.9534609913825989,"cost":0.0,"kl":0.13350601308047771,"entropy":0.5180847346782684},{"step":161,"reward":-2.393087148666382,"core":-0.6344398856163025,"cost":0.0,"kl":0.1469961814582348,"entropy":0.5156343430280685},{"step":163,"reward":-4.07710599899292,"core":-1.2630027532577515,"cost":-0.011718052439391613,"kl":0.13017960265278816,"entropy":0.5123481750488281},{"step":165,"reward":-3.7086293697357178,"core":-1.1509371995925903,"cost":0.0,"kl":0.12728491984307766,"entropy":0.650940790772438},{"step":167,"reward":-0.8587837815284729,"core":-0.10947493463754654,"cost":0.0,"kl":0.15412487462162971,"entropy":0.526451826095581},{"step":169,"reward":-1.805606484413147,"core":-0.37541118264198303,"cost":0.0,"kl":0.1129685528576374,"entropy":0.40736228972673416},{"step":171,"reward":-0.39877161383628845,"core":0.13064005970954895,"cost":0.0,"kl":0.13776619732379913,"entropy":0.5488280057907104},{"step":173,"reward":-1.4425420761108398,"core":-0.33456432819366455,"cost":0.0,"kl":0.13173695653676987,"entropy":0.5365863740444183},{"step":175,"reward":-2.5740854740142822,"core":-0.7416870594024658,"cost":-0.012167848646640778,"kl":0.12560448236763477,"entropy":0.48664120584726334},{"step":177,"reward":-4.14892578125,"core":-1.2956408262252808,"cost":-0.0019116803305223584,"kl":0.11843414418399334,"entropy":0.4960464835166931},{"step":179,"reward":-0.12741531431674957,"core":0.1395002156496048,"cost":0.0,"kl":0.14744671806693077,"entropy":0.4733887389302254},{"step":181,"reward":-3.1963584423065186,"core":-0.889426052570343,"cost":-0.03412202373147011,"kl":0.17951499670743942,"entropy":0.5059255063533783},{"step":183,"reward":-3.4058990478515625,"core":-0.9663189053535461,"cost":-0.018213192000985146,"kl":0.08896614797413349,"entropy":0.5134434103965759},{"step":185,"reward":-4.857083797454834,"core":-1.5096336603164673,"cost":0.0,"kl":0.09162204712629318,"entropy":0.47100479900836945},{"step":187,"reward":-2.6040127277374268,"core":-0.732196033000946,"cost":0.0,"kl":0.1507725566625595,"entropy":0.5556737929582596},{"step":189,"reward":-2.68432879447937,"core":-0.7446837425231934,"cost":-0.00476194778457284,"kl":0.14974533766508102,"entropy":0.6981363445520401},{"step":191,"reward":-2.2888638973236084,"core":-0.5790430903434753,"cost":-0.003651011036708951,"kl":0.10940543562173843,"entropy":0.5567461997270584},{"step":193,"reward":-1.7953640222549438,"core":-0.5167979598045349,"cost":0.0,"kl":0.1315611731261015,"entropy":0.5575318485498428},{"step":195,"reward":-0.8817993998527527,"core":-0.10092794895172119,"cost":0.0,"kl":0.10671606659889221,"entropy":0.535357728600502},{"step":197,"reward":-4.486854076385498,"core":-1.3803958892822266,"cost":-0.0057393708266317844,"kl":0.1370833981782198,"entropy":0.6863155663013458},{"step":199,"reward":-1.6756092309951782,"core":-0.3269941508769989,"cost":0.0,"kl":0.17892292886972427,"entropy":0.5600415915250778},{"step":201,"reward":-1.7408686876296997,"core":-0.35670605301856995,"cost":-0.010844815522432327,"kl":0.09799197688698769,"entropy":0.5503211617469788},{"step":203,"reward":-3.864802360534668,"core":-1.1519643068313599,"cost":0.0,"kl":0.2742317505180836,"entropy":0.6057491600513458},{"step":205,"reward":-2.1260669231414795,"core":-0.5829594731330872,"cost":0.0,"kl":0.14946372620761395,"entropy":0.6106411069631577},{"step":207,"reward":-3.120854616165161,"core":-0.8646491169929504,"cost":-0.02143915928900242,"kl":0.13881590403616428,"entropy":0.5952710658311844},{"step":209,"reward":-2.143280506134033,"core":-0.5615890622138977,"cost":0.0,"kl":0.1886201649904251,"entropy":0.6715198308229446},{"step":211,"reward":-1.454830527305603,"core":-0.34605729579925537,"cost":0.0,"kl":0.1893635280430317,"entropy":0.5947487205266953},{"step":213,"reward":-2.583017349243164,"core":-0.7471024394035339,"cost":-0.005228746682405472,"kl":0.279888566583395,"entropy":0.6176187992095947},{"step":215,"reward":-3.0928914546966553,"core":-0.9403583407402039,"cost":-0.006005417089909315,"kl":0.1858346238732338,"entropy":0.5914084166288376},{"step":217,"reward":-3.203582763671875,"core":-0.9426838755607605,"cost":0.0,"kl":0.15875005722045898,"entropy":0.5781827121973038},{"step":219,"reward":-1.5025577545166016,"core":-0.3791404664516449,"cost":0.0,"kl":0.17687901481986046,"entropy":0.6344633251428604},{"step":221,"reward":-2.5903384685516357,"core":-0.7701285481452942,"cost":0.0,"kl":0.18978802859783173,"entropy":0.5707826167345047},{"step":223,"reward":-0.9429314732551575,"core":-0.17441512644290924,"cost":0.0,"kl":0.15515169501304626,"entropy":0.6135737597942352},{"step":225,"reward":-4.195123195648193,"core":-1.2838932275772095,"cost":-0.01714782416820526,"kl":0.31912703812122345,"entropy":0.6661580950021744},{"step":227,"reward":-3.401639938354492,"core":-1.036111831665039,"cost":0.0,"kl":0.1600898914039135,"entropy":0.5850751996040344},{"step":229,"reward":-0.8413658142089844,"core":-0.15161730349063873,"cost":0.0,"kl":0.18443929590284824,"entropy":0.48576439917087555},{"step":231,"reward":-1.9518970251083374,"core":-0.47215816378593445,"cost":0.0,"kl":0.1464559007436037,"entropy":0.593670055270195},{"step":233,"reward":-2.402980089187622,"core":-0.5963430404663086,"cost":-0.01478751003742218,"kl":0.13387368991971016,"entropy":0.5376644730567932},{"step":235,"reward":-2.2117865085601807,"core":-0.551105260848999,"cost":-0.0056528388522565365,"kl":0.12970372289419174,"entropy":0.5404382944107056},{"step":237,"reward":-5.054760456085205,"core":-1.612379550933838,"cost":-0.018528057262301445,"kl":0.11115415766835213,"entropy":0.5369099602103233},{"step":239,"reward":-2.603221893310547,"core":-0.7519462704658508,"cost":0.0,"kl":0.15541650727391243,"entropy":0.5282716453075409},{"step":241,"reward":-1.753454566001892,"core":-0.4301571846008301,"cost":0.0,"kl":0.15239135921001434,"entropy":0.5059295892715454},{"step":243,"reward":-1.5223287343978882,"core":-0.35330188274383545,"cost":-0.032108061015605927,"kl":0.14417153969407082,"entropy":0.47056417167186737},{"step":245,"reward":-4.368110656738281,"core":-1.3438892364501953,"cost":0.0,"kl":0.18260474503040314,"entropy":0.5284114554524422},{"step":247,"reward":-0.950599730014801,"core":-0.12294360250234604,"cost":0.0,"kl":0.29195650294423103,"entropy":0.4547317326068878},{"step":249,"reward":-3.4517743587493896,"core":-1.0511928796768188,"cost":-0.00472775986418128,"kl":0.1345187984406948,"entropy":0.5135254561901093},{"step":251,"reward":-2.594114065170288,"core":-0.6339908242225647,"cost":-0.03174058720469475,"kl":0.13488846272230148,"entropy":0.5288874879479408},{"step":253,"reward":-3.606553316116333,"core":-1.1302274465560913,"cost":-0.0144113227725029,"kl":0.14648460783064365,"entropy":0.5263864547014236},{"step":255,"reward":-0.06042603775858879,"core":0.22759701311588287,"cost":0.0,"kl":0.08560372330248356,"entropy":0.46507464349269867},{"step":257,"reward":-3.8532416820526123,"core":-1.2158187627792358,"cost":0.0,"kl":0.10167793557047844,"entropy":0.5624128133058548},{"step":259,"reward":-1.6631765365600586,"core":-0.2837631106376648,"cost":0.0,"kl":0.17427745647728443,"entropy":0.4726325124502182},{"step":261,"reward":-6.718575954437256,"core":-2.104144811630249,"cost":-0.01826988346874714,"kl":0.13902858830988407,"entropy":0.5061274543404579},{"step":263,"reward":-4.095556259155273,"core":-1.2899090051651,"cost":-0.007875367067754269,"kl":0.09667389839887619,"entropy":0.553821012377739},{"step":265,"reward":-3.448450803756714,"core":-1.0505000352859497,"cost":-0.0013229167088866234,"kl":0.20672862604260445,"entropy":0.6378981471061707},{"step":267,"reward":-3.6515533924102783,"core":-1.1140270233154297,"cost":0.0,"kl":0.1304105892777443,"entropy":0.6607215851545334},{"step":269,"reward":-1.4983172416687012,"core":-0.39940205216407776,"cost":0.0,"kl":0.10786098800599575,"entropy":0.6388204991817474},{"step":271,"reward":-2.7723381519317627,"core":-0.8363161087036133,"cost":0.0,"kl":0.1261398158967495,"entropy":0.5678126215934753},{"step":273,"reward":-2.3819313049316406,"core":-0.6766389012336731,"cost":0.0,"kl":0.12922669015824795,"entropy":0.5301476269960403},{"step":275,"reward":-2.8417809009552,"core":-0.8182871341705322,"cost":-0.009827101603150368,"kl":0.17061226442456245,"entropy":0.6366472840309143},{"step":277,"reward":-2.5335171222686768,"core":-0.6627994775772095,"cost":-0.013111460022628307,"kl":0.13868458941578865,"entropy":0.5857202112674713},{"step":279,"reward":-2.8976714611053467,"core":-0.82244873046875,"cost":-0.00011286722292425111,"kl":0.14448615908622742,"entropy":0.6042296439409256},{"step":281,"reward":-3.029895067214966,"core":-0.8784357905387878,"cost":0.0,"kl":0.2019440419971943,"entropy":0.6759751588106155},{"step":283,"reward":-3.792919874191284,"core":-1.1606208086013794,"cost":-0.00743716349825263,"kl":0.12366118468344212,"entropy":0.569078654050827},{"step":285,"reward":-1.7421897649765015,"core":-0.4398970603942871,"cost":0.0,"kl":0.11859019286930561,"entropy":0.5796575397253036},{"step":287,"reward":-1.1866921186447144,"core":-0.1766478270292282,"cost":0.0,"kl":0.10383599810302258,"entropy":0.5950638502836227},{"step":289,"reward":-3.540212869644165,"core":-1.0960723161697388,"cost":0.0,"kl":0.14350692927837372,"entropy":0.7013992071151733},{"step":291,"reward":-1.266965627670288,"core":-0.1992262601852417,"cost":0.0,"kl":0.15743185207247734,"entropy":0.5779760479927063},{"step":293,"reward":-0.32911941409111023,"core":-0.016708284616470337,"cost":-0.0012656619073823094,"kl":0.14036759920418262,"entropy":0.5368275791406631},{"step":295,"reward":-1.8027116060256958,"core":-0.3964775800704956,"cost":0.0,"kl":0.16986917331814766,"entropy":0.5384553670883179},{"step":297,"reward":-1.5771805047988892,"core":-0.3811485767364502,"cost":0.0,"kl":0.19843454286456108,"entropy":0.5333601832389832},{"step":299,"reward":-2.1587464809417725,"core":-0.5823138952255249,"cost":0.0,"kl":0.14871926233172417,"entropy":0.6530572921037674},{"step":301,"reward":-1.092629313468933,"core":-0.15105409920215607,"cost":-0.009485931135714054,"kl":0.2674386128783226,"entropy":0.4957578107714653},{"step":303,"reward":-1.968354344367981,"core":-0.5120689868927002,"cost":0.0,"kl":0.15618891641497612,"entropy":0.6157076954841614},{"step":305,"reward":-0.5364282727241516,"core":0.015193323604762554,"cost":0.0,"kl":0.13760682195425034,"entropy":0.4774365499615669},{"step":307,"reward":-4.074521541595459,"core":-1.197184443473816,"cost":-0.01185174286365509,"kl":0.1709871143102646,"entropy":0.6403283774852753},{"step":309,"reward":-2.0271036624908447,"core":-0.5059340596199036,"cost":-0.009709776379168034,"kl":0.1897677220404148,"entropy":0.5796509459614754},{"step":311,"reward":-1.3823074102401733,"core":-0.2922530472278595,"cost":0.0,"kl":0.12204139307141304,"entropy":0.5617746710777283},{"step":313,"reward":-0.6299760937690735,"core":-0.05961709842085838,"cost":0.0,"kl":0.18404106423258781,"entropy":0.5906381607055664},{"step":315,"reward":-1.137526035308838,"core":-0.17650671303272247,"cost":0.0,"kl":0.15883945301175117,"entropy":0.6246110796928406},{"step":317,"reward":-1.932177186012268,"core":-0.5280144810676575,"cost":-0.013323917053639889,"kl":0.2263641431927681,"entropy":0.5500769913196564},{"step":319,"reward":-1.577903151512146,"core":-0.39569368958473206,"cost":0.0,"kl":0.14705348387360573,"entropy":0.5450948625802994},{"step":321,"reward":-1.8867534399032593,"core":-0.5605906844139099,"cost":-0.0107415234670043,"kl":0.12818593531847,"entropy":0.5085293725132942},{"step":323,"reward":-1.2636456489562988,"core":-0.3099188804626465,"cost":0.0,"kl":0.17203552275896072,"entropy":0.6168940961360931},{"step":325,"reward":-1.3798011541366577,"core":-0.30042514204978943,"cost":0.0,"kl":0.13850115984678268,"entropy":0.6499890089035034},{"step":327,"reward":-2.876682996749878,"core":-0.7356523871421814,"cost":-0.03422561660408974,"kl":0.15131516382098198,"entropy":0.5599064975976944},{"step":329,"reward":-1.9890093803405762,"core":-0.482467383146286,"cost":0.0,"kl":0.16882053390145302,"entropy":0.5367245525121689},{"step":331,"reward":-1.7486720085144043,"core":-0.42350777983665466,"cost":0.0,"kl":0.1817212775349617,"entropy":0.5328855216503143},{"step":333,"reward":-0.8623310923576355,"core":-0.07733616977930069,"cost":0.0,"kl":0.18383801355957985,"entropy":0.4374134838581085},{"step":335,"reward":-2.9513657093048096,"core":-0.8029070496559143,"cost":-0.030665868893265724,"kl":0.1837985645979643,"entropy":0.5858835726976395},{"step":337,"reward":-3.358600616455078,"core":-1.0345077514648438,"cost":0.0,"kl":0.14438890479505062,"entropy":0.5395415499806404},{"step":339,"reward":-1.5734440088272095,"core":-0.3477318584918976,"cost":-0.018550926819443703,"kl":0.2510984428226948,"entropy":0.5087367966771126},{"step":341,"reward":-0.2613469064235687,"core":0.17431457340717316,"cost":0.0,"kl":0.1587259192019701,"entropy":0.3859883025288582},{"step":343,"reward":-2.325368881225586,"core":-0.6119363903999329,"cost":0.0,"kl":0.15428396873176098,"entropy":0.5196014568209648},{"step":345,"reward":-2.1350739002227783,"core":-0.5518272519111633,"cost":-0.019481820985674858,"kl":0.19743382930755615,"entropy":0.4626322239637375},{"step":347,"reward":-0.6334103941917419,"core":0.008936186321079731,"cost":0.0,"kl":0.19053439795970917,"entropy":0.4753960892558098},{"step":349,"reward":-2.4124021530151367,"core":-0.5985960960388184,"cost":-0.0010657713282853365,"kl":0.16158637776970863,"entropy":0.5571563020348549},{"step":351,"reward":-2.6365840435028076,"core":-0.7690346837043762,"cost":0.0,"kl":0.2270343415439129,"entropy":0.613338977098465},{"step":353,"reward":-0.9388659596443176,"core":-0.11041878908872604,"cost":0.0,"kl":0.12727616541087627,"entropy":0.5814563557505608},{"step":355,"reward":-2.550177574157715,"core":-0.7204170227050781,"cost":-0.000877111277077347,"kl":0.18500623479485512,"entropy":0.6046324968338013},{"step":357,"reward":-1.908299446105957,"core":-0.467113733291626,"cost":0.0,"kl":0.1700311191380024,"entropy":0.5728339552879333},{"step":359,"reward":-3.564192533493042,"core":-1.09018874168396,"cost":0.0,"kl":0.1334227379411459,"entropy":0.5201079621911049},{"step":361,"reward":-1.916269302368164,"core":-0.47370943427085876,"cost":0.0,"kl":0.21693525090813637,"entropy":0.5963898748159409},{"step":363,"reward":-1.2595821619033813,"core":-0.23696394264698029,"cost":0.0,"kl":0.14905955828726292,"entropy":0.4572082757949829},{"step":365,"reward":-1.3459488153457642,"core":-0.3181348145008087,"cost":0.0,"kl":0.1856280378997326,"entropy":0.6105455607175827},{"step":367,"reward":-0.9050182700157166,"core":-0.12812410295009613,"cost":0.0,"kl":0.2271151915192604,"entropy":0.647847130894661},{"step":369,"reward":-1.9038654565811157,"core":-0.41899561882019043,"cost":-0.00932561606168747,"kl":0.5550522282719612,"entropy":0.5398105084896088},{"step":371,"reward":-0.8305027484893799,"core":-0.010636073537170887,"cost":0.0,"kl":0.14132697694003582,"entropy":0.46885745227336884},{"step":373,"reward":-5.4541754722595215,"core":-1.6598776578903198,"cost":-0.046300213783979416,"kl":0.19591183960437775,"entropy":0.5808718353509903},{"step":375,"reward":-0.5215534567832947,"core":-0.03426055237650871,"cost":0.0,"kl":0.17062872648239136,"entropy":0.5959379523992538},{"step":377,"reward":-3.279130697250366,"core":-1.0250602960586548,"cost":0.0,"kl":0.15955984964966774,"entropy":0.5642404407262802},{"step":379,"reward":-1.1742576360702515,"core":-0.17536143958568573,"cost":-0.025945022702217102,"kl":0.17697858065366745,"entropy":0.49491437524557114},{"step":381,"reward":-1.629791259765625,"core":-0.377138614654541,"cost":0.0,"kl":0.12580437771975994,"entropy":0.5343084782361984},{"step":383,"reward":-1.2991564273834229,"core":-0.2653186321258545,"cost":0.0,"kl":0.132606141269207,"entropy":0.5543943643569946},{"step":385,"reward":-3.1553192138671875,"core":-0.9135618209838867,"cost":-0.01854865998029709,"kl":0.15428298898041248,"entropy":0.5596077144145966},{"step":387,"reward":-2.134038209915161,"core":-0.6344358325004578,"cost":0.0,"kl":0.17721694707870483,"entropy":0.4671151116490364},{"step":389,"reward":-2.0295655727386475,"core":-0.49609872698783875,"cost":0.0,"kl":0.1598295122385025,"entropy":0.6199018359184265},{"step":391,"reward":-0.45178666710853577,"core":-0.0713341236114502,"cost":0.0,"kl":0.1205338928848505,"entropy":0.5266993269324303},{"step":393,"reward":-1.354027271270752,"core":-0.2552647292613983,"cost":0.0,"kl":0.12483890354633331,"entropy":0.5871987640857697}],"nonzero_advantage":[{"step":1,"nonzero":1.0},{"step":3,"nonzero":1.0},{"step":5,"nonzero":1.0},{"step":7,"nonzero":1.0},{"step":9,"nonzero":1.0},{"step":11,"nonzero":1.0},{"step":13,"nonzero":1.0},{"step":15,"nonzero":1.0},{"step":17,"nonzero":1.0},{"step":19,"nonzero":1.0},{"step":21,"nonzero":1.0},{"step":23,"nonzero":1.0},{"step":25,"nonzero":1.0},{"step":27,"nonzero":1.0},{"step":29,"nonzero":1.0},{"step":31,"nonzero":1.0},{"step":33,"nonzero":1.0},{"step":35,"nonzero":1.0},{"step":37,"nonzero":1.0},{"step":39,"nonzero":1.0},{"step":41,"nonzero":1.0},{"step":43,"nonzero":1.0},{"step":45,"nonzero":1.0},{"step":47,"nonzero":1.0},{"step":49,"nonzero":1.0},{"step":51,"nonzero":1.0},{"step":53,"nonzero":1.0},{"step":55,"nonzero":1.0},{"step":57,"nonzero":1.0},{"step":59,"nonzero":1.0},{"step":61,"nonzero":1.0},{"step":63,"nonzero":1.0},{"step":65,"nonzero":1.0},{"step":67,"nonzero":1.0},{"step":69,"nonzero":1.0},{"step":71,"nonzero":1.0},{"step":73,"nonzero":1.0},{"step":75,"nonzero":1.0},{"step":77,"nonzero":1.0},{"step":79,"nonzero":1.0},{"step":81,"nonzero":1.0},{"step":83,"nonzero":1.0},{"step":85,"nonzero":1.0},{"step":87,"nonzero":1.0},{"step":89,"nonzero":1.0},{"step":91,"nonzero":1.0},{"step":93,"nonzero":1.0},{"step":95,"nonzero":1.0},{"step":97,"nonzero":1.0},{"step":99,"nonzero":1.0},{"step":101,"nonzero":1.0},{"step":103,"nonzero":1.0},{"step":105,"nonzero":1.0},{"step":107,"nonzero":1.0},{"step":109,"nonzero":1.0},{"step":111,"nonzero":1.0},{"step":113,"nonzero":1.0},{"step":115,"nonzero":1.0},{"step":117,"nonzero":1.0},{"step":119,"nonzero":1.0},{"step":121,"nonzero":1.0},{"step":123,"nonzero":1.0},{"step":125,"nonzero":1.0},{"step":127,"nonzero":1.0},{"step":129,"nonzero":1.0},{"step":131,"nonzero":1.0},{"step":133,"nonzero":1.0},{"step":135,"nonzero":1.0},{"step":137,"nonzero":1.0},{"step":139,"nonzero":1.0},{"step":141,"nonzero":1.0},{"step":143,"nonzero":1.0},{"step":145,"nonzero":1.0},{"step":147,"nonzero":1.0},{"step":149,"nonzero":1.0},{"step":151,"nonzero":1.0},{"step":153,"nonzero":1.0},{"step":155,"nonzero":1.0},{"step":157,"nonzero":1.0},{"step":159,"nonzero":1.0},{"step":161,"nonzero":1.0},{"step":163,"nonzero":1.0},{"step":165,"nonzero":1.0},{"step":167,"nonzero":1.0},{"step":169,"nonzero":1.0},{"step":171,"nonzero":1.0},{"step":173,"nonzero":1.0},{"step":175,"nonzero":1.0},{"step":177,"nonzero":1.0},{"step":179,"nonzero":1.0},{"step":181,"nonzero":1.0},{"step":183,"nonzero":1.0},{"step":185,"nonzero":1.0},{"step":187,"nonzero":1.0},{"step":189,"nonzero":1.0},{"step":191,"nonzero":1.0},{"step":193,"nonzero":1.0},{"step":195,"nonzero":1.0},{"step":197,"nonzero":1.0},{"step":199,"nonzero":1.0},{"step":201,"nonzero":1.0},{"step":203,"nonzero":1.0},{"step":205,"nonzero":1.0},{"step":207,"nonzero":1.0},{"step":209,"nonzero":1.0},{"step":211,"nonzero":1.0},{"step":213,"nonzero":1.0},{"step":215,"nonzero":1.0},{"step":217,"nonzero":1.0},{"step":219,"nonzero":1.0},{"step":221,"nonzero":1.0},{"step":223,"nonzero":1.0},{"step":225,"nonzero":1.0},{"step":227,"nonzero":1.0},{"step":229,"nonzero":1.0},{"step":231,"nonzero":1.0},{"step":233,"nonzero":1.0},{"step":235,"nonzero":1.0},{"step":237,"nonzero":1.0},{"step":239,"nonzero":1.0},{"step":241,"nonzero":1.0},{"step":243,"nonzero":1.0},{"step":245,"nonzero":1.0},{"step":247,"nonzero":1.0},{"step":249,"nonzero":0.8125},{"step":251,"nonzero":1.0},{"step":253,"nonzero":1.0},{"step":255,"nonzero":1.0},{"step":257,"nonzero":1.0},{"step":259,"nonzero":1.0},{"step":261,"nonzero":1.0},{"step":263,"nonzero":1.0},{"step":265,"nonzero":1.0},{"step":267,"nonzero":1.0},{"step":269,"nonzero":1.0},{"step":271,"nonzero":1.0},{"step":273,"nonzero":1.0},{"step":275,"nonzero":1.0},{"step":277,"nonzero":1.0},{"step":279,"nonzero":1.0},{"step":281,"nonzero":1.0},{"step":283,"nonzero":1.0},{"step":285,"nonzero":1.0},{"step":287,"nonzero":1.0},{"step":289,"nonzero":1.0},{"step":291,"nonzero":1.0},{"step":293,"nonzero":1.0},{"step":295,"nonzero":1.0},{"step":297,"nonzero":1.0},{"step":299,"nonzero":1.0},{"step":301,"nonzero":1.0},{"step":303,"nonzero":1.0},{"step":305,"nonzero":1.0},{"step":307,"nonzero":1.0},{"step":309,"nonzero":1.0},{"step":311,"nonzero":1.0},{"step":313,"nonzero":1.0},{"step":315,"nonzero":1.0},{"step":317,"nonzero":1.0},{"step":319,"nonzero":1.0},{"step":321,"nonzero":1.0},{"step":323,"nonzero":1.0},{"step":325,"nonzero":1.0},{"step":327,"nonzero":1.0},{"step":329,"nonzero":1.0},{"step":331,"nonzero":1.0},{"step":333,"nonzero":1.0},{"step":335,"nonzero":1.0},{"step":337,"nonzero":1.0},{"step":339,"nonzero":1.0},{"step":341,"nonzero":1.0},{"step":343,"nonzero":1.0},{"step":345,"nonzero":1.0},{"step":347,"nonzero":1.0},{"step":349,"nonzero":1.0},{"step":351,"nonzero":1.0},{"step":353,"nonzero":1.0},{"step":355,"nonzero":1.0},{"step":357,"nonzero":1.0},{"step":359,"nonzero":1.0},{"step":361,"nonzero":1.0},{"step":363,"nonzero":1.0},{"step":365,"nonzero":1.0},{"step":367,"nonzero":1.0},{"step":369,"nonzero":1.0},{"step":371,"nonzero":1.0},{"step":373,"nonzero":1.0},{"step":375,"nonzero":1.0},{"step":377,"nonzero":1.0},{"step":379,"nonzero":1.0},{"step":381,"nonzero":1.0},{"step":383,"nonzero":1.0},{"step":385,"nonzero":1.0},{"step":387,"nonzero":1.0},{"step":389,"nonzero":1.0},{"step":391,"nonzero":1.0},{"step":393,"nonzero":1.0}]}],"thermostat":{"run_id":"1790958861","target":0.5421527110888842,"steps":[{"step":1,"entropy":0.5596009348860905,"zeta":0.0},{"step":2,"entropy":0.5647671385236839,"zeta":0.0},{"step":3,"entropy":0.5287483105580189,"zeta":0.0},{"step":4,"entropy":0.5282855896545852,"zeta":0.0},{"step":5,"entropy":0.5293615818220425,"zeta":0.0},{"step":6,"entropy":0.5337085963311489,"zeta":0.001},{"step":7,"entropy":0.5063055244532357,"zeta":0.002},{"step":8,"entropy":0.5025701914137093,"zeta":0.004},{"step":9,"entropy":0.5417020609804432,"zeta":0.008},{"step":10,"entropy":0.5323711153534475,"zeta":0.016},{"step":11,"entropy":0.4977045619892185,"zeta":0.032},{"step":12,"entropy":0.4924646037902713,"zeta":0.05},{"step":13,"entropy":0.4868943668702726,"zeta":0.05},{"step":14,"entropy":0.48609496461723367,"zeta":0.05},{"step":15,"entropy":0.43389656090107115,"zeta":0.05},{"step":16,"entropy":0.4303362297525792,"zeta":0.05},{"step":17,"entropy":0.4851716304458853,"zeta":0.05},{"step":18,"entropy":0.4838086718611692,"zeta":0.05},{"step":19,"entropy":0.5248711923882062,"zeta":0.05},{"step":20,"entropy":0.5240605138944154,"zeta":0.05},{"step":21,"entropy":0.4783713234286393,"zeta":0.05},{"step":22,"entropy":0.48423285164980256,"zeta":0.05},{"step":23,"entropy":0.4555616411319414,"zeta":0.05},{"step":24,"entropy":0.45856148393004176,"zeta":0.05},{"step":25,"entropy":0.44311043257129473,"zeta":0.05},{"step":26,"entropy":0.4472166362835786,"zeta":0.05},{"step":27,"entropy":0.5294438820725387,"zeta":0.05},{"step":28,"entropy":0.534384701779258,"zeta":0.05},{"step":29,"entropy":0.534667103337856,"zeta":0.05},{"step":30,"entropy":0.5408216509491535,"zeta":0.05},{"step":31,"entropy":0.587676761009399,"zeta":0.05},{"step":32,"entropy":0.5968567696056404,"zeta":0.025},{"step":33,"entropy":0.6079702558085867,"zeta":0.0125},{"step":34,"entropy":0.6191514623918224,"zeta":0.00625},{"step":35,"entropy":0.577638383820491,"zeta":0.003125},{"step":36,"entropy":0.5823396347841575,"zeta":0.0015625},{"step":37,"entropy":0.5969739858676162,"zeta":0.00078125},{"step":38,"entropy":0.5890371227982173,"zeta":0.000390625},{"step":39,"entropy":0.4576118808758726,"zeta":0.0001953125},{"step":40,"entropy":0.4566567476923094,"zeta":0.000390625},{"step":41,"entropy":0.6012673926393343,"zeta":0.00078125},{"step":42,"entropy":0.6032819521234183,"zeta":0.000390625},{"step":43,"entropy":0.597078950747263,"zeta":0.0001953125},{"step":44,"entropy":0.5973927451936408,"zeta":0.0},{"step":45,"entropy":0.6288982907349618,"zeta":0.0},{"step":46,"entropy":0.6275985540155373,"zeta":0.0},{"step":47,"entropy":0.6338345773437755,"zeta":0.0},{"step":48,"entropy":0.6328103794359992,"zeta":0.0},{"step":49,"entropy":0.6302177398526014,"zeta":0.0},{"step":50,"entropy":0.6244065470504431,"zeta":0.0},{"step":51,"entropy":0.6137544983648235,"zeta":0.0},{"step":52,"entropy":0.6084969029661036,"zeta":0.0},{"step":53,"entropy":0.6365848700321062,"zeta":0.0},{"step":54,"entropy":0.6322306292222155,"zeta":0.0},{"step":55,"entropy":0.49596493454051876,"zeta":0.0},{"step":56,"entropy":0.489419639634394,"zeta":0.0001},{"step":57,"entropy":0.4965615713560286,"zeta":0.0002},{"step":58,"entropy":0.4927841644715647,"zeta":0.0004},{"step":59,"entropy":0.5581067531069531,"zeta":0.0008},{"step":60,"entropy":0.5518297003801689,"zeta":0.0004},{"step":61,"entropy":0.5513001216675105,"zeta":0.0002},{"step":62,"entropy":0.5461586232304928,"zeta":0.0001},{"step":63,"entropy":0.5392920987069854,"zeta":0.0},{"step":64,"entropy":0.5312676619570432,"zeta":0.0001},{"step":65,"entropy":0.42073488137458614,"zeta":0.0002},{"step":66,"entropy":0.4165585932931283,"zeta":0.0004},{"step":67,"entropy":0.5031518404249122,"zeta":0.0008},{"step":68,"entropy":0.4989733706287158,"zeta":0.0016},{"step":69,"entropy":0.41773153518360495,"zeta":0.0032},{"step":70,"entropy":0.416616216478817,"zeta":0.0064},{"step":71,"entropy":0.49642833549051235,"zeta":0.0128},{"step":72,"entropy":0.4944229716858377,"zeta":0.0256},{"step":73,"entropy":0.4068087198784072,"zeta":0.05},{"step":74,"entropy":0.40875791189955263,"zeta":0.05},{"step":75,"entropy":0.45966946794106966,"zeta":0.05},{"step":76,"entropy":0.46266829957273015,"zeta":0.05},{"step":77,"entropy":0.5132476886713921,"zeta":0.05},{"step":78,"entropy":0.5163850643755578,"zeta":0.05},{"step":79,"entropy":0.45538677771738223,"zeta":0.05},{"step":80,"entropy":0.45818523089370494,"zeta":0.05},{"step":81,"entropy":0.5304951023004676,"zeta":0.05},{"step":82,"entropy":0.5346653172828355,"zeta":0.05},{"step":83,"entropy":0.4187994670311717,"zeta":0.05},{"step":84,"entropy":0.424534624340196,"zeta":0.05},{"step":85,"entropy":0.5163697605657398,"zeta":0.05},{"step":86,"entropy":0.5211031553516682,"zeta":0.05},{"step":87,"entropy":0.5287372274095675,"zeta":0.05},{"step":88,"entropy":0.533055376040083,"zeta":0.05},{"step":89,"entropy":0.5048367495041164,"zeta":0.05},{"step":90,"entropy":0.5099935058229775,"zeta":0.05},{"step":91,"entropy":0.5352703858306892,"zeta":0.05},{"step":92,"entropy":0.5422518284059947,"zeta":0.05},{"step":93,"entropy":0.5127119520355514,"zeta":0.025},{"step":94,"entropy":0.5166608119242336,"zeta":0.05},{"step":95,"entropy":0.4960563923007906,"zeta":0.05},{"step":96,"entropy":0.5003970703440623,"zeta":0.05},{"step":97,"entropy":0.6195829755196246,"zeta":0.05},{"step":98,"entropy":0.6233846218086286,"zeta":0.025},{"step":99,"entropy":0.6286155556533272,"zeta":0.0125},{"step":100,"entropy":0.630076315909022,"zeta":0.00625},{"step":101,"entropy":0.6135362243380442,"zeta":0.003125},{"step":102,"entropy":0.6154471100597934,"zeta":0.0015625},{"step":103,"entropy":0.575405488999531,"zeta":0.00078125},{"step":104,"entropy":0.5787062553659134,"zeta":0.000390625},{"step":105,"entropy":0.5914617817007163,"zeta":0.0001953125},{"step":106,"entropy":0.5912530616769156,"zeta":0.0},{"step":107,"entropy":0.6092598865428147,"zeta":0.0},{"step":108,"entropy":0.6092340978670986,"zeta":0.0},{"step":109,"entropy":0.65956905719013,"zeta":0.0},{"step":110,"entropy":0.659549453140454,"zeta":0.0},{"step":111,"entropy":0.5696695661194842,"zeta":0.0},{"step":112,"entropy":0.5718133543757999,"zeta":0.0},{"step":113,"entropy":0.6406396702156518,"zeta":0.0},{"step":114,"entropy":0.6422017404684651,"zeta":0.0},{"step":115,"entropy":0.6061026972239069,"zeta":0.0},{"step":116,"entropy":0.6028111432294415,"zeta":0.0},{"step":117,"entropy":0.6306518278135703,"zeta":0.0},{"step":118,"entropy":0.6310284503654212,"zeta":0.0},{"step":119,"entropy":0.5826489184590803,"zeta":0.0},{"step":120,"entropy":0.5800749668502495,"zeta":0.0},{"step":121,"entropy":0.5445289594955947,"zeta":0.0},{"step":122,"entropy":0.5432982601327532,"zeta":0.0},{"step":123,"entropy":0.5190875223704896,"zeta":0.0},{"step":124,"entropy":0.5155559593879044,"zeta":0.0001},{"step":125,"entropy":0.5843500484979766,"zeta":0.0002},{"step":126,"entropy":0.5830398259855131,"zeta":0.0001},{"step":127,"entropy":0.6149100482759714,"zeta":0.0},{"step":128,"entropy":0.6112703152497436,"zeta":0.0},{"step":129,"entropy":0.5643701916534104,"zeta":0.0},{"step":130,"entropy":0.5596071983052291,"zeta":0.0},{"step":131,"entropy":0.5574626651108453,"zeta":0.0},{"step":132,"entropy":0.5518583736331086,"zeta":0.0},{"step":133,"entropy":0.6036022754500161,"zeta":0.0},{"step":134,"entropy":0.6009555034835864,"zeta":0.0},{"step":135,"entropy":0.5177561031544571,"zeta":0.0},{"step":136,"entropy":0.5150687018612171,"zeta":0.0001},{"step":137,"entropy":0.538058343906272,"zeta":0.0002},{"step":138,"entropy":0.5361067726936591,"zeta":0.0004},{"step":139,"entropy":0.5414213573410602,"zeta":0.0008},{"step":140,"entropy":0.5410920819742282,"zeta":0.0016},{"step":141,"entropy":0.5273406619227192,"zeta":0.0032},{"step":142,"entropy":0.526265388051459,"zeta":0.0064},{"step":143,"entropy":0.5804846699108637,"zeta":0.0128},{"step":144,"entropy":0.5808064443087478,"zeta":0.0064},{"step":145,"entropy":0.38981901559572907,"zeta":0.0032},{"step":146,"entropy":0.39088031972150566,"zeta":0.0064},{"step":147,"entropy":0.4758779638629788,"zeta":0.0128},{"step":148,"entropy":0.4759783893098021,"zeta":0.0256},{"step":149,"entropy":0.5190903986240479,"zeta":0.05},{"step":150,"entropy":0.5211068562203437,"zeta":0.05},{"step":151,"entropy":0.595171883040677,"zeta":0.05},{"step":152,"entropy":0.6007407030335702,"zeta":0.025},{"step":153,"entropy":0.47679270774081645,"zeta":0.0125},{"step":154,"entropy":0.4789749477733002,"zeta":0.025},{"step":155,"entropy":0.5835990556107248,"zeta":0.05},{"step":156,"entropy":0.5884477683058721,"zeta":0.025},{"step":157,"entropy":0.5225852502038382,"zeta":0.0125},{"step":158,"entropy":0.5263911987692983,"zeta":0.025},{"step":159,"entropy":0.5832470496169061,"zeta":0.05},{"step":160,"entropy":0.587040735323206,"zeta":0.025},{"step":161,"entropy":0.5417542781026546,"zeta":0.0125},{"step":162,"entropy":0.5419742079720891,"zeta":0.025},{"step":163,"entropy":0.560719504256334,"zeta":0.05},{"step":164,"entropy":0.5619376098619168,"zeta":0.025},{"step":165,"entropy":0.5482704146217574,"zeta":0.0125},{"step":166,"entropy":0.5483165740374482,"zeta":0.00625},{"step":167,"entropy":0.5800934909361388,"zeta":0.003125},{"step":168,"entropy":0.5804820580604491,"zeta":0.0015625},{"step":169,"entropy":0.6774522479023518,"zeta":0.00078125},{"step":170,"entropy":0.6790151876970266,"zeta":0.000390625},{"step":171,"entropy":0.5946238740161118,"zeta":0.0001953125},{"step":172,"entropy":0.595150635022497,"zeta":0.0},{"step":173,"entropy":0.6378321857759821,"zeta":0.0},{"step":174,"entropy":0.6385414346414199,"zeta":0.0},{"step":175,"entropy":0.6245590743980557,"zeta":0.0},{"step":176,"entropy":0.6221983614540982,"zeta":0.0},{"step":177,"entropy":0.5397608918826687,"zeta":0.0},{"step":178,"entropy":0.5396954405992502,"zeta":0.0001},{"step":179,"entropy":0.5360861856378998,"zeta":0.0002},{"step":180,"entropy":0.5355736085294227,"zeta":0.0004},{"step":181,"entropy":0.5533302945021011,"zeta":0.0008},{"step":182,"entropy":0.5488217103644226,"zeta":0.0004},{"step":183,"entropy":0.5917458981286781,"zeta":0.0002},{"step":184,"entropy":0.591005235197813,"zeta":0.0001},{"step":185,"entropy":0.5660405687773234,"zeta":0.0},{"step":186,"entropy":0.5662943417800294,"zeta":0.0},{"step":187,"entropy":0.6393533883023509,"zeta":0.0},{"step":188,"entropy":0.6374688758595431,"zeta":0.0},{"step":189,"entropy":0.5533475274608025,"zeta":0.0},{"step":190,"entropy":0.5537338408492875,"zeta":0.0},{"step":191,"entropy":0.5236094473812712,"zeta":0.0},{"step":192,"entropy":0.5230130297957368,"zeta":0.0001},{"step":193,"entropy":0.5837023816418639,"zeta":0.0002},{"step":194,"entropy":0.5833539274714827,"zeta":0.0001},{"step":195,"entropy":0.5507325457461806,"zeta":0.0},{"step":196,"entropy":0.5508159527777999,"zeta":0.0},{"step":197,"entropy":0.5697166035957049,"zeta":0.0},{"step":198,"entropy":0.5671993939634739,"zeta":0.0},{"step":199,"entropy":0.6086009591306107,"zeta":0.0},{"step":200,"entropy":0.6084625169017995,"zeta":0.0},{"step":201,"entropy":0.5828898048933447,"zeta":0.0},{"step":202,"entropy":0.5815814592988752,"zeta":0.0},{"step":203,"entropy":0.5111799974068741,"zeta":0.0},{"step":204,"entropy":0.5073243838000621,"zeta":0.0001},{"step":205,"entropy":0.5806917588081933,"zeta":0.0002},{"step":206,"entropy":0.5765287774295706,"zeta":0.0001},{"step":207,"entropy":0.532082105274712,"zeta":0.0},{"step":208,"entropy":0.5302733222420606,"zeta":0.0001},{"step":209,"entropy":0.5856953978590921,"zeta":0.0002},{"step":210,"entropy":0.5829532860393623,"zeta":0.0001},{"step":211,"entropy":0.5329263957752686,"zeta":0.0},{"step":212,"entropy":0.532915770673254,"zeta":0.0001},{"step":213,"entropy":0.5941008871919861,"zeta":0.0002},{"step":214,"entropy":0.5934772163046913,"zeta":0.0001},{"step":215,"entropy":0.5695778639580303,"zeta":0.0},{"step":216,"entropy":0.5697125046270798,"zeta":0.0},{"step":217,"entropy":0.42681469468626804,"zeta":0.0},{"step":218,"entropy":0.42958830106843665,"zeta":0.0001},{"step":219,"entropy":0.5107951011567576,"zeta":0.0002},{"step":220,"entropy":0.512099981494571,"zeta":0.0004},{"step":221,"entropy":0.5104207561313471,"zeta":0.0008},{"step":222,"entropy":0.5120636741612271,"zeta":0.0016},{"step":223,"entropy":0.5679437143507092,"zeta":0.0032},{"step":224,"entropy":0.568770764657771,"zeta":0.0016},{"step":225,"entropy":0.5319314195085528,"zeta":0.0008},{"step":226,"entropy":0.5335183308468993,"zeta":0.0016},{"step":227,"entropy":0.5888085703877568,"zeta":0.0032},{"step":228,"entropy":0.5901227433620985,"zeta":0.0016},{"step":229,"entropy":0.5032932615556249,"zeta":0.0008},{"step":230,"entropy":0.5031308970434321,"zeta":0.0016},{"step":231,"entropy":0.5811769253764674,"zeta":0.0032},{"step":232,"entropy":0.580117272679055,"zeta":0.0016},{"step":233,"entropy":0.604884479032355,"zeta":0.0008},{"step":234,"entropy":0.6045638625740996,"zeta":0.0004},{"step":235,"entropy":0.6101177906051722,"zeta":0.0002},{"step":236,"entropy":0.6105599342744416,"zeta":0.0001},{"step":237,"entropy":0.6478089430451889,"zeta":0.0},{"step":238,"entropy":0.6478377826292717,"zeta":0.0},{"step":239,"entropy":0.554586162508893,"zeta":0.0},{"step":240,"entropy":0.5546875377167519,"zeta":0.0},{"step":241,"entropy":0.5658861413084438,"zeta":0.0},{"step":242,"entropy":0.5640145087958541,"zeta":0.0},{"step":243,"entropy":0.5707424695513456,"zeta":0.0},{"step":244,"entropy":0.5698611162405605,"zeta":0.0},{"step":245,"entropy":0.5285205653398986,"zeta":0.0},{"step":246,"entropy":0.5251345853137356,"zeta":0.0001},{"step":247,"entropy":0.5472701888752626,"zeta":0.0002},{"step":248,"entropy":0.5482662323565848,"zeta":0.0001},{"step":249,"entropy":0.5738719784434015,"zeta":0.0},{"step":250,"entropy":0.5721555036578571,"zeta":0.0},{"step":251,"entropy":0.5633229273624689,"zeta":0.0},{"step":252,"entropy":0.562243628298888,"zeta":0.0},{"step":253,"entropy":0.5187970805107818,"zeta":0.0},{"step":254,"entropy":0.5166323208682665,"zeta":0.0001},{"step":255,"entropy":0.5003135906876974,"zeta":0.0002},{"step":256,"entropy":0.4975587325064226,"zeta":0.0004},{"step":257,"entropy":0.5260068996755858,"zeta":0.0008},{"step":258,"entropy":0.525262920445501,"zeta":0.0016},{"step":259,"entropy":0.5082697859617966,"zeta":0.0032},{"step":260,"entropy":0.5074387759561381,"zeta":0.0064},{"step":261,"entropy":0.5346640702427461,"zeta":0.0128},{"step":262,"entropy":0.533969829348178,"zeta":0.0256},{"step":263,"entropy":0.5203534148983382,"zeta":0.05},{"step":264,"entropy":0.5175440023630217,"zeta":0.05},{"step":265,"entropy":0.5372755229293577,"zeta":0.05},{"step":266,"entropy":0.5385797172920865,"zeta":0.05},{"step":267,"entropy":0.5167908175743819,"zeta":0.05},{"step":268,"entropy":0.5158124938858404,"zeta":0.05},{"step":269,"entropy":0.5173926179853591,"zeta":0.05},{"step":270,"entropy":0.5168151713109296,"zeta":0.05},{"step":271,"entropy":0.5278731027507169,"zeta":0.05},{"step":272,"entropy":0.529116160726776,"zeta":0.05},{"step":273,"entropy":0.5189186584921327,"zeta":0.05},{"step":274,"entropy":0.5210433874880604,"zeta":0.05},{"step":275,"entropy":0.590021296239022,"zeta":0.05},{"step":276,"entropy":0.5930855231117375,"zeta":0.025},{"step":277,"entropy":0.5450969197150283,"zeta":0.0125},{"step":278,"entropy":0.5504762093167382,"zeta":0.00625},{"step":279,"entropy":0.5501785096588173,"zeta":0.003125},{"step":280,"entropy":0.55301640826088,"zeta":0.0015625},{"step":281,"entropy":0.5548882626086927,"zeta":0.00078125},{"step":282,"entropy":0.5599894380734872,"zeta":0.000390625},{"step":283,"entropy":0.5127122447495457,"zeta":0.0001953125},{"step":284,"entropy":0.5145816334612879,"zeta":0.000390625},{"step":285,"entropy":0.5376792428904187,"zeta":0.00078125},{"step":286,"entropy":0.540918895622747,"zeta":0.0015625},{"step":287,"entropy":0.5601651149756933,"zeta":0.003125},{"step":288,"entropy":0.5606049607799097,"zeta":0.0015625},{"step":289,"entropy":0.6147531438663286,"zeta":0.00078125},{"step":290,"entropy":0.6172803093695525,"zeta":0.000390625},{"step":291,"entropy":0.5421041272211375,"zeta":0.0001953125},{"step":292,"entropy":0.5435782482583417,"zeta":0.000390625},{"step":293,"entropy":0.5832928601737069,"zeta":0.0001953125},{"step":294,"entropy":0.5838925159697796,"zeta":0.0},{"step":295,"entropy":0.574419544345949,"zeta":0.0},{"step":296,"entropy":0.5731231485322148,"zeta":0.0},{"step":297,"entropy":0.52204323297334,"zeta":0.0},{"step":298,"entropy":0.5219276218138246,"zeta":0.0001},{"step":299,"entropy":0.5905407436177931,"zeta":0.0002},{"step":300,"entropy":0.590899623953569,"zeta":0.0001},{"step":301,"entropy":0.5278649459774046,"zeta":0.0},{"step":302,"entropy":0.527120804393643,"zeta":0.0001},{"step":303,"entropy":0.5826407696058702,"zeta":0.0002},{"step":304,"entropy":0.5841731375087862,"zeta":0.0001},{"step":305,"entropy":0.6004538634828444,"zeta":0.0},{"step":306,"entropy":0.6020963497854912,"zeta":0.0},{"step":307,"entropy":0.5704061243638607,"zeta":0.0},{"step":308,"entropy":0.5702156750095745,"zeta":0.0},{"step":309,"entropy":0.4952056324485817,"zeta":0.0},{"step":310,"entropy":0.4965127107412158,"zeta":0.0001},{"step":311,"entropy":0.5835725489648106,"zeta":0.0002},{"step":312,"entropy":0.5826632279445665,"zeta":0.0001},{"step":313,"entropy":0.6184337521261632,"zeta":0.0},{"step":314,"entropy":0.6182145593716606,"zeta":0.0},{"step":315,"entropy":0.5370528873714887,"zeta":0.0},{"step":316,"entropy":0.5368260626288704,"zeta":0.0001},{"step":317,"entropy":0.6053874123855943,"zeta":0.0002},{"step":318,"entropy":0.6057222180330251,"zeta":0.0001},{"step":319,"entropy":0.4783817345587947,"zeta":0.0},{"step":320,"entropy":0.4812206535967828,"zeta":0.0001},{"step":321,"entropy":0.5346814914664809,"zeta":0.0002},{"step":322,"entropy":0.5361491709988874,"zeta":0.0004},{"step":323,"entropy":0.49902402875748475,"zeta":0.0008},{"step":324,"entropy":0.5017719722830631,"zeta":0.0016},{"step":325,"entropy":0.6329131476306875,"zeta":0.0032},{"step":326,"entropy":0.6362702217366631,"zeta":0.0016},{"step":327,"entropy":0.5976414588015153,"zeta":0.0008},{"step":328,"entropy":0.5994820904935675,"zeta":0.0004},{"step":329,"entropy":0.6625982094510589,"zeta":0.0002},{"step":330,"entropy":0.6625858867319075,"zeta":0.0001},{"step":331,"entropy":0.6162026535125132,"zeta":0.0},{"step":332,"entropy":0.6160429572146483,"zeta":0.0},{"step":333,"entropy":0.5918681783175752,"zeta":0.0},{"step":334,"entropy":0.5913388846114677,"zeta":0.0},{"step":335,"entropy":0.6032918709251462,"zeta":0.0},{"step":336,"entropy":0.6028640811606837,"zeta":0.0},{"step":337,"entropy":0.612207658544305,"zeta":0.0},{"step":338,"entropy":0.6108346331579962,"zeta":0.0},{"step":339,"entropy":0.6351423048053356,"zeta":0.0},{"step":340,"entropy":0.6356113438488222,"zeta":0.0},{"step":341,"entropy":0.554926668030538,"zeta":0.0},{"step":342,"entropy":0.5515603106576444,"zeta":0.0},{"step":343,"entropy":0.5280500610719582,"zeta":0.0},{"step":344,"entropy":0.5277289390007025,"zeta":0.0001},{"step":345,"entropy":0.6094231284435008,"zeta":0.0002},{"step":346,"entropy":0.6067161049251328,"zeta":0.0001},{"step":347,"entropy":0.43835831070217396,"zeta":0.0},{"step":348,"entropy":0.4352463542466047,"zeta":0.0001},{"step":349,"entropy":0.5571346976097115,"zeta":0.0002},{"step":350,"entropy":0.5534471568217625,"zeta":0.0001},{"step":351,"entropy":0.5849218095556271,"zeta":0.0},{"step":352,"entropy":0.5825812424684893,"zeta":0.0},{"step":353,"entropy":0.5409749803899375,"zeta":0.0},{"step":354,"entropy":0.5402442237532545,"zeta":0.0001},{"step":355,"entropy":0.5750423961466216,"zeta":0.0002},{"step":356,"entropy":0.5735739094892243,"zeta":0.0001},{"step":357,"entropy":0.5313418071099468,"zeta":0.0},{"step":358,"entropy":0.529750450880118,"zeta":0.0001},{"step":359,"entropy":0.4922026842668397,"zeta":0.0002},{"step":360,"entropy":0.4908012603951066,"zeta":0.0004},{"step":361,"entropy":0.5160712500659903,"zeta":0.0008},{"step":362,"entropy":0.5137756253254621,"zeta":0.0016},{"step":363,"entropy":0.6107088175676666,"zeta":0.0032},{"step":364,"entropy":0.610532726186494,"zeta":0.0016},{"step":365,"entropy":0.5085993953882039,"zeta":0.0008},{"step":366,"entropy":0.5074804525517592,"zeta":0.0016},{"step":367,"entropy":0.4515834497639587,"zeta":0.0032},{"step":368,"entropy":0.4539405571568949,"zeta":0.0064},{"step":369,"entropy":0.5420760908244798,"zeta":0.0128},{"step":370,"entropy":0.5431001012343815,"zeta":0.0256},{"step":371,"entropy":0.45205258211290683,"zeta":0.0128},{"step":372,"entropy":0.4532313137156477,"zeta":0.0256},{"step":373,"entropy":0.5410057892302975,"zeta":0.05},{"step":374,"entropy":0.5412669552233887,"zeta":0.05},{"step":375,"entropy":0.5546890714687365,"zeta":0.05},{"step":376,"entropy":0.5552814068033061,"zeta":0.025},{"step":377,"entropy":0.48611785909791294,"zeta":0.0125},{"step":378,"entropy":0.4888230294930644,"zeta":0.025},{"step":379,"entropy":0.511353416596027,"zeta":0.05},{"step":380,"entropy":0.5115689870358867,"zeta":0.05},{"step":381,"entropy":0.5511663228706786,"zeta":0.05},{"step":382,"entropy":0.5530093333205829,"zeta":0.025},{"step":383,"entropy":0.48981544020390305,"zeta":0.0125},{"step":384,"entropy":0.48983743781108074,"zeta":0.025},{"step":385,"entropy":0.5990049152482059,"zeta":0.05},{"step":386,"entropy":0.5984721344316299,"zeta":0.025},{"step":387,"entropy":0.5412844247809326,"zeta":0.0125},{"step":388,"entropy":0.5420941851625883,"zeta":0.025},{"step":389,"entropy":0.5581061062134367,"zeta":0.05},{"step":390,"entropy":0.5562656435280745,"zeta":0.025},{"step":391,"entropy":0.48062905485388385,"zeta":0.0125},{"step":392,"entropy":0.4827248322569294,"zeta":0.025},{"step":393,"entropy":0.5441594085687904,"zeta":0.05},{"step":394,"entropy":0.5428310805702696,"zeta":0.025},{"step":395,"entropy":0.5733151630855344,"zeta":0.0125},{"step":396,"entropy":0.5729150979386202,"zeta":0.00625},{"step":397,"entropy":0.5081859352732311,"zeta":0.003125},{"step":398,"entropy":0.5066427218735778,"zeta":0.00625},{"step":399,"entropy":0.5020369874279497,"zeta":0.0125},{"step":400,"entropy":0.5024051249734516,"zeta":0.025},{"step":401,"entropy":0.5579247013628236,"zeta":0.05},{"step":402,"entropy":0.5585310121619044,"zeta":0.025},{"step":403,"entropy":0.5228536595963275,"zeta":0.0125},{"step":404,"entropy":0.5234987733997747,"zeta":0.025},{"step":405,"entropy":0.5280887164299715,"zeta":0.05},{"step":406,"entropy":0.5285207232537792,"zeta":0.05},{"step":407,"entropy":0.5751241702412228,"zeta":0.05},{"step":408,"entropy":0.5756551111330902,"zeta":0.025},{"step":409,"entropy":0.5109066579643232,"zeta":0.0125},{"step":410,"entropy":0.5127509724772397,"zeta":0.025},{"step":411,"entropy":0.5670205945079783,"zeta":0.05},{"step":412,"entropy":0.5688640914862626,"zeta":0.025},{"step":413,"entropy":0.5847960663686468,"zeta":0.0125},{"step":414,"entropy":0.5858925887536462,"zeta":0.00625},{"step":415,"entropy":0.5773528970840318,"zeta":0.003125},{"step":416,"entropy":0.5783611651815423,"zeta":0.0015625},{"step":417,"entropy":0.5447736475890561,"zeta":0.00078125},{"step":418,"entropy":0.5444056511447625,"zeta":0.000390625},{"step":419,"entropy":0.5564932026135609,"zeta":0.0001953125},{"step":420,"entropy":0.556968267091653,"zeta":0.0},{"step":421,"entropy":0.5348482702837817,"zeta":0.0},{"step":422,"entropy":0.537055372011216,"zeta":0.0001},{"step":423,"entropy":0.5852396241282255,"zeta":0.0002},{"step":424,"entropy":0.5858767466179218,"zeta":0.0001},{"step":425,"entropy":0.5471767461523367,"zeta":0.0},{"step":426,"entropy":0.5474292074415118,"zeta":0.0},{"step":427,"entropy":0.5808252628961758,"zeta":0.0},{"step":428,"entropy":0.5810919470509237,"zeta":0.0},{"step":429,"entropy":0.5556217855817184,"zeta":0.0},{"step":430,"entropy":0.5551554407585123,"zeta":0.0},{"step":431,"entropy":0.5273501996801919,"zeta":0.0},{"step":432,"entropy":0.5298494723902534,"zeta":0.0001},{"step":433,"entropy":0.5868990314660721,"zeta":0.0002},{"step":434,"entropy":0.5856336006521746,"zeta":0.0001},{"step":435,"entropy":0.5682588157175299,"zeta":0.0},{"step":436,"entropy":0.5684119689923668,"zeta":0.0},{"step":437,"entropy":0.530359262685439,"zeta":0.0},{"step":438,"entropy":0.5295876989024632,"zeta":0.0001},{"step":439,"entropy":0.6054274517397754,"zeta":0.0002},{"step":440,"entropy":0.6057487802888666,"zeta":0.0001},{"step":441,"entropy":0.5784512284941195,"zeta":0.0},{"step":442,"entropy":0.5773617701802876,"zeta":0.0},{"step":443,"entropy":0.5759845423393027,"zeta":0.0},{"step":444,"entropy":0.5758725908369987,"zeta":0.0},{"step":445,"entropy":0.5127045457215169,"zeta":0.0},{"step":446,"entropy":0.5113299379246122,"zeta":0.0001},{"step":447,"entropy":0.5084006660511137,"zeta":0.0002},{"step":448,"entropy":0.5084191279935336,"zeta":0.0004},{"step":449,"entropy":0.5248872954036762,"zeta":0.0008},{"step":450,"entropy":0.5238937812955567,"zeta":0.0016},{"step":451,"entropy":0.6174109400336106,"zeta":0.0032},{"step":452,"entropy":0.6172844696506795,"zeta":0.0016},{"step":453,"entropy":0.4620873714412939,"zeta":0.0008},{"step":454,"entropy":0.4623508026852749,"zeta":0.0016},{"step":455,"entropy":0.5871882379071051,"zeta":0.0032},{"step":456,"entropy":0.5859458096387146,"zeta":0.0016},{"step":457,"entropy":0.5354764869701217,"zeta":0.0008},{"step":458,"entropy":0.5352186110618851,"zeta":0.0016},{"step":459,"entropy":0.5058697948237982,"zeta":0.0032},{"step":460,"entropy":0.505374349328636,"zeta":0.0064},{"step":461,"entropy":0.6179424075610345,"zeta":0.0128},{"step":462,"entropy":0.6181595845583848,"zeta":0.0064},{"step":463,"entropy":0.5385137756991706,"zeta":0.0032},{"step":464,"entropy":0.5387550629644171,"zeta":0.0064},{"step":465,"entropy":0.5917943243077386,"zeta":0.0128},{"step":466,"entropy":0.594273246932609,"zeta":0.0064},{"step":467,"entropy":0.5501078437171925,"zeta":0.0032},{"step":468,"entropy":0.5496440594267445,"zeta":0.0016},{"step":469,"entropy":0.5519233712240568,"zeta":0.0008},{"step":470,"entropy":0.5529185367415663,"zeta":0.0004},{"step":471,"entropy":0.5529850538453948,"zeta":0.0002},{"step":472,"entropy":0.5544134779331522,"zeta":0.0001},{"step":473,"entropy":0.5479735667594839,"zeta":0.0},{"step":474,"entropy":0.5493366053944526,"zeta":0.0},{"step":475,"entropy":0.5392808490029202,"zeta":0.0},{"step":476,"entropy":0.5371368439188633,"zeta":0.0001},{"step":477,"entropy":0.5631165623378156,"zeta":0.0002},{"step":478,"entropy":0.5628456964333332,"zeta":0.0001},{"step":479,"entropy":0.5583999841903298,"zeta":0.0},{"step":480,"entropy":0.5581088218395525,"zeta":0.0},{"step":481,"entropy":0.55443166796673,"zeta":0.0},{"step":482,"entropy":0.5544566557575248,"zeta":0.0},{"step":483,"entropy":0.5483631565150219,"zeta":0.0},{"step":484,"entropy":0.5468092114010445,"zeta":0.0},{"step":485,"entropy":0.5470764128387187,"zeta":0.0},{"step":486,"entropy":0.5461170572065981,"zeta":0.0},{"step":487,"entropy":0.6244730172601995,"zeta":0.0},{"step":488,"entropy":0.6240318285302792,"zeta":0.0},{"step":489,"entropy":0.5401213050153317,"zeta":0.0},{"step":490,"entropy":0.538074900425465,"zeta":0.0001},{"step":491,"entropy":0.5716480621816227,"zeta":0.0002},{"step":492,"entropy":0.5700764228983205,"zeta":0.0001},{"step":493,"entropy":0.5027607048687647,"zeta":0.0},{"step":494,"entropy":0.5027307969893886,"zeta":0.0001},{"step":495,"entropy":0.5841803188908611,"zeta":0.0002},{"step":496,"entropy":0.5824409234221386,"zeta":0.0001},{"step":497,"entropy":0.5232112103168395,"zeta":0.0},{"step":498,"entropy":0.5243735716884746,"zeta":0.0001},{"step":499,"entropy":0.5851227478858118,"zeta":0.0002},{"step":500,"entropy":0.5852372111695818,"zeta":0.0001},{"step":501,"entropy":0.571744423194392,"zeta":0.0},{"step":502,"entropy":0.5709317271804495,"zeta":0.0},{"step":503,"entropy":0.5184571833858158,"zeta":0.0},{"step":504,"entropy":0.5193911905151241,"zeta":0.0001},{"step":505,"entropy":0.5080731330654773,"zeta":0.0002},{"step":506,"entropy":0.5073034891442786,"zeta":0.0004},{"step":507,"entropy":0.520760906116881,"zeta":0.0008},{"step":508,"entropy":0.5193555743903474,"zeta":0.0016},{"step":509,"entropy":0.5608016799646348,"zeta":0.0032},{"step":510,"entropy":0.5605573556490918,"zeta":0.0016},{"step":511,"entropy":0.5471470417603549,"zeta":0.0008},{"step":512,"entropy":0.5466671200348088,"zeta":0.0004},{"step":513,"entropy":0.47828967261105676,"zeta":0.0002},{"step":514,"entropy":0.4778097856381212,"zeta":0.0004},{"step":515,"entropy":0.5769397137017136,"zeta":0.0008},{"step":516,"entropy":0.577314659215459,"zeta":0.0004},{"step":517,"entropy":0.4925488264748069,"zeta":0.0002},{"step":518,"entropy":0.4913564211672763,"zeta":0.0004},{"step":519,"entropy":0.5700976148441228,"zeta":0.0008},{"step":520,"entropy":0.5674734416932661,"zeta":0.0004},{"step":521,"entropy":0.5432132410385397,"zeta":0.0002},{"step":522,"entropy":0.5433348284511228,"zeta":0.0001},{"step":523,"entropy":0.5583597012233662,"zeta":0.0},{"step":524,"entropy":0.5577724021690056,"zeta":0.0},{"step":525,"entropy":0.5385718571069193,"zeta":0.0},{"step":526,"entropy":0.5385109311247027,"zeta":0.0001},{"step":527,"entropy":0.5414797060309452,"zeta":0.0002},{"step":528,"entropy":0.5428989208890018,"zeta":0.0004},{"step":529,"entropy":0.4354446324496427,"zeta":0.0002},{"step":530,"entropy":0.4335229535951153,"zeta":0.0004},{"step":531,"entropy":0.5332032253530627,"zeta":0.0008},{"step":532,"entropy":0.5333455099998565,"zeta":0.0016},{"step":533,"entropy":0.5291922930298222,"zeta":0.0032},{"step":534,"entropy":0.5305433606764871,"zeta":0.0064},{"step":535,"entropy":0.44647902914620047,"zeta":0.0128},{"step":536,"entropy":0.4452843180251515,"zeta":0.0256},{"step":537,"entropy":0.5218477891347995,"zeta":0.05},{"step":538,"entropy":0.5215983637233245,"zeta":0.05},{"step":539,"entropy":0.563237761674381,"zeta":0.05},{"step":540,"entropy":0.5645064539104501,"zeta":0.025},{"step":541,"entropy":0.5560831258268109,"zeta":0.0125},{"step":542,"entropy":0.5546028021410626,"zeta":0.00625},{"step":543,"entropy":0.500386634134476,"zeta":0.003125},{"step":544,"entropy":0.5004314559381846,"zeta":0.00625},{"step":545,"entropy":0.5342699182731582,"zeta":0.0125},{"step":546,"entropy":0.5327382629126831,"zeta":0.025},{"step":547,"entropy":0.524145248886164,"zeta":0.05},{"step":548,"entropy":0.5239175112434518,"zeta":0.05},{"step":549,"entropy":0.49526499404316654,"zeta":0.05},{"step":550,"entropy":0.49723837581801944,"zeta":0.05},{"step":551,"entropy":0.5539434912301056,"zeta":0.05},{"step":552,"entropy":0.5547474822132173,"zeta":0.025},{"step":553,"entropy":0.50987551341781,"zeta":0.0125},{"step":554,"entropy":0.5096592061512112,"zeta":0.025},{"step":555,"entropy":0.5003809779804275,"zeta":0.05},{"step":556,"entropy":0.5005387734771091,"zeta":0.05},{"step":557,"entropy":0.5082388429433566,"zeta":0.05},{"step":558,"entropy":0.5097743274413956,"zeta":0.05},{"step":559,"entropy":0.560923241263026,"zeta":0.05},{"step":560,"entropy":0.5612960481776123,"zeta":0.025},{"step":561,"entropy":0.5914708383046249,"zeta":0.0125},{"step":562,"entropy":0.5918815360521752,"zeta":0.00625},{"step":563,"entropy":0.5567703729456696,"zeta":0.003125},{"step":564,"entropy":0.5576075686576563,"zeta":0.0015625},{"step":565,"entropy":0.5308008258217695,"zeta":0.00078125},{"step":566,"entropy":0.5299186230443457,"zeta":0.0015625},{"step":567,"entropy":0.5842988304848322,"zeta":0.003125},{"step":568,"entropy":0.585809842238795,"zeta":0.0015625},{"step":569,"entropy":0.5173265423541146,"zeta":0.00078125},{"step":570,"entropy":0.5183750707042293,"zeta":0.0015625},{"step":571,"entropy":0.5356392829261513,"zeta":0.003125},{"step":572,"entropy":0.5355383117174929,"zeta":0.00625},{"step":573,"entropy":0.536851183922129,"zeta":0.0125},{"step":574,"entropy":0.5371726877085594,"zeta":0.025},{"step":575,"entropy":0.5808365005239634,"zeta":0.05},{"step":576,"entropy":0.5790231328125948,"zeta":0.025},{"step":577,"entropy":0.5706301048607879,"zeta":0.0125},{"step":578,"entropy":0.570192314987791,"zeta":0.00625},{"step":579,"entropy":0.5948450028808644,"zeta":0.003125},{"step":580,"entropy":0.5949862159175457,"zeta":0.0015625},{"step":581,"entropy":0.4985776364484226,"zeta":0.00078125},{"step":582,"entropy":0.4984528617738949,"zeta":0.0015625},{"step":583,"entropy":0.6210775363440033,"zeta":0.003125},{"step":584,"entropy":0.6218682494677636,"zeta":0.0015625},{"step":585,"entropy":0.5083518161434929,"zeta":0.00078125},{"step":586,"entropy":0.5067423900748922,"zeta":0.0015625},{"step":587,"entropy":0.47496347703714054,"zeta":0.003125},{"step":588,"entropy":0.47404369879135705,"zeta":0.00625},{"step":589,"entropy":0.5485384792490886,"zeta":0.0125},{"step":590,"entropy":0.5514844345231936,"zeta":0.00625},{"step":591,"entropy":0.553390384518431,"zeta":0.003125},{"step":592,"entropy":0.5554717162570705,"zeta":0.0015625},{"step":593,"entropy":0.5028056077799069,"zeta":0.00078125},{"step":594,"entropy":0.5033480309844547,"zeta":0.0015625},{"step":595,"entropy":0.57522699848455,"zeta":0.003125},{"step":596,"entropy":0.5737008164373437,"zeta":0.0015625},{"step":597,"entropy":0.5966467879520115,"zeta":0.00078125},{"step":598,"entropy":0.5961317352156402,"zeta":0.000390625},{"step":599,"entropy":0.5618451720120727,"zeta":0.0001953125},{"step":600,"entropy":0.5634844585623567,"zeta":0.0}]}}