| { | |
| "prefill": { | |
| "memory": { | |
| "unit": "MB", | |
| "max_ram": 1493.024768, | |
| "max_vram": 43014.684672, | |
| "max_reserved": 42087.743488, | |
| "max_allocated": 42065.570304 | |
| }, | |
| "latency": { | |
| "unit": "s", | |
| "mean": 0.037890100097656254, | |
| "stdev": 0.0008360956935448878, | |
| "values": [ | |
| 0.039532577514648434, | |
| 0.03775779342651367, | |
| 0.03729081726074219, | |
| 0.037489631652832034, | |
| 0.03737968063354492 | |
| ] | |
| }, | |
| "throughput": { | |
| "unit": "tokens/s", | |
| "value": 844.5477820730118 | |
| }, | |
| "energy": null, | |
| "efficiency": null | |
| }, | |
| "decode": { | |
| "memory": { | |
| "unit": "MB", | |
| "max_ram": 1493.024768, | |
| "max_vram": 43826.282496, | |
| "max_reserved": 42895.147008, | |
| "max_allocated": 42662.806016 | |
| }, | |
| "latency": { | |
| "unit": "s", | |
| "mean": 2.199382656097412, | |
| "stdev": 0.00898152331265916, | |
| "values": [ | |
| 2.216529022216796, | |
| 2.1937663440704336, | |
| 2.1978081874847417, | |
| 2.1979938869476325, | |
| 2.190815839767456 | |
| ] | |
| }, | |
| "throughput": { | |
| "unit": "tokens/s", | |
| "value": 90.02526206664352 | |
| }, | |
| "energy": null, | |
| "efficiency": null | |
| }, | |
| "per_token": { | |
| "memory": null, | |
| "latency": { | |
| "unit": "s", | |
| "mean": 0.02183325814141167, | |
| "stdev": 0.0005212691161057472, | |
| "values": [ | |
| 0.029347072601318358, | |
| 0.02262201690673828, | |
| 0.023965343475341797, | |
| 0.021531103134155273, | |
| 0.02134383964538574, | |
| 0.021348800659179688, | |
| 0.02144771194458008, | |
| 0.021367071151733398, | |
| 0.0214136962890625, | |
| 0.021444799423217774, | |
| 0.021631328582763672, | |
| 0.021643264770507813, | |
| 0.02167692756652832, | |
| 0.021710399627685548, | |
| 0.02169481658935547, | |
| 0.021601152420043946, | |
| 0.02165872001647949, | |
| 0.021706527709960937, | |
| 0.021705120086669923, | |
| 0.021762367248535155, | |
| 0.021770816802978515, | |
| 0.021766912460327147, | |
| 0.021722240447998045, | |
| 0.021723648071289063, | |
| 0.021749824523925782, | |
| 0.021739391326904296, | |
| 0.02199225616455078, | |
| 0.02174723243713379, | |
| 0.021734783172607422, | |
| 0.021787424087524414, | |
| 0.021712255477905274, | |
| 0.02176963233947754, | |
| 0.02179020881652832, | |
| 0.021766815185546875, | |
| 0.02172598457336426, | |
| 0.021785728454589842, | |
| 0.02184556770324707, | |
| 0.021842559814453124, | |
| 0.021833215713500977, | |
| 0.021850719451904296, | |
| 0.021775487899780274, | |
| 0.021766143798828123, | |
| 0.02170585632324219, | |
| 0.021779232025146485, | |
| 0.021785472869873045, | |
| 0.021925024032592774, | |
| 0.02171891212463379, | |
| 0.021805408477783204, | |
| 0.02187264060974121, | |
| 0.02187049674987793, | |
| 0.0218702392578125, | |
| 0.02184988784790039, | |
| 0.021831552505493165, | |
| 0.021871904373168945, | |
| 0.021849504470825197, | |
| 0.021812383651733398, | |
| 0.021807296752929688, | |
| 0.02185897636413574, | |
| 0.021690784454345705, | |
| 0.02188934326171875, | |
| 0.021849056243896485, | |
| 0.021892255783081054, | |
| 0.021870208740234376, | |
| 0.021681728363037108, | |
| 0.021855007171630858, | |
| 0.02153990364074707, | |
| 0.021656831741333007, | |
| 0.021794431686401366, | |
| 0.02193631935119629, | |
| 0.029309440612792968, | |
| 0.021998207092285157, | |
| 0.021921087265014648, | |
| 0.021899328231811524, | |
| 0.021797439575195313, | |
| 0.02189766311645508, | |
| 0.021949920654296875, | |
| 0.021826976776123046, | |
| 0.021930559158325195, | |
| 0.021840383529663086, | |
| 0.021912895202636718, | |
| 0.02201795196533203, | |
| 0.02197823905944824, | |
| 0.02196803283691406, | |
| 0.02204854393005371, | |
| 0.021927839279174806, | |
| 0.022026304244995118, | |
| 0.021970655441284178, | |
| 0.021984319686889648, | |
| 0.021955455780029297, | |
| 0.021968416213989258, | |
| 0.021939231872558595, | |
| 0.022139936447143554, | |
| 0.02198259162902832, | |
| 0.0219870719909668, | |
| 0.021998079299926757, | |
| 0.022019775390625, | |
| 0.022041343688964845, | |
| 0.022055744171142578, | |
| 0.022011423110961915, | |
| 0.021683359146118165, | |
| 0.021492511749267577, | |
| 0.021508415222167968, | |
| 0.021610591888427736, | |
| 0.021473407745361328, | |
| 0.021433055877685545, | |
| 0.02146201515197754, | |
| 0.021375808715820312, | |
| 0.021380096435546874, | |
| 0.021473215103149413, | |
| 0.021683359146118165, | |
| 0.021675872802734374, | |
| 0.021704704284667968, | |
| 0.021701887130737306, | |
| 0.021738496780395508, | |
| 0.021661855697631835, | |
| 0.021823808670043944, | |
| 0.0218492488861084, | |
| 0.02174492835998535, | |
| 0.021795711517333984, | |
| 0.021731903076171875, | |
| 0.02178492736816406, | |
| 0.021862207412719728, | |
| 0.021814464569091797, | |
| 0.021730527877807618, | |
| 0.021747392654418947, | |
| 0.021714624404907228, | |
| 0.021825439453125, | |
| 0.02169264030456543, | |
| 0.021713151931762695, | |
| 0.021663936614990234, | |
| 0.021675840377807617, | |
| 0.02172902488708496, | |
| 0.02171366310119629, | |
| 0.02174492835998535, | |
| 0.02178985595703125, | |
| 0.021789791107177735, | |
| 0.02175276756286621, | |
| 0.021762176513671874, | |
| 0.021842208862304688, | |
| 0.02177894401550293, | |
| 0.021775264739990235, | |
| 0.02176304054260254, | |
| 0.021687776565551757, | |
| 0.021745759963989256, | |
| 0.021684928894042967, | |
| 0.02180031967163086, | |
| 0.021425344467163085, | |
| 0.021657632827758788, | |
| 0.021671968460083006, | |
| 0.021820640563964842, | |
| 0.02187788772583008, | |
| 0.021898048400878906, | |
| 0.02196361541748047, | |
| 0.02189641571044922, | |
| 0.021949888229370117, | |
| 0.02186204719543457, | |
| 0.02190812873840332, | |
| 0.02188889694213867, | |
| 0.021675775527954102, | |
| 0.02162735939025879, | |
| 0.02165235137939453, | |
| 0.021707551956176758, | |
| 0.02143484878540039, | |
| 0.02149184036254883, | |
| 0.021748640060424804, | |
| 0.021891647338867188, | |
| 0.021848480224609376, | |
| 0.02185100746154785, | |
| 0.02187014389038086, | |
| 0.021819520950317382, | |
| 0.021775007247924805, | |
| 0.021891103744506837, | |
| 0.021874496459960938, | |
| 0.021861024856567383, | |
| 0.021540607452392578, | |
| 0.021673759460449218, | |
| 0.021816608428955078, | |
| 0.021896640777587892, | |
| 0.02182806396484375, | |
| 0.021955968856811524, | |
| 0.022031007766723634, | |
| 0.021960800170898437, | |
| 0.022021055221557617, | |
| 0.021929088592529296, | |
| 0.022040800094604494, | |
| 0.022005023956298827, | |
| 0.02198476791381836, | |
| 0.021997472763061524, | |
| 0.022040063858032227, | |
| 0.02198204803466797, | |
| 0.02195145606994629, | |
| 0.021917184829711913, | |
| 0.022082143783569336, | |
| 0.022013023376464845, | |
| 0.02198784065246582, | |
| 0.021985984802246093, | |
| 0.022008224487304686, | |
| 0.021927743911743163, | |
| 0.02147443199157715, | |
| 0.021369056701660158, | |
| 0.021374015808105468, | |
| 0.02133296012878418, | |
| 0.021273151397705078, | |
| 0.021375680923461916, | |
| 0.02134115219116211, | |
| 0.02140924835205078, | |
| 0.021337631225585938, | |
| 0.022026336669921875, | |
| 0.02168684768676758, | |
| 0.021585599899291992, | |
| 0.021703168869018553, | |
| 0.021655424118041992, | |
| 0.021714975357055664, | |
| 0.02156368064880371, | |
| 0.021787071228027345, | |
| 0.021758432388305663, | |
| 0.021763423919677734, | |
| 0.021708320617675782, | |
| 0.02182419204711914, | |
| 0.021792415618896485, | |
| 0.021794912338256835, | |
| 0.021691999435424804, | |
| 0.021776735305786134, | |
| 0.02166819190979004, | |
| 0.021843967437744142, | |
| 0.021770559310913085, | |
| 0.021832544326782225, | |
| 0.021779392242431642, | |
| 0.0217741756439209, | |
| 0.021696735382080078, | |
| 0.021841535568237303, | |
| 0.021770143508911134, | |
| 0.021863872528076173, | |
| 0.021792064666748046, | |
| 0.021830207824707033, | |
| 0.021761215209960938, | |
| 0.021843008041381836, | |
| 0.02174236869812012, | |
| 0.02185625648498535, | |
| 0.02175049591064453, | |
| 0.021806655883789064, | |
| 0.021730079650878906, | |
| 0.021819551467895507, | |
| 0.021772127151489257, | |
| 0.02187398338317871, | |
| 0.021663423538208007, | |
| 0.021886144638061523, | |
| 0.021798336029052734, | |
| 0.02187913513183594, | |
| 0.02183660888671875, | |
| 0.021887231826782226, | |
| 0.02184815979003906, | |
| 0.021905792236328124, | |
| 0.021881919860839844, | |
| 0.021952127456665037, | |
| 0.021919647216796876, | |
| 0.021915391921997072, | |
| 0.02191801643371582, | |
| 0.02195155143737793, | |
| 0.021899871826171875, | |
| 0.021985727310180662, | |
| 0.021782112121582032, | |
| 0.021962400436401366, | |
| 0.02191663932800293, | |
| 0.021988479614257813, | |
| 0.02191139221191406, | |
| 0.022001440048217774, | |
| 0.021911008834838867, | |
| 0.021917343139648438, | |
| 0.021897119522094728, | |
| 0.02202579116821289, | |
| 0.021875999450683595, | |
| 0.021925600051879882, | |
| 0.02189423942565918, | |
| 0.021955839157104494, | |
| 0.021890464782714843, | |
| 0.021950048446655275, | |
| 0.021813343048095703, | |
| 0.02194633674621582, | |
| 0.02195769691467285, | |
| 0.022031328201293946, | |
| 0.021926816940307618, | |
| 0.02196086311340332, | |
| 0.021849472045898436, | |
| 0.022041376113891602, | |
| 0.021983392715454103, | |
| 0.022031103134155273, | |
| 0.021996864318847655, | |
| 0.022117055892944337, | |
| 0.02207926368713379, | |
| 0.02209190368652344, | |
| 0.021988767623901367, | |
| 0.02206502342224121, | |
| 0.021940576553344728, | |
| 0.022121536254882813, | |
| 0.022028640747070314, | |
| 0.022071008682250978, | |
| 0.021483295440673827, | |
| 0.021322399139404296, | |
| 0.021335935592651366, | |
| 0.02121196746826172, | |
| 0.021095680236816405, | |
| 0.02118332862854004, | |
| 0.02145804786682129, | |
| 0.02141779136657715, | |
| 0.02138729667663574, | |
| 0.021476512908935548, | |
| 0.02165670394897461, | |
| 0.02165433692932129, | |
| 0.02172051239013672, | |
| 0.02164796829223633, | |
| 0.021689216613769532, | |
| 0.021590208053588866, | |
| 0.02176585578918457, | |
| 0.02161129570007324, | |
| 0.02177030372619629, | |
| 0.021693407058715822, | |
| 0.02175056076049805, | |
| 0.02171513557434082, | |
| 0.0217541446685791, | |
| 0.02164303970336914, | |
| 0.021761024475097656, | |
| 0.021723871231079103, | |
| 0.021827552795410158, | |
| 0.02168364715576172, | |
| 0.021767328262329102, | |
| 0.021723295211791994, | |
| 0.02187411117553711, | |
| 0.021684703826904298, | |
| 0.022009056091308595, | |
| 0.021746400833129884, | |
| 0.021779327392578124, | |
| 0.021797056198120116, | |
| 0.021850559234619142, | |
| 0.021800064086914064, | |
| 0.021880895614624023, | |
| 0.021776832580566407, | |
| 0.021808671951293945, | |
| 0.021796415328979492, | |
| 0.02184217643737793, | |
| 0.021788639068603517, | |
| 0.021946815490722655, | |
| 0.02187071990966797, | |
| 0.021883808135986327, | |
| 0.02174604797363281, | |
| 0.02191059112548828, | |
| 0.022557760238647463, | |
| 0.021941503524780272, | |
| 0.021842304229736327, | |
| 0.021875616073608398, | |
| 0.021873119354248047, | |
| 0.02194095993041992, | |
| 0.02182371139526367, | |
| 0.021946016311645507, | |
| 0.02181011199951172, | |
| 0.021960447311401367, | |
| 0.021831392288208008, | |
| 0.02185910415649414, | |
| 0.02180294418334961, | |
| 0.021954816818237306, | |
| 0.021772415161132812, | |
| 0.021898784637451173, | |
| 0.02187593650817871, | |
| 0.02190083122253418, | |
| 0.02186332893371582, | |
| 0.021973440170288086, | |
| 0.021879072189331054, | |
| 0.02202406311035156, | |
| 0.02190764808654785, | |
| 0.022033056259155272, | |
| 0.021932159423828125, | |
| 0.0219583683013916, | |
| 0.022119392395019533, | |
| 0.022042560577392577, | |
| 0.021935199737548827, | |
| 0.02208176040649414, | |
| 0.021839263916015626, | |
| 0.022129728317260743, | |
| 0.02195612716674805, | |
| 0.021954015731811525, | |
| 0.021965503692626953, | |
| 0.02201190376281738, | |
| 0.022010688781738282, | |
| 0.021871648788452148, | |
| 0.022074975967407227, | |
| 0.022027551651000978, | |
| 0.022036224365234374, | |
| 0.021947999954223633, | |
| 0.022081024169921876, | |
| 0.02200377655029297, | |
| 0.02206719970703125, | |
| 0.022012639999389648, | |
| 0.021951488494873047, | |
| 0.021966304779052735, | |
| 0.02203766441345215, | |
| 0.02202614402770996, | |
| 0.021597183227539063, | |
| 0.021417375564575195, | |
| 0.021375104904174803, | |
| 0.021378175735473633, | |
| 0.02124470329284668, | |
| 0.021448480606079103, | |
| 0.02135590362548828, | |
| 0.021406591415405272, | |
| 0.021387935638427735, | |
| 0.02148137664794922, | |
| 0.021792320251464842, | |
| 0.021642431259155274, | |
| 0.021756799697875976, | |
| 0.021642751693725586, | |
| 0.021767839431762696, | |
| 0.021679584503173827, | |
| 0.021840480804443358, | |
| 0.021773920059204102, | |
| 0.021792640686035158, | |
| 0.02173676872253418, | |
| 0.021795999526977538, | |
| 0.021719104766845704, | |
| 0.02174745559692383, | |
| 0.021709728240966796, | |
| 0.021778335571289064, | |
| 0.02170070457458496, | |
| 0.02182054328918457, | |
| 0.021694528579711915, | |
| 0.021765567779541015, | |
| 0.02167849540710449, | |
| 0.02171945571899414, | |
| 0.021613855361938477, | |
| 0.02168409538269043, | |
| 0.02170172882080078, | |
| 0.02164556884765625, | |
| 0.02161987113952637, | |
| 0.021691263198852537, | |
| 0.02164182472229004, | |
| 0.02174611282348633, | |
| 0.02166316795349121, | |
| 0.021715776443481445, | |
| 0.02171014404296875, | |
| 0.02177020835876465, | |
| 0.021672927856445312, | |
| 0.021710464477539063, | |
| 0.021587488174438476, | |
| 0.021537792205810546, | |
| 0.02151046371459961, | |
| 0.021684736251831056, | |
| 0.021700992584228515, | |
| 0.021579744338989258, | |
| 0.021655328750610353, | |
| 0.02161471939086914, | |
| 0.021631935119628905, | |
| 0.02165305519104004, | |
| 0.02166329574584961, | |
| 0.02175644874572754, | |
| 0.021761632919311522, | |
| 0.021840831756591798, | |
| 0.021863872528076173, | |
| 0.02185558319091797, | |
| 0.021831584930419923, | |
| 0.02176361656188965, | |
| 0.02177142333984375, | |
| 0.021769664764404298, | |
| 0.02189619255065918, | |
| 0.021961952209472658, | |
| 0.021843168258666994, | |
| 0.02189411163330078, | |
| 0.021814048767089842, | |
| 0.021952384948730468, | |
| 0.02182111930847168, | |
| 0.021736799240112306, | |
| 0.021818815231323244, | |
| 0.021985727310180662, | |
| 0.021856767654418945, | |
| 0.021934623718261718, | |
| 0.02186198425292969, | |
| 0.02193903923034668, | |
| 0.021879199981689454, | |
| 0.022030399322509765, | |
| 0.021993568420410156, | |
| 0.021932031631469725, | |
| 0.021975135803222655, | |
| 0.021859519958496092, | |
| 0.021967647552490234, | |
| 0.021871103286743163, | |
| 0.02193401527404785, | |
| 0.021945632934570313, | |
| 0.021993631362915038, | |
| 0.021945856094360353, | |
| 0.02196169662475586, | |
| 0.021878816604614257, | |
| 0.021994815826416016, | |
| 0.021964704513549805, | |
| 0.021893503189086914, | |
| 0.021938623428344725, | |
| 0.021959232330322265, | |
| 0.021930784225463868 | |
| ] | |
| }, | |
| "throughput": { | |
| "unit": "tokens/s", | |
| "value": 91.60336890839721 | |
| }, | |
| "energy": null, | |
| "efficiency": null | |
| } | |
| } |