h%aSSKJr SSKrSSKrSSKrSSKrSSKrSSKrSSKrSSK r SSK r SSK r SSK rSSKrSSKJr SSKJrJr SSKJrJrJrJr SSKr\R8"\5rSSjrSr Sr!S r"S r#S r$S r%\S :Xa\%"5 gg)) annotationsN) setup_logger)add_io_bindings_as_tensorsget_initial_inputs_and_outputs) AutoConfigAutoModelForCausalLM AutoTokenizerBitsAndBytesConfigc URS;GaSnURS:XaURS:Xa[SSS[R S9n[ R"URS:wa URO URURURURURSSUURS 0S 9 nO[ R"URS:wa URO URURURURURSURS:XaSOS S 9RUR 5nUR'5 URS:Xa[R("U5nU$[*R,"5nURS:XaSSUR04OSn[*R."UR0XE/S9nU$!["an[%S U5 [ R"URS:wa URO URURURURURSSS 9RUR 5nSnAGN.SnAff=f)Npt-eager pt-compileint4cudaTnf4) load_in_4bitbnb_4bit_use_double_quantbnb_4bit_quant_typebnb_4bit_compute_dtypeflash_attention_280GB) cache_dir torch_dtypeuse_auth_tokentrust_remote_code use_cacheattn_implementationquantization_config max_memorysdpa)rrrrrrz&Try to load a model using eager mode: eagerrCUDAExecutionProvider device_idCPUExecutionProvider) sess_options providers)benchmark_typeonnx_precisiondevicer torchfloat16rfrom_pretrained hf_dir_path model_namerrauthtrustr$to target_device ExceptionprintevalcompileortSessionOptionsInferenceSessiononnx_model_path)argsmodel bnb_configer&eps m/var/www/fran/franai/venv/lib/python3.13/site-packages/onnxruntime/transformers/models/llama/benchmark_e2e.py get_modelrB8s0 88   & (T[[F-B+!*.$)',}} J)88$($4$4$:  .. ,,#yy"&**$7$. NNF3 E ),<<(,(8(8B(>D$$DOO"nn $ 0 0#'99&*jj"@D v@U)<[a"T''(.    , .MM%(E L))+ {{f$%{DNN&C D' $$T%9%9 `de L9 )>B,<<(,(8(8B(>D$$DOO"nn $ 0 0#'99&*jj"(/"T''( )sBG** J 4B JJ c`URS:Xa&[R"5 U"S0UD6nSSS5 SnURS;a:URS:wa)[RR UR 5 O1[XX@RUR5nUR5 [R"5n[U5HnURS;aa[R"5 U"S0UD6nURS:wa)[RR UR 5 SSS5 MtURU5 UR5 M [R"5nX- U- n X4$!,(df  GNd=f!,(df  M=f)Nrr cpu)r(r+no_gradr*r synchronizer3ruse_fp16use_buffer_sharesynchronize_inputstime perf_counterrangerun_with_iobindingsynchronize_outputs) r<r=runsinputsoutputs io_bindingstart_endavgs rA run_inferencerXxsE l* ]]_ofoGJ 88 ;;%  JJ " "4#5#5 6/w W[WlWlm %%'    E 4[   "< </&/;;%'JJ**4+=+=>!  $ $Z 0  * * ,    C ;$ C <5_ !s F OS?S?S>/S@9 URSASB[SCSD9 URSESF[SGSD9 URSHSI[SJSD9 URSK[SLSD9 UR5n[RRUR5 [R"UR5 SMUR ;aW[#USNUR$R'5SO35 UR(SP:XaUR(SQUR*04UlUR SM:XaUR,(dSR5eUR.R1SS5UlUR2R1SS5Ul[#USTUR45 UR4SU;d UR4SV:XaUR$S?:XaS4OSWUlUR$S?:waSXUR*3O UR$nUR4SW:Xa[R6O[R8nUR SM:XaSMOSYn[#USZU5 [#US[U5 [#US\U5 [#US]UR4SW:H5 UR:=(a USM:HUlU$)^Nz-btz--benchmark-typeT)r rr8)typerequiredchoicesz-mz --model-nameFzrrrrzcuda:ptr3rr\rH)argparseArgumentParser add_argumentstrospathjoinintr+r is_available parse_argsnprandomseed manual_seedr(setattrr*upperrr$r; batch_sizessplitprompt_lengths precisionr,float32rI)parserr<r3rr\s rAget_argsrs  $ $ &F   1     K    D    t     S-0D   p    !    S(G^DC   B   !P             0 t    /    **1133  }3B oCC lcB sA6    DIINN499 dii  ###*t{{/@/@/B.CCT,UV  " "&= ='+'>'>dnn@]&^D # e###S%SS#''--c2D--33C8D D"DNN3..$4469QVZVaVaejVjqw N15 u0DeDNN+,$++M#'>>V#;%--K))U2UF D/=1 D-- D(F# D*dnn67 11EfoD Krcc^9^:[5n[S5 [RUR5 Sn[ UR 5n[R"USS9nSSS5 [R"URS:wa URO URURURURS9n[ R"URS:wa URO URURURURS9n[#U5n/n[$R&"UR(UR*5GH]upx[-U5[-U5p[RSUSU35 [/5 XR0-n X;a\[3[4R6"SUS UR S UR S US US USUSUSUR SUS355eX/U-n Xx/n [RS5 [9XX4X5up[;XUR<X5upUS-nXxU- -n[RSUS35 [RSXxU- -S35 U R?UU/5 [RS5 [/5 [9XX4X5upU SRA5nURBSnURDn[GUS5(a URHOURJURL-n[NRP"XpRR[NRTS9n/n/n[VRX"5nUU ::Ga~[;XSX5unn UR[U5 [VRX"5nU SRBSS:aU S R]S5S- nUR_SS!9RaSURb5ReUSURb5n[NRf"U SSU5Ri5nOU SSS2SSS24n[NRj"USS!9nUU-URl:HnURoUURl5RqUS/5n[VRX"5n UR[U U- 5 [NRr"UU/SS!9nUS- nUU S'[NRr"U S U)Ru[NRv5RqUS5/S5U S 'S"U ;a1[NRx"U S"SS!9S#RqUS5S-U S"'U SRBSS:wa"U SSS2SS2SS24R{5U S'U SR}5 UR~S$:Xa U S%U S%'GOUR(d[UR5H#n!U S&U!S'3U S(U!S'3'U S&U!S)3U S(U!S)3'M% U S RBSn"[UR5Hn![NRP"UUU"UURRURS9n#[NRP"UUU"UURRURS9n$U RS&U!S'3U#R{5S&U!S)3U$R{505 M UU ::aGM~[VRX"5n%URS#5 UR(abS*m9[U5m:[U5n&[[U9U:4S+jU55n[U5n'[RS,U&U'- S-T9S.T:S-S/35 []U5[U5- n(U(S-n)USU(- -n*[RS0U)S35 [RS1U*S35 US#n+U+S-n,USU+- -n-[RS2U,S35 [RS3U-S35 UR0S4-n.[]USU.5[USU.5- n/U/S-n0USU/- -n1[RS5U.S6U0S35 [RS7U.S6U1S35 []U5[U5- n2U2S-n3USU2- -n4[RS5UR0S6U3S35 [RS7UR0S6U4S35 U%U- n5XxUR0-U5- -n6[RS8U5S935 [RS:XxUR0-U5- -S35 [RS;5 U R?U)U*U,U-U0U1U3U4U5U6/ 5 UR[U 5 GM` S>UR~S?[RR5S@SA3n8[UU8UR05 g!,(df  G N=f![a*n7[RSmain..js,STSZSZS\<]S\41SVQYS\<]<]s0) object_hookr)rrrzRunning batch size = z, prompt length = z2 A prompt of size z was not found in 'zv'. There are a couple of solutions to fix this. 1) You can change one of the keys in 'z' to be z). If za < actual prompt's length, the benchmark E2E tool will repeat the first word in the prompt until zB = actual prompt's length. If zm > actual prompt's length, the benchmark E2E tool will automatically trim the actual prompt's length so that zd = actual prompt's length. 2) You can add a new key-value entry in 'z' of the form 'z,': 'your prompt goes here'. zMeasuring prompt processing...iz&Average Latency of Prompt Processing: z msz)Average Throughput of Prompt Processing: z tpszMeasuring token generation... input_idshead_dim)r*dtypelogitsattention_mask)dim position_idsrrpast_key_valueszpresent.z.keyzpast_key_values.z.value c>UTT-:$rZrE)acc_timeanomaly_threshold_factor min_time_ss rArr sH7OR\7\,\rcz Filtered out z$ anomaly accelerator times that are zx greater than z ms...zAverage Latency of Sampling: z Average Throughput of Sampling: z"Latency of First Token Generated: z%Throughput of First Token Generated: rjriz Tokens Generated: rkzWall-Clock Latency: z szWall-Clock Throughput: zAdding results to CSVz$Could not benchmark at batch size = z - benchmark__e2e_z%Y-%m-%d_%H:%M:%Sz.csv)Orrrrrs__dict__open prompts_filejsonloadrr-r.r/rr0r1r rB itertoolsproductrrrr[generation_lengthNotImplementedErrortextwrapdedentrbrXnum_runsextendcloneshapenum_key_value_headshasattrr hidden_sizenum_attention_headsr+zerosr3boolrKrLappendsum unsqueezerepeat vocab_sizeviewgathersqueezeargmax eos_token_id masked_fillreshapecatr2int64max contiguouszero_r\rIrMnum_hidden_layersrupdatepopanomaly_filteringminlenlistfilterr4datetimenowrx);r<size_to_promptfr^r_r=all_csv_metrics batch_sizer` max_lengthra csv_metricsrQrRaccelerator_prompt_latency_saccelerator_prompt_latency_msaccelerator_prompt_thrpt all_token_idscurrent_length num_heads head_sizehas_eosaccelerator_timessampling_timeswall_clock_start_timeaccelerator_time_latency_ssampling_start_timeprompt_end_indicesidxsnext_token_logits next_tokens tokens_to_addsampling_end_timeinew_sequence_length present_key present_valuewall_clock_end_time orig_sizenew_sizeavg_sampling_latency_savg_sampling_latency_msavg_sampling_thrptfirst_token_latency_sfirst_token_latency_msfirst_token_thrpthalfwayhalfway_token_latency_shalfway_token_latency_mshalfway_token_thrptall_token_latency_sall_token_latency_msall_token_thrptwall_clock_latency_swall_clock_thrptr?rurrs; @@rAmainrbs :D KK N d A12]^ ! ' ' ,,2..yy** F -- ,,2..yy** I dOEO%.%6%6t7G7GI\I\%]! $' OS5GM +J<7I-YZ "%;%;;  .%22?@STXTeTeSfgGGKGXGXFYYaboapq((57XYfXgg((57derdssJJNJ[J[I\\klykz{   !/0:=!1 H t KK8 99$vZgpOF4A$t}}^d4n 1 (-I4,O )'1Ea5a'b $ KK@A^@__bc d KK;JZvJv#>FDVDVZ`ZtZtDt kk*5G5GuzzZG!# N$($5$5$7 ! J.6CDQRTZ6d3*G!(()CD'+&7&7&9#8$**1-1)/0@)A)E)Ea)H1)L&*444;6#4#45j!V->->? ). WX5F4(P(X(X(Z%(/(9!R((C%#ll+<"E "K/93I3II!, 7 7AWAW X ` `blnoap q $($5$5$7!%%&7:M&MN % =-*Hb Q !#'4{#+099,-}}U[[/I/Q/QR\^_/`acd,'("V+-2YYvn7MST-UVW-X-`-`akmn-ors-sF>*8$**1-2(/(9!RaR((C(N(N(PGH%!'');;$&078I0JF,-.."6#;#;<=DxPQsRVEW=X!1!D9:?FRSQTTZG[?\!1!F;<=+11A*B*H*H*K'"6#;#;<&+kk&%/%#'#5#5"&"2"2 ' ). &%/%#'#5#5"&"2"2 )  "*1#T 2K4J4J4L"*1#V 4m6N6N6P#=o!J.^#'"3"3"5   ! !! $%%+-( !23  12 $(\^op%!01 #I$8#99]^v]wxGHRUYHYGZZ`a&)%83~;N%N "&'E $",4K0K"L  KK3G9T>T.TXl-lm  KK./C.DBG H KK)*I_I_9_cw8w*x)yy}~  KK/ 0   +&*%,'(#($    " "; /u&^~DKK=h.?.?.C.C.EFW-XX\]H(D,B,BCg ! ^ t KK>zlJ\]j\kknopnqr s s ts,hS-hJ2h h i !ii __main__)r<zargparse.Namespace)& __future__rrrrerrloggingrrrKnumpyrpandasror+benchmark_helperr llama_inputsrr transformersrrr r onnxruntimer8 getLogger__name__rrrBrXrbr[rxrrrErcrAr+s@#     )S\\   8 $=@> 12aHzDz zFrc