
    h.                        S SK r S SKrS SKrS SKr S SKJrJr  S SK	J
r
  S SKJr  S SKJr  S SKJr  S SKJr  S SKJr  S S	KJr  S S
KJrJr  \R6                  " \5      \R6                  " S5      :  a  \" S\ S35      e/ SQrS\4S jr SS\S\S-  S\4S jjr S\!\"\\4      S\4S jr#S\!\"\\4      S\S\4S jr$S r%SS jr&\'S:X  a  \%" 5         gg! \ a    S SKJrJr   Nf = f)    N)PackageNotFoundErrorversion)Path)Any)load_dataset)	evaluator)ORTModelForQuestionAnswering)__version__)r   )AutoTokenizerpipelinez1.13.1z1Please install optimum>=1.13.1. Current version: .)z5bert-large-uncased-whole-word-masking-finetuned-squadzdeepset/roberta-base-squad2z%distilbert-base-cased-distilled-squadpackage_namec                 :     [        U 5      $ ! [         a     g f = fN)r   r   )r   s    i/var/www/fran/franai/venv/lib/python3.13/site-packages/onnxruntime/transformers/models/bert/eval_squad.pyget_package_versionr   7   s$    |$$ s   
 
model_id	onnx_pathuse_io_bindingc                    Ucv  [         R                  " U SX#S9n[        R                  R	                  SU 5      nUR                  U5        [        R                  R	                  US5      n[        SU5        XA4$ [         R                  " [        R                  R                  U5      [        U5      R                  UUS9nXA4$ )a  Load onnx model given pretrained model name and optional ONNX model path. If onnx_path is None,
the default onnx model from optimum will be used.

Args:
    model_id (str): pretrained model name or checkpoint path
    onnx_path (Optional[str], optional): path of onnx model to evaluate. Defaults to None.

Returns:
    model: ORTModel for the onnx model
    onnx_path: the path of onnx model
T)exportproviderr   r   z
model.onnxzModel is exported to onnx file:)	file_namer   r   )
r	   from_pretrainedospathjoinsave_pretrainedprintdirnamer   name)r   r   r   r   modelsave_onnx_dirs         r   load_onnx_modelr$   >   s     ,<<TH
 S(3m,GGLL=	/;  -<<GGOOI&9o**)	
     resultscsv_filenamec                    [        USSSS9 n/ SQn[        R                  " X#S9nUR                  5         U  H  nUR	                  U5        M     UR                  5         SSS5        [        SU 35        g! , (       d  f       N= f)	zOutput a CSV file with detail of each test results.

Args:
    results (List[Dict[str, Any]]): list of JSON results.
    csv_filename (str): path of output CSV file
a asciimodenewlineencoding)pretrained_model_namer   r   disable_fused_attention
batch_sizesequence_lengthr   exactf1totalHasAns_exact	HasAns_f1HasAns_total
best_exactbest_exact_threshbest_f1best_f1_threshtotal_time_in_secondssamples_per_secondlatency_in_seconds
fieldnamesNz&Detail results are saved to csv file: )opencsv
DictWriterwriteheaderwriterowflushr   )r&   r'   csv_filecolumn_names
csv_writerresults         r   output_detailsrM   b   s}     
lb7	Cx
. ^^HF
 F'  	; 
D> 
2<.
AB? 
D	Cs   AA77
Bmetric_namec           	      \   [        USSSS9 n/ SQn[        U  Vs1 s H  oUS   iM	     sn5      nUR                  5         [        U  Vs1 s H  oUS   iM	     sn5      nUR                  5         [        U  Vs1 s H  oUS   iM	     sn5      nUR                  5         / n	U H#  n
U H  nU	R                  S	U S
U
 35        M     M%     [        R
                  " X4U	-   S9nUR                  5         U H  n0 n0 nUR                  [        R                  U	S5      5        U  H  nUS   U:X  d  M  XR   (       d  M  UR                  5        VVs0 s H  u  nnUU;   d  M  UU_M     nnnU(       d  UR                  U5        US   nUS   n
S	U S
U
 3nUU	;   d  My  XR   UU'   M     U(       d  M  U	 H  nUR                  US5      UU'   M     UR                  U5        M     UR                  5         SSS5        [        SU SU 35        gs  snf s  snf s  snf s  snnf ! , (       d  f       N5= f)zOutput a CSV file with summary of a metric on combinations of batch_size and sequence_length.

Args:
    results (List[Dict[str, Any]]): list of JSON results.
    csv_filename (str): path of output CSV file
    metric_name (str): the metric to summarize
r)   r*   r+   r,   )r0   r   r   r1   r   r   r2   r3   b_srA   NzSummary results for z are saved to csv file: )rC   listsortappendrD   rE   rF   updatedictfromkeysitemsgetrG   rH   r   )r&   r'   rN   rI   header_namesrL   
model_listbatch_sizessequence_lengths	key_namesr3   r2   rK   r"   rowvalueskvheaderskeys                       r   output_summaryre      s1    
lb7	Cx
 WEW6+.WEF
wGwV<0wGH Qv(9!: QR	/O)
  1ZL?2C!DE *  0 ^^H	9QR
 EC FMM$--	267!+&%/F4G4G06T1!|BStq!tGT

7+!'!5J&,->&?Oj\O+<=Ci'&,&9s " s$C%zz#r2CH %##C(1  4 	i 
Dl 
 -El^
TU[ F H !R* UI 
D	Cs_   HH
 H	H
 H7H
B#H,H7H
HH"1HH+AHH
H+c            	      l   [        5       n [        U 5        S H&  n[        U5      nU(       d  M  [        U S3U5        M(     U R                  nU R                  (       aF  [
        R                  R                  U R                  5      (       d  [        SU R                   35      e[
        R                  R                  SS5      S:H  n/ n[        R                  " U5      nU R                   GH  nXvl        [        US-  S5      Ul        U R                  c  [        S	5        ["        R"                  " 5       n[%        X0R                  U R&                  U R(                  5      u  p["        R"                  " 5       U-
  n[        S
US S35        [        U	R*                  5        XyR*                  R,                  :  a  [        S5      e[/        SXSU R0                  S9n[3        S5      n[        S5        ["        R"                  " 5       n[5        SU R6                  S:  a  SU R6                   S3OSS9n["        R"                  " 5       U-
  n[        SUS S35        [        S5        ["        R"                  " 5       nUR9                  UUSSS9n["        R"                  " 5       U-
  n[        SUS S35        U R&                  US'   XOS'   X?S'   XS '   U R0                  US!'   XS"'   U R(                  US#'   [        U5        UR;                  U5        GM"     [=        US$5        S% H  n[?        UU S&3U5        M     g )'N)zonnxruntime-gpuonnxruntimeonnxtorchtransformersoptimumdatasetsevaluatez versionz Onnx model path does not exist: ORT_DISABLE_FUSED_ATTENTION01      z4Exporting onnx model. It might take a few minutes...z!Onnx model exported or loaded in z.1fz secondszTsequence length should not be larger than {ort_model.config.max_position_embeddings}zquestion-answeringT)r"   	tokenizerquestion_firstr2   zLoading dataset...squadr   zvalidation[:]
validation)splitzDataset loaded in z<Evaluating squad_v2 with ORT. It might take a few minutes...squad_v2)model_or_pipelinedatametricsquad_v2_formatzEvaluation done in r   r1   r0   r   r2   r3   r   z
detail.csv)r5   r4   r?   z.csv) parse_argumentsr   r   
model_namerh   r   r   existsRuntimeErrorenvironrY   r   r   r]   model_max_lengthmin
doc_stridetimer$   r   r   configmax_position_embeddingsr   r2   r   r   r6   computerT   rM   re   )argsr!   package_versionr0   r1   all_resultsrs   r3   
start_time	ort_modelr   latencyqa_pipelinetask_evaluatorsquad_datasetrL   rN   s                    r   mainr      s   D	$Kv-d3?TF(#_5 w
 !OOyy		22=dii[IJJ jjnn-JCPTWWK--.CDI00%4""?a#7=	99HIYY[
./DiiQUQ^Q^`d`s`st	))+
*1'#hGHi--EEEuvv 	W[hlhwhw
 ##78"#YY[
$WTXT^T^abTbl4::,a4Phtu))+
*"73-x89LMYY[
'') 	 ( 
 ))+
*#GC=9:!]]z,C()*?&''{#|$3 !#'#6#6 f6"[ 1^ ;-<{{m4$8+F =r%   c           	         [         R                  " 5       nUR                  SSS[        [        S   S[         3S9  UR                  SSS	[
        S
/SS9  UR                  SS[
        SSS9  UR                  SS[
        SSS9  UR                  SS[        S SS9  UR                  SSSSS9  UR                  SSSSS9  UR                  SS9  UR                  U 5      nU$ ) Nz-mz--model_nameFr   z=Checkpoint directory or pre-trained model names in the list: )requiredtypedefaulthelpz-sz--sequence_lengths+i  zFSequence lengths for onnx model inputs. It could have multiple values.)nargsr   r   r   z-bz--batch_size   zbatch size for inference.)r   r   r   z-tz--totalz+Total samples to test. 0 means all samples.z--onnxzbOptional onnx model path. If not specified, optimum will be used to export onnx model for testing.z
--providerCUDAExecutionProviderzRSelect which Execution Provider to use for runs. Default is CUDAExecutionProvider.)r   r   r   z--use_io_binding
store_truezUse IO Binding for GPU.)r   actionr   )r   )argparseArgumentParseradd_argumentstrPRETRAINED_SQUAD_MODELSintset_defaults
parse_args)argvparserr   s      r   r~   r~     s0   $$&F
'*LMdLef   U   (   ic1Cpq
q   'a	   *U<Vop
u-T"DKr%   __main__)Nr   Fr   )(r   rD   r   r   importlib.metadatar   r   ImportErrorimportlib_metadatapathlibr   typingr   rl   r   rm   r   optimum.onnxruntimer	   optimum.versionr
   optimum_version	packagingversion_checkrj   r   r   parser   r   r   boolr$   rR   rV   rM   re   r   r~   __name__ r%   r   <module>r      s5  2  
 	 A@   !  < : . 0'-*=*=h*GG
I/IZZ[\
]] c  kp!!!Dj!cg!H&CDc3h0 &C &CR>VDc3h0 >V >VRU >VBCGL3l zF Q	  A@@As   C C&%C&