U
    h)                     @   s  d dl Z d dlZd dlZzd dlmZmZ W n$ ek
rP   d dlmZmZ Y nX d dlm	Z	 d dl
mZmZmZmZ d dlZd dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZmZ e ee dk rede ddddgZ!e"dddZ#d%e"ee" e$dddZ%eee"ef  e"dddZ&eee"ef  e"e"dddZ'd d! Z(d&d"d#Z)e*d$kre(  dS )'    N)PackageNotFoundErrorversion)Path)AnyDictListOptional)load_dataset)	evaluator)ORTModelForQuestionAnswering)ORTModel)__version__)r   )AutoTokenizerpipelinez1.6.0z+Please install optimum>=1.6.0. The version z was found.z5bert-large-uncased-whole-word-masking-finetuned-squadzdeepset/roberta-base-squad2z%distilbert-base-cased-distilled-squadpackage_namec                 C   s&   z
t | W S  tk
r    Y d S X d S )N)r   r   r    r   S/tmp/pip-unpacked-wheel-socb9apf/onnxruntime/transformers/models/bert/eval_squad.pyget_package_version.   s    
r   CUDAExecutionProviderF)model_id	onnx_pathuse_io_bindingc                 C   s   t j| dd}|dk	r`t|j|_|dkrFtd|_t|||_	qtd|_t||_	n(t
j|j |j}|dkr|d ||_||fS )a  Load onnx model given pretrained model name and optional ONNX model path. If onnx_path is None,
    the default onnx model from optimum will be used.

    Args:
        model_id (str): pretrained model name or checkpoint path
        onnx_path (Optional[str], optional): path of onnx model to evaluate. Defaults to None.

    Returns:
        model: ORTModel for the onnx model
        onnx_path: the path of onnx model
    T)Zfrom_transformersNZCPUExecutionProviderzcuda:0cpuZcuda)r   from_pretrainedr   name
model_nametorchZdevicer   Z
load_modelmodelospathjoinZmodel_save_diras_posixtor   )r   r   providerr   r   r   r   r   load_onnx_model5   s    
r%   )resultscsv_filenamec                 C   s   t |ddddd}ddddd	d
ddddddddddddddg}tj||d}|  | D ]}|| qX|  W 5 Q R X td|  dS )zOutput a CSV file with detail of each test results.

    Args:
        results (List[Dict[str, Any]]): list of JSON results.
        csv_filename (str): path of output CSV file
    a asciimodenewlineencodingpretrained_model_namer   r$   disable_fused_attention
batch_sizesequence_lengthr   exactf1totalZHasAns_exactZ	HasAns_f1ZHasAns_totalZ
best_exactZbest_exact_threshZbest_f1Zbest_f1_threshZtotal_time_in_secondssamples_per_secondZlatency_in_seconds
fieldnamesz&Detail results are saved to csv file: N)opencsv
DictWriterwriteheaderwriterowflushprint)r&   r'   csv_fileZcolumn_names
csv_writerresultr   r   r   output_detailsX   s8    rC   )r&   r'   metric_namec              	      s  t |dddd}ddddd	g td
d | D }|  tdd | D }|  tdd | D }|  g }|D ]$}|D ]}	|d|	 d|  qqxtj| | d}
|
  |D ]}i }i }|dd |D  | D ]r}|d |kr|| r fdd| D }|s|| |d }	|d }d|	 d| }||kr|| ||< q|r|D ] }||krr|| nd||< q\|
	| q|
  W 5 Q R X td| d|  dS )a  Output a CSV file with summary of a metric on combinations of batch_size and sequence_length.

    Args:
        results (List[Dict[str, Any]]): list of JSON results.
        csv_filename (str): path of output CSV file
        metric_name (str): the metric to summarize
    r(   r)   r*   r+   r/   r   r$   r0   r   c                 S   s   h | ]}|d  qS )r   r   .0rB   r   r   r   	<setcomp>   s     z!output_summary.<locals>.<setcomp>c                 S   s   h | ]}|d  qS )r1   r   rE   r   r   r   rG      s     c                 S   s   h | ]}|d  qS )r2   r   rE   r   r   r   rG      s     bZ_sr7   c                 S   s   i | ]
}|d qS )r)   r   )rF   kr   r   r   
<dictcomp>   s      z"output_summary.<locals>.<dictcomp>c                    s   i | ]\}}| kr||qS r   r   )rF   rI   vheader_namesr   r   rJ      s       r1   r2   zSummary results for z are saved to csv file: N)r9   listsortappendr:   r;   r<   updateitemsr=   r>   r?   )r&   r'   rD   r@   Z
model_listZbatch_sizessequence_lengthsZ	key_namesr2   r1   rA   r   rowvaluesrB   headerskeyr   rL   r   output_summary   sN    
rX   c                  C   s  t  } t|  dD ] }t|}|rt| d| q| j}| jr^tj| js^td| j tj	
dddk}g }t|}| jD ]}||_t|d d|_t|| j| j| j\}}	t|j ||jjkrtd	td
||d| jd}
td
}td| jdkrd| j dndd}|j|
|ddd}| j|d< ||d< ||d< |	|d< | j|d< ||d< | j|d< t| || qt|d dD ]}t|| d| qd S )N)zonnxruntime-gpuZonnxruntimeonnxr   transformersZoptimumdatasetsevaluatez versionz Onnx model path does not exist: ZORT_DISABLE_FUSED_ATTENTION01      zTsequence length should not be larger than {ort_model.config.max_position_embeddings}zquestion-answeringT)r   	tokenizerZquestion_firstr1   Zsquadr   zvalidation[:]Z
validation)splitZsquad_v2)Zmodel_or_pipelinedataZmetricZsquad_v2_formatr$   r0   r/   r   r1   r2   r   z
detail.csv)r4   r3   r6   z.csv)parse_argumentsr?   r   r   rY   r   r    existsRuntimeErrorenvirongetr   r   rS   Zmodel_max_lengthminZ
doc_strider%   r$   r   configZmax_position_embeddingsr   r1   r
   r	   r5   ZcomputerP   rC   rX   )argsr   Zpackage_versionr/   r0   Zall_resultsra   r2   Z	ort_modelr   Zqa_pipelineZtask_evaluatorZsquad_datasetrB   rD   r   r   r   main   s\    


    &



rm   c                 C   s   t  }|jdddttd dt d |jddd	td
gdd |jddtddd |jddtddd |jddtd dd |jddddd |jddddd |jdd || }|S ) Nz-mz--model_nameFr   z=Checkpoint directory or pre-trained model names in the list: )requiredtypedefaulthelpz-sz--sequence_lengths+i  zFSequence lengths for onnx model inputs. It could have multiple values.)nargsro   rp   rq   z-bz--batch_size   zbatch size for inference.)ro   rp   rq   z-tz--totalz+Total samples to test. 0 means all samples.z--onnxzbOptional onnx model path. If not specified, optimum will be used to export onnx model for testing.z
--providerr   zRSelect which Execution Provider to use for runs. Default is CUDAExecutionProvider.)rn   rp   rq   z--use_io_binding
store_truezUse IO Binding for GPU.)rn   actionrq   )r   )argparseArgumentParseradd_argumentstrPRETRAINED_SQUAD_MODELSintset_defaults
parse_args)argvparserrl   r   r   r   re      sT    		
re   __main__)Nr   F)N)+rw   r:   r   importlib.metadatar   r   ImportErrorZimportlib_metadatapathlibr   typingr   r   r   r   r   r[   r	   r\   r
   Zoptimum.onnxruntimer   Z optimum.onnxruntime.modeling_ortr   Zoptimum.versionr   Zoptimum_version	packagingZversion_checkrZ   r   r   parser{   rz   r   boolr%   rC   rX   rm   re   __name__r   r   r   r   <module>   sJ          #)A9
6
