U
    qhS                     @  s$  d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlmZ d dlmZmZmZmZmZ d dlmZ d dlZd dlZd dlZd dlmZmZmZmZmZmZ d dlm Z  d dl!m"Z" d d	l#m$Z$ d d
l%m&Z& G dd de
j'Z(dddddZ)G dd dZ*dS )    )annotationsN)defaultdict)AnyCallableIterablePatternSequence)urlretrieve)ONNX_ML
ModelProto	NodeProto	TypeProtoValueInfoProtonumpy_helper)Backend)TestCase)load_model_tests)TestItemc                   @  s   e Zd ZdS )!BackendIsNotSupposedToImplementItN)__name__
__module____qualname__ r   r   E/tmp/pip-unpacked-wheel-xnis5xre/onnx/backend/test/runner/__init__.pyr      s   r   intz2Callable[[Callable[..., Any]], Callable[..., Any]])timesreturnc                   s$    dkst ddd fdd}|S )N   Callable[..., Any])funcr   c                   s&   t  dddd fdd}|S )Nr   )argskwargsr   c               	     sd   t dd D ]P}z | |W   S  tk
r\   t| d |krJ td|  Y qX qd S )Nr   z times tried   )range	Exceptionprinttimesleep)r    r!   i)r   r   r   r   wrapped'   s    z/retry_execute.<locals>.wrapper.<locals>.wrapped)	functoolswraps)r   r)   r   )r   r   wrapper&   s    
zretry_execute.<locals>.wrapper)AssertionError)r   r-   r   r,   r   retry_execute#   s    r/   c                	   @  sh  e Zd ZdJdddddddZd	d
dddZd	d dddZd	d dddZd	d dddZd dddZe	ddddZ
e	ddddZe	dddd Ze	d
dd!d"ZedKd#d#d$d$ddd%d&d'Zeed(d)d	d	dd*d+d,Zed)d	d-d.d/ZdLd	d	d1d2d3d4dd5d6d7ZedMd;d<d	d=d>d?d@dAZd)d	ddBdCdDZd	dEdFddGdHdIZdS )NRunnerNztype[Backend]z
str | Nonezdict | NoneNone)backendparent_moduletest_kwargsr   c                 C  s   || _ || _t | _t | _t | _|p*i | _tt| _	t
ddD ]}| |d qBt
ddD ]}| |d q^t
ddD ]}| |d qzt
ddD ]}| |d	 qt
d
dD ]}| |d qd S )Nnode)kindNoderealRealsimpleZSimplezpytorch-convertedZPyTorchConvertedzpytorch-operatorZPyTorchOperator)r2   _parent_moduleset_include_patterns_exclude_patterns_xfail_patterns_test_kwargsr   dict_test_itemsr   _add_model_test)selfr2   r3   r4   rtctZotr   r   r   __init__8   s"    

zRunner.__init__strztype[unittest.TestCase])namer   c                 C  s&   t t|tjfi }| jr"| j|_|S N)typerH   unittestr   r;   r   )rD   rI   	test_caser   r   r   _get_test_caseZ   s    zRunner._get_test_case)patternr   c                 C  s   | j t| | S rJ   )r=   addrecompilerD   rO   r   r   r   include`   s    zRunner.includec                 C  s   | j t| | S rJ   )r>   rP   rQ   rR   rS   r   r   r   excluded   s    zRunner.excludec                 C  s   | j t| | S rJ   )r?   rP   rQ   rR   rS   r   r   r   xfailh   s    zRunner.xfail)r   c                 C  sF   dd l }| j D ].\}}| D ]}|j|j||j|_q"q| S )Nr   )pytestrB   itemsvaluesmarkZonnx_coverageprotor   )rD   rW   category	items_mapitemr   r   r   enable_reportl   s
    zRunner.enable_reportzdict[str, dict[str, TestItem]]c                   s   i }| j  D ]\}}i ||< | D ]\ }| jr^t fdd| jD s^td|j|_| jD ]*}| rdtd|j	 d|j|_qd| j
D ]}| rt|j|_q|||  < q&q|S )Nc                 3  s   | ]}|  V  qd S rJ   )search).0rT   rI   r   r   	<genexpr>{   s     z.Runner._filtered_test_items.<locals>.<genexpr>zno matched include patternzmatched exclude pattern "")rB   rX   r=   anyrL   skipr   r>   r`   rO   r?   ZexpectedFailure)rD   filteredr\   r]   r^   rU   rV   r   rb   r   _filtered_test_itemst   s(    



zRunner._filtered_test_itemsz"dict[str, type[unittest.TestCase]]c                 C  s^   i }| j  D ]J\}}d| d}| |}t| D ]\}}t|||j q8|||< q|S )zList of test cases to be applied on the parent scope
        Example usage:
            globals().update(BackendTest(backend).test_cases)
        ZOnnxBackendZTest)rh   rX   rN   sortedsetattrr   )rD   
test_casesr\   r]   Ztest_case_namerM   rI   r^   r   r   r   rk      s    

zRunner.test_caseszunittest.TestSuitec                 C  s:   t  }t| j dd dD ]}|t j| q|S )zTestSuite that can be run by TestRunner
        Example usage:
            unittest.TextTestRunner().run(BackendTest(backend).test_suite)
        c                 S  s   | j jS rJ   	__class__r   Zclr   r   r   <lambda>       z#Runner.test_suite.<locals>.<lambda>key)rL   Z	TestSuiteri   rk   rY   ZaddTestsZdefaultTestLoaderZloadTestsFromTestCase)rD   ZsuiteZcaser   r   r   
test_suite   s     
zRunner.test_suitec                 C  sN   |  d}t| j dd dD ](}t| D ]\}}t|||j q0q |S )zOne single unittest.TestCase that hosts all the test functions
        Example usage:
            onnx_backend_tests = BackendTest(backend).tests
        ZOnnxBackendTestc                 S  s   | j jS rJ   rl   rn   r   r   r   ro      rp   zRunner.tests.<locals>.<lambda>rq   )rN   ri   rh   rY   rX   rj   r   )rD   testsr]   rI   r^   r   r   r   rt      s    
 
zRunner.testszSequence[Any]float)ref_outputsoutputsrtolatol	model_dirr   c           	      C  s  zt jt|t| W nL tk
rf } z.tdt| dt| d|pJdd|W 5 d }~X Y nX tt|D ]D}t|| tt	frt|| tt	fst
dt||  d| dt||  d|pdd		tt|| D ](}| j|| | || | |||d
 qqtt || jt js\||  ||  krtt
||  d||  qtt j|| j|| j || jtkrt j|| ||  qtt jj|| || ||d qtd S )Nz Unable to compare expected type z and runtime type z (known test=?)zUnexpected type z for outputs[z]. Expected type is z).)rz   z != )rx   ry   )npZtestingZassert_equallen	TypeErrorrK   r#   
isinstancelisttupler.   assert_similar_outputsZ
issubdtypeZdtypenumbertolistobjectZassert_array_equalZassert_allclose)	clsrv   rw   rx   ry   rz   er(   jr   r   r   r      sF    	"0


   zRunner.assert_similar_outputs   r   )
model_testrz   
models_dirr   c              
   C  s   ~t jdd}zzT|  |js$ttd|j	 d|j  t
|j|j td tj|j| W n: tk
r } ztd|j	 d|   W 5 d }~X Y nX W 5 t|j X d S )NF)deletezStart downloading model z from Donez!Failed to prepare data for model z: )tempfileNamedTemporaryFileosremoverI   closeurlr.   r%   
model_namer	   onnxutilsZ_extract_model_safer$   )r   r   rz   r   Zdownload_filer   r   r   r   download_model   s    

zRunner.download_model)r   r   c              	   C  s   t jt dt jdd}t dt j|d}t j||j}t jt j|dst j|rd}| d| }t j|r|d	7 }qjt|| qqjt 	| | j
|||d
 |S )N	ONNX_HOME~.onnxONNX_MODELSmodels
model.onnxr   z.old.r   )r   rz   r   )r   path
expandusergetenvjoinr   existsshutilmovemakedirsr   )r   r   	onnx_homer   rz   Zbidestr   r   r   prepare_model_data   s*    
  zRunner.prepare_model_dataZCPUZCUDAr   z#list[ModelProto | NodeProto | None]zIterable[str]r   )r\   	test_name	test_funcreport_itemdevicesr!   r   c           	        sL    dstd ddd fdd}|D ]}|| q:d S )NZtest_z!Test name must start with test_: rH   r1   )devicer   c                   s    d    j kr6td d dtj  d  tdddd fdd	}t	|j < d S )
N_zDuplicated test name "z" in category "rd   zBackend doesn't support device r   )r    device_test_kwargr   c               
     sj   z|}|  f|W S  t k
rd } z,dtjks@dtjkrTtd d|  W 5 d }~X Y nX d S )Nz-vz	--verbosezTest z is effectively skipped: )r   sysargvr%   )r    r   Zmerged_kwargsr   )r   device_test_namer!   r   r   r   device_test_func'  s    zCRunner._add_test.<locals>.add_device_test.<locals>.device_test_func)
lowerrB   
ValueErrorrL   ZskipIfr2   Zsupports_devicer*   r+   r   )r   r   r\   r!   r   rD   r   r   )r   r   r   add_device_test   s     	 z)Runner._add_test.<locals>.add_device_test)
startswithr   )	rD   r\   r   r   r   r   r!   r   r   r   r   r   	_add_test  s
    

zRunner._add_testr    Fr   r   boolz
np.ndarray)xseedrI   randomr   c                 C  s   | j jstd|d|  d| j jjdkrBtd|d|  dtdd | j jjjD }|r|tjj	|d}||tj
S t|}t||| tj
S )	z8Generates a random tensor based on the input definition.zMInput expected to have tensor type. Unable to generate random data for model z and input .r   zMCurrently limited to float tensors. Unable to generate random data for model c                 s  s"   | ]}| d r|jndV  qdS )	dim_valuer   N)HasFieldr   )ra   dr   r   r   rc   K  s   z-Runner.generate_dummy_data.<locals>.<genexpr>)r   )rK   tensor_typeNotImplementedErrorZ	elem_typer   shapeZdimr}   r   Zdefault_rngZfloat32prodZarangeZreshapeZastype)r   r   rI   r   r   gennr   r   r   generate_dummy_data<  s     

zRunner.generate_dummy_data)r   r6   r   c                   sj   d g dddd fdd}j jkrPj|d j | fjj   n|d j |  d S )Nr   rH   r1   )	test_selfr   r   c           !   
     s  j d k	rj drtjtjtjtddddj }tj|sZt	d|dtj
tdtjdd}tdtj|d	d
}tj|j}tj|st| d}n.jd kr҈}nj}tj|d}d}tsd|krd S t|}|d< tjdr@tjjr@j|s@tdjj||f|}	|	d k	s`t|rt|d}
t|
}W 5 Q R X tj|d}tj|st| i }dd |jjD }d}g }tt |jj!D ]}|jj!| j"|krqtj|d| d}|#| |d7 }|jj!|  j$ djdd}|| j"< t|d}
|
%tj&'|(  W 5 Q R X qtj)|d }g }tt |jj*D ]<}| d| d}tj|r|#| qd } qڐq|d krTtj+,|}|-d |}t.|D ]L\}}tj|d| d}t|d}
|
%tj&'|(  W 5 Q R X qn6t.|D ],\}}tj|d| d}t/0|| q\nt11tj|dD ]p}t2j|dd}t3|d  }t3|	-|}t4 fd!d"|d# D }j5|||6d$j7|6d%j8|d& qt11tj|d'D ]}g }t t11tj|d(}t|D ]4}tj|d| d}9|||jj!| j: qNg }t t11tj|d)}t|D ]4}tj|d| d} 9| ||jj*| j: qt3|	-|}j5|||6d$j7|6d%j8|d& q&d S )*Nzonnx/backend/test/data/light/z..zUnable to find model r   r   r   r   r   r   lightTr   FZ
ai_onnx_mlr   is_compatiblezNot compatible with backendrbZtest_data_set_0c                 S  s   h | ]
}|j qS r   rb   )ra   r(   r   r   r   	<setcomp>  s     z6Runner._add_model_test.<locals>.run.<locals>.<setcomp>Zinput_z.pbr   )r   rI   r   wbZ_output_Zoutput_ztest_data_*.npzbytes)encodinginputsc                 3  s*   | ]"}t  ttfst n V  qd S rJ   )r   r   rA   r}   array)ra   fr   r   r   rc     s   z6Runner._add_model_test.<locals>.run.<locals>.<genexpr>rw   rx   ry   )rx   ry   rz   ztest_data_set*z
input_*.pbzoutput_*.pb);r   r   r   r   normpathr   dirname__file__r   FileNotFoundErrorr   r   r   r   rz   r   r
   r   loadhasattrr2   callabler   rL   SkipTestpreparer.   openmkdirgraphZinitializerr#   r~   inputrI   appendr   writer   Z
from_arrayZSerializeToStringsplitextoutput	referenceZReferenceEvaluatorrun	enumerater   copyglobr}   r   r   r   getrx   ry   _load_protorK   )!r   r   r!   Zmodel_pb_pathr   r   rz   Z	use_dummymodelZprepared_modelr   ZonxZtest_data_setZfeedsZinitsZn_inputr   r(   rI   valueprefixZexpected_outputsrefrw   oZtest_data_npzZ	test_datarv   Ztest_data_dirZ
inputs_numZ
input_fileZref_outputs_numZoutput_fileZmodel_markerr   rD   r   r   r   Z  s   

 








   
$


&
  z#Runner._add_model_test.<locals>.runZModel)rI   r@   r   )rD   r   r6   r   r   r   r   rC   U  s     
zRunner._add_model_testzlist[np.ndarray | list[Any]]r   )proto_filenametarget_listmodel_type_protor   c           
   	   C  s   t |d}| }|drBt }|| |t| nx|drt	 }|| t
|}t|tjsxt|| n6|drt }	|	| |t|	 ntd W 5 Q R X d S )Nr   Zsequence_typer   Zoptional_typezRLoading proto of that specific type (Map/Sparse Tensor) is currently not supported)r   readr   r   ZSequenceProtoZParseFromStringr   r   Zto_listZTensorProtoZto_arrayr   r}   Zndarrayr.   ZOptionalProtoZto_optionalr%   )
rD   r   r   r   r   Zprotobuf_contentsequenceZtensortoptionalr   r   r   r     s&    






zRunner._load_proto)NN)N)r   )r   r   F)r   r   r   rG   rN   rT   rU   rV   r_   propertyrh   rk   rs   rt   classmethodr   r/   r   r   r   staticmethodr   rC   r   r   r   r   r   r0   7   sD     " + )      $r0   )+
__future__r   r*   r   r   rQ   r   r   r   r&   rL   collectionsr   typingr   r   r   r   r   urllib.requestr	   Znumpyr}   r   Zonnx.referencer
   r   r   r   r   r   Zonnx.backend.baser   Z onnx.backend.test.case.test_caser   Zonnx.backend.test.loaderr   Zonnx.backend.test.runner.itemr   r   r   r/   r0   r   r   r   r   <module>   s.    