
    hq                     R   S r SSKrSSKrSSKrSSKrSSKJr  SSKrSSKrSSKJ	r
  SSKJr  SSKJrJrJrJrJr  SSKJr  SSKJr  SSKJrJrJr  SS	KJr  SS
KJr  SSK J!r!  \"" \" SS5      5      r#\"" \" SS5      5      r$Sr%1 Skr&\RN                  " S5      (       a  \&1 Sk-  r&\RP                  S:X  a  \&1 Sk-  r&\RN                  " S5      (       a  \&1 Sk-  r&S r) " S S5      r*S S jr+ " S S\RX                  5      r-\-R]                  5         \/S:X  a  \R`                  " SS9  gg)!aS  
These test evaluates the python runtime (class ReferenceEvaluator) against
all the backend tests (in onnx/backend/test/case/node) and checks
the runtime produces the expected outputs.

You may run one specific test with following command line:

::

    python onnx/test/reference_evaluator_backend_test.py TestOnnxBackEndWithReferenceEvaluator.test_group_normalization_example

You may bypass a test newly added by adding to the global variable `SKIP_TESTS`.
You may refine the absolute or relative tolerance for a test by
adding an item in method `setUpClass` and attributes
`atol` or `rtol`.
    N)getenv)object_)assert_allclose)ONNX_MLOptionalProtoSequenceProtoTensorProtoload)__file__)bfloat16_to_float32to_listto_optionalReferenceEvaluator)to_array_extended)cast_toORT_MAX_IR_SUPPORTED_VERSION8$ORT_MAX_ONNX_OPSET_SUPPORTED_VERSION18i  >   test_scan_sumtest_scatter_with_axistest_scatter_without_axistest__simple_gradient_of_add$test__simple_gradient_of_add_and_mul+test_center_crop_pad_crop_axes_hwc_expandedz1.21.5>   test_cast_FLOAT_to_BFLOAT16test_castlike_FLOAT_to_BFLOAT16(test_castlike_FLOAT_to_BFLOAT16_expandedwin32>   test_regex_full_match_basictest_regex_full_match_empty!test_image_decoder_decode_bmp_rgb!test_image_decoder_decode_png_rgb!test_image_decoder_decode_pnm_rgb"test_image_decoder_decode_jpeg_bgr"test_image_decoder_decode_jpeg_rgb"test_image_decoder_decode_tiff_rgb"test_image_decoder_decode_webp_rgb"test_regex_full_match_email_domain$test_image_decoder_decode_jpeg2k_rgb(test_image_decoder_decode_jpeg_grayscale>	   test_dft	test_stfttest_dft_axistest_stft_cputest_dft_inversetest_dft_opset19test_dft_axis_opset19test_stft_with_windowtest_dft_inverse_opset19c                 T   S n[        [        X R                  5       5      5      (       aJ  U R                  [        R
                  5      nUR                  [        R
                  5      n[        X45        gU R                  5       UR                  5       :w  a  [        SU  SU S35      eg)z
Compares two arrays knowing they contain strings.
Raises an exception if the test fails.

:param expected: expected array
:param value: value
c                 <     [        U 5        g! [         a     gf = f)NTF)float
ValueError)xs    d/var/www/fran/franai/venv/lib/python3.13/site-packages/onnx/test/reference_evaluator_backend_test.pyis_float(assert_allclose_string.<locals>.is_floatq   s#    	!H 		s    
zMismatches  != .N)	allmapravelastypenpfloat32r   tolistAssertionError)expectedvaluer<   expected_floatvalue_floats        r;   assert_allclose_stringrL   h   s     3x)*++!4ll2::.4??. ;xjUG1!EFF /    c                       \ rS rSrSr\S 5       r\S 5       r\S 5       r\S 5       r	S r
S r\S	 5       r\S
 5       rS r SS jrS r     SS jrSrg)OnnxBackendTest   a  
Definition of a backend test. It starts with a folder,
in this folder, one onnx file must be there, then a subfolder
for each test to run with this model.

:param folder: test folder
:param onnx_path: onnx file
:param onnx_model: loaded onnx file
:param tests: list of test
c                    / nU  HU  n[         R                  R                  U5      S   nUR                  S5      S   nUR	                  [        U5      U45        MW     UR                  5         U Vs/ s H  oUS   PM	     sn$ s  snf )Nr   _   )ospathsplitextsplitappendintsort)	filenamestempfnameirR   s         r;   _sortOnnxBackendTest._sort   su    A77##A&q)D

3#AKKQ$  			"#d!d###s   2Bc                    [         R                  R                  U 5      (       d  [        SU < S35      e[	        U S5       nUR                  5       nS S S 5        [        R                  WU 5      $ ! , (       d  f       N$= f)NFile not found: r?   rb)rU   rV   existsFileNotFoundErroropenreadrO   _read_proto_from_serialized)fullr^   
serializeds      r;   _read_proto_from_file%OnnxBackendTest._read_proto_from_file   sb    ww~~d###&6tha$@AA$J :::tLL s   A00
A>c           
         [         R                  R                  U5      (       d  [        SU< S35      e[	        US5       nUR                  5       n S S S 5        [        [        4[        [        4[        [        4/nS nU H'  u  pVU" 5       n UR                  U 5         U" U5      s  $    [        SU< SU SU S S < S35      Ue! , (       d  f       N{= f! [         a  nUn S nAMi  S nAff = f! [         a  nUn S nAM  S nAff = f)Nrd   r?   re   zUnable to read z, error is z, content is d   )rU   rV   rf   rg   rh   ri   r	   r   r   r   r   r   ParseFromStringr9   	ExceptionRuntimeError)	rl   rk   r^   proto_typesexcptcvtobjes	            r;   rj   +OnnxBackendTest._read_proto_from_serialized   s   ww~~d###&6tha$@AA$J  +,G$K(

 "GB$C##J/s8O # dX[ 6$Tc*-Q0
 	'  " C  sH   CC2 C
C
C/"C*$C2*C//C22
D	<DD	c                     / nU HH  n[         R                  R                  X5      n[        R	                  U5      nUR                  U5        MJ     U$ N)rU   rV   joinrO   rm   rY   )foldernamesresr_   rk   rx   s         r;   _loadOnnxBackendTest._load   sE    D77<<-D!77=CJJsO  
rM   c                 P    U R                   R                   SU R                  < S3$ )usual())	__class____name__r~   selfs    r;   __repr__OnnxBackendTest.__repr__   s$    ..))*!DKK?!<<rM   c                 6   [         R                  R                  U5      (       d  [        SU< S35      e[         R                  " U5      nU Vs/ s H-  n[         R                  R                  U5      S   S;   d  M+  UPM/     nn[        U5      S:w  a  [        SU< SU< S35      eXl        [         R                  R                  XS	   5      U l
        [        U R                  5      U l        / U l        U GH  n[         R                  R                  X5      n[         R                  R                  U5      (       d  MI  [         R                  " U5       Vs/ s H-  n[         R                  R                  U5      S   S
;   d  M+  UPM/     nn[        R!                  S U 5       5      n[        R!                  S U 5       5      n	U R                  R#                  [        R%                  Xh5      [        R%                  Xi5      S.5        GM     g s  snf s  snf )NzUnable to find folder r?   rS      .onnxrT   z$There is more than one onnx file in z ().r   >   .pbc              3   T   #    U  H  oR                  S 5      (       d  M  Uv   M      g7f)input_N
startswith.0cs     r;   	<genexpr>+OnnxBackendTest.__init__.<locals>.<genexpr>   s     .W"QX@Vqq"   (	(c              3   T   #    U  H  oR                  S 5      (       d  M  Uv   M      g7f)output_Nr   r   s     r;   r   r      s      0!!\\)%<AArr   )inputsoutputs)rU   rV   rf   rg   listdirrW   lenr9   r~   r}   	onnx_pathr
   
onnx_modeltestsisdirrO   ra   rY   r   )
r   r~   contentr   onxsubrk   pbr   r   s
             r;   __init__OnnxBackendTest.__init__   s   ww~~f%%#&<VJa$HII**V$!J'QRWW%5%5a%8%<	%Iq'Js8q=6vj3'L  f!f5t~~.
C77<<,Dww}}T""!#D!1X!1ARWW5E5Ea5H5LPW5Wa!1X(...W".WW)// 0!0  

!!"1"7"7"E#2#8#8#G  K Ys   *H=H*HHc                 Z    [         R                  R                  U R                  5      S   $ )zReturns the test name.rS   )rU   rV   rX   r~   r   s    r;   r_   OnnxBackendTest.name   s      ww}}T[[)"--rM   c           	      b   U R                   R                  SS5      R                  S5      S   nUR                  S5      (       a  U R                  nU$ SUR                  SS5       SU R                  SS   3nSU;   d  US	:X  a+  [        S
U R                  < SU< SU R                    S35      eU$ )N\/nodetest__-rR      test__test_AvgPool1d_AvgPool1dzname=z	, folder=z, self.folder=r?   )r~   replacerX   endswithr_   rG   )r   r~   fnames      r;   r   OnnxBackendTest.fname   s    $$T3/55c:2>??6""IIE  V^^C56a		!"GEe|u(HH$DII=	&>$++VWX  rM   c                 ,    [        U R                  5      $ )zReturns the number of tests.)r   r   r   s    r;   __len__OnnxBackendTest.__len__   s    4::rM   Nc	                    US:X  a  [        S5      eUc  SnUc  Sn[        U[        R                  5      (       Ga  [        U[        R                  5      (       Ga  US:X  aE  UR                  [        R
                  :X  a  Sn	O&UR                  [        R                  :X  a  Sn	OUn	OUn	UR                  [        :X  a   [        X45        OUR                  [        R                  [        R
                  [        R                  4;   nU(       a   [        UUUU	US9  OUR                  UR                  :w  a:  [        SU SU S	U R                  < SUR                   SUR                   S35      eUR!                  5       UR!                  5       :w  a&  [        SU SU S	U R                  < SU SU S35      eUR"                  UR"                  :w  aZ  [        SU SU S	U R                  < SUR"                   SUR"                   SU SU SU S[        R                  " U5       S35      eg[%        US5      (       a  UR                  UR                  :w  a5  [        SU SU S	U R                  < SUR                  < SU< SU S35      eUR'                  UR"                  5      (       d4  [        SU SU S	U R                  < SUR"                   SU< SU S35      egg[        U[(        5      (       a  [        U[(        5      (       d$  [        S[+        U5       SU S U S!U S"U S35      e[-        U5      [-        U5      :w  a0  [        S#[-        U5       S$[-        U5       SU S U S!U S"U S35      e[/        X45       H  u  pU R1                  XXXVUS%9  M     g[3        S&[+        U5       S'U S U S35      e! [         a)  n
[        SU SU S	U R                  < S
U S3	5      U
eSn
A
ff = f! [         ag  n
 XC-
  nO! [         a    Sn Of = f[        SU SU S	U R                  < SU	 SU SU SU SU SU S[        R                  " U5       S35      U
eSn
A
ff = f)(ay  
Compares the expected output and the output produced
by the runtime. Raises an exception if not equal.

:param index: test index
:param i_output: output index
:param desired: expected output
:param output: output
:param rtol: relative tolerance
:param atol: absolute tolerance
:param comment: addition text to give more insights to the user
:param inputs: inputs to the model
 z"Argument comment should be filled.Nr   h㈵>g-q=Output 	 of test  in folder z failed, comment=r?   )atolrtol	equal_nanz failed (rtol=z, atol=z), comment=z
---
z
----
z
-----
z
------INPUTS----
z has unexpected type z (expecting z.)z has unexpected values z failed (expected shape=z but shape=is_compatiblez failed (desired.dtype=z	, output=z failed (desired.shape=z-Expected result is 'list' but output type is z for output z
, comment=z
--EXPECTED--
z	
--GOT--
zExpected has z but output has )r   r   commentz$Comparison not implemented for type z and output )rs   
isinstancerD   ndarraydtyperE   float64dtype_objectrL   rG   r~   float16r   r9   pprintpformatrF   shapehasattrr   listtyper   zip_compare_resultsNotImplementedError)r   indexi_outputdesiredoutputr   r   r   r   rtlexr   diffabs                  r;   r    OnnxBackendTest._compare_results   s     b=CDD<D<Dgrzz**&"**--19}}

2" "**4#"C==L0".w? !("**bjj"**1U UI &+ ' &%)%(*3& #==FLL8"0")(9UG;t{{o ^77=||nLQXQ^Q^P__a!c#  #>>+v}}>"0")(9UG;t{{o ^99?WIUW!Y# 
 ==FLL0(!(9UG;t{{o V++2==/V\\N S##*)77) <#H$89O8PPQS  1 11==FLL0(!(9UG;t{{o V**1--):)F:[Y`Xaabd  ++GMM::(!(9UG;t{{o V**1--	&;W^V__`b  ; 2 &&fd++$CDL>Q]^f]g 	)9'+fXUVX  7|s6{*$#CL>1A#f+l[cZd 	)9'+fXUVX  G,%%Q &  -
 &6tG}o\RZQ[[efmennop S * ",%hZy{4;;/Yjkrjsstu!""  . 	&,'-'7#- ,'+,"0")(9UG;t{{o ^)),WTF+gYgV]U^ _%%+HIdV;OPVP^P^_ePfOggh!j# $&	&	&sO   >N> O4 >
O1$O,,O14
Q%?PQ PQ PAQ  Q%c                 &    SU R                   ;   a  gg)z!Tells if a test is random or not.	bernoulliTF)r~   r   s    r;   	is_randomOnnxBackendTest.is_randoml  s    $++%rM   c                    UcC  / n[        [        U 5      5       H'  n	UR                  U R                  UUU	UUUUS95        M)     U$ U(       a  [	        S5        [        U R                  U   S   5       H.  u  p[	        SU
< SUR                   SUR                   35        M0     [	        S5        [        U R                  U   S   5       H.  u  p[	        S	U
< SUR                   SUR                   35        M0     U" U R                  5      nU" U/U R                  U   S   Q76 nU R                  U   S   n[        U5      [        U5      :w  a5  [        S
U SU R                  < S[        U5       S[        U5       S3	5      eU R                  U   S   U R                  U   S   US.n[        [        X5      5       GH	  u  n	u  nnU R                  5       (       a  UR                  UR                  :w  a;  [        SU	 SU SU R                  < SUR                   SUR                  < S35      eUR                  UR                  :w  a:  [        SU	 SU SU R                  < SUR                   SUR                   S35      eM  U R                  UU	UUUUUS-   [        U R                  5      -   U R                  U   S   S9  GM     U$ )a  
Executes a tests or all tests if index is None.
The function crashes if the tests fails.

:param load_fct: loading function, takes a loaded onnx graph,
    and returns an object
:param run_fct: running function, takes the result of previous
    function, the inputs, and returns the outputs
:param index: index of the test to run or all.
:param rtol: relative tolerance
:param atol: absolute tolerance
:param comment: additional information for the user
:param print_io: prints out the input and output
)r   r   r   r   print_ioz------ INPUTSr   zinput z, shape=z, dtype=z------ EXPECTED OUTPUTSr   zoutput z"Unexpected number of output (test z	, folder z), got z, expected r?   )r   rH   resultsr   r   r   z failed (type mismatch r>   r   z failed (shape mismatch 
)r   r   r   r   )ranger   rY   runprint	enumerater   r   r   r   rG   r~   r   r   r   str)r   load_fctrun_fctr   r   r   r   r   r   r`   kvrx   gotrH   ry   os                    r;   r   OnnxBackendTest.runr  s   0 =C3t9%

HH !! '!)  
 & J/"!$**U"3H"=>qe8AGG9HQWWIFG ?+,!$**U"3I">?uHQWWIXaggYGH @ t'c8DJJu-h78::e$Y/s8s8}$ 4UG9T[[O T3xjCM?!= 
 jj'1

5))4

 #3x#56IAv1~~77agg%(!!IeWK O**+''$qwwkE  77agg%(!!IeWK O++,77)4yD  & %%#dNS-AA::e,X6 & 	 7. 
rM   )r~   r   r   r   )r   r   r   N)NHz>r   r   F)r   
__module____qualname____firstlineno____doc__staticmethodra   rm   rj   r   r   r   propertyr_   r   r   r   r   r   __static_attributes__ rM   r;   rO   rO      s    	 $ $ M M  6  =: . . 
 

 TXob UrM   rO   c              #     #    [         R                  R                  [        5      n[         R                  R	                  USU 5      n[         R                  R                  U5      (       d7  SR	                  [         R                  " U5      5      n[        SU< SU 35      e[         R                  " U5      nU H  nUb  U" U5      (       d  M  [         R                  R	                  X65      n[        (       d  SU;   a  MG  [         R                  " U5      nU Vs/ s H-  n[         R                  R                  U5      S   S;   d  M+  UPM/     n	n[        U	5      S	:X  d  M  [        U5      v   M     gs  snf 7f)
ao  
Collects test from a sub folder of `onnx/backend/test`.
Works as an enumerator to start processing them
without waiting or storing too much of them.

:param series: which subfolder to load, possible values:
    (`'node'`, ...)
:param fct_filter: function `lambda testname: boolean`
    to load or skip the test, None for all
:return: list of @see cl OnnxBackendTest
datar   z"Unable to find series of tests in z, subfolders:
N
ai_onnx_mlrS   r   rT   )rU   rV   dirnamebackend_folderr}   rf   r   rg   r   rW   r   rO   )
series
fct_filterrootr   r   r   tr~   r   r   s
             r;   enumerate_onnx_testsr    s     77??>*D
'',,tVV
,C77>>#))BJJt,-0yQ
 	
 JJsOE!*Q--c%w<61**V$!J'QRWW%5%5a%8%<	%Iq'Js8q=!&))  Ks   DE9*E4	E4E9"E9c                   @   \ rS rSr\R
                  R                  \R
                  R                  \R
                  R                  \	5      5      S5      r
\S 5       rS rS r\SS j5       r\SS.S	 j5       r     SS jr\S 5       r\SS j5       r\S 5       rSrg
)%TestOnnxBackEndWithReferenceEvaluatori  onnx_backend_test_codec                    ^  S H<  n[        U5       H*  nUS SS4U 4S jjn[        [        UR                  U5        M,     M>     g )N)r   zpytorch-convertedzpytorch-operatorsimpler   Fc                   > UR                   [        TS[        5       5      ;   a  TR                  R	                  US 45        g [        TS0 5      n[        TS0 5      n[        U5      S:X  d  [        U5      S:X  a  [        S5      eU R                  U[        TS/ 5      [        TS/ 5      [        TS/ 5      [        TS	/ 5      [        TS
/ 5      [        TS/ 5      UUUUUS9  g )N	skip_testr   r   r   zrtol or atol is empty.	successesmissedskippedload_failedexec_failedmismatch)verboser   r   check_other_runtimer   )r   getattrsetr  rY   r   rG   common_test_onnx_test_run)r   ter  r  r   r   r   clss          r;   _test_FTestOnnxBackEndWithReferenceEvaluator.add_test_methods.<locals>._test_  s     xx73SU#CC**B:6"33D"33D4yA~Ta,-EFF22["5Xr2Y3]B7]B7Z4 '!!,?!) 3 rM   )r  setattrr
  r   )r  r~   r  r  s   `   r;   add_test_methods6TestOnnxBackEndWithReferenceEvaluator.add_test_methods  s@    QF*62  T1u2 =rxxP7 3 RrM   c                    ^ Sm/ n[        SU4S j5       H  nUR                  U5        M     U R                  [        U5      S5        g )Ntest_absr   c                    > U T:H  $ r|   r   r~   r_   s    r;   <lambda>RTestOnnxBackEndWithReferenceEvaluator.test_onnx_backend_test_abs.<locals>.<lambda>  s	    fnrM   rT   r  rY   assertEqualr   r   coder  r_   s      @r;   test_onnx_backend_test_abs@TestOnnxBackEndWithReferenceEvaluator.test_onnx_backend_test_abs  s?    &v/LMBKKO NTA&rM   c                    ^ Sm/ n[        SU4S j5       H  nUR                  U5        M     U R                  [        U5      S5        g )Ntest_expand_shape_model1r  c                    > U T:H  $ r|   r   r%  s    r;   r&  bTestOnnxBackEndWithReferenceEvaluator.test_onnx_backend_test_expand_shape_model1.<locals>.<lambda>  s	    $rM   rT   r(  r*  s      @r;   *test_onnx_backend_test_expand_shape_model1PTestOnnxBackEndWithReferenceEvaluator.test_onnx_backend_test_expand_shape_model1  s?    )&x1NOBKKO PTA&rM   r   c                     [        XS9$ )Nr  r   )rx   r  s     r;   r   .TestOnnxBackEndWithReferenceEvaluator.load_fct  s    !#77rM   r5  c                   [        U S5      (       a  U R                  nOS[        U S5      (       a*  U R                  5        Vs/ s H  oDR                  PM     nnO[	        S[        U 5       S35      e[        U5      [        U5      :  a.  [        S[        U5       S[        U R                  5       S35      eSn[        [        U5      5       H  n[        X&   [        R                  5      (       d  M&  X&   R                  [        R                  :X  d  MH  U R                  U   R                  R                   ["        R$                  :w  d  M  SnM     U(       Gak  ['        U5      n[        [        U5      5       GHG  n[        X&   [        R                  5      (       d  M'  X&   R                  [        R                  :X  d  MI  U R                  U   R                  R                   ["        R$                  :w  d  M  X&   R)                  5       n[        R*                  " UR,                  S	   [        R.                  S
9n[        UR,                  S	   5       H  n	[1        Xy   5      n
XU	'   M     [3        UR5                  [        R.                  5      R7                  X&   R,                  5      ["        R8                  S5      X&'   GMJ     [        [        U5      5       Vs0 s H
  ocU   X&   _M     nnU R;                  S U5      nU$ s  snf s  snf )Ninput_names
get_inputszDUnable to extract the number to guess the number of inputs for type r?   zGot z inputs but expecting FTr   )r   )r   r8  r9  r_   AttributeErrorr   r   rG   r   r   rD   r   r   uint16input_typestensor_type	elem_typer	   UINT16r   rB   emptyr   rE   r   r   rC   reshapeBFLOAT16r   )rx   r  r   r8  rR   rewriter`   xrxfieelfeedsr   s                r;   r   -TestOnnxBackEndWithReferenceEvaluator.run_fct!  sT   3&&//KS,''+.>>+;<+;a66+;K<K VW[\_W`Vaabc  {c&k) s6{m#9#coo:N9OqQ  s6{#A69bjj11IOOryy0OOA&22<<@R@RR $ &\F3v;'vy"**55	2994*66@@KDVDVV*B"((1+RZZ@B#BHHQK008!#2 1 !(		"**-55fiooF#,,!FI (  5:#f+4FG4FqQ*4FGggdE"
M =H Hs   LLNc                 `	  ^ US:  a  [        SUR                  5        US:  a  [        S5        U R                  UR                  [        U5      5        U R	                  [        U5      S5         US:  a  [        S5        US:  a`  UR                  US.S	 j[        R                  U
R                  UR                  S 5      U	R                  UR                  S 5      S
U S3US9  OUR                  [        R                  [        R                  U
R                  UR                  U
R                  UR                  S 5      5      U	R                  UR                  U	R                  UR                  S 5      5      SUS9  US:  a&  [        S5        US:  a  [        UR                  5        UR                  XR                  UR                  S 5      U	R                  UR                  S 5      45        US:  a  [        S%5        g g ! [         a  nUS:  a  [        SU[        U5      5        UR                  X45        [!        SUR                   S3S5       nUR#                  UR                  R%                  5       5        S S S 5        Ue! , (       d  f       Ue= fS nAf[&        [(        4 GaK  nUS:  a  [        SU[        U5      5        UR                  X45        [!        SUR                   S3S5       nUR#                  UR                  R%                  5       5        S S S 5        O! , (       d  f       O= fUc  UeSU;   a  [        S5        SSKJm  [.        nUR                  R0                   H!  nUR2                  S;   d  M  UR4                  n  O   UR                  R6                  [8        :  d
  U[.        :  a  [        S5         S nAg UR                  U4S jS SSSS9  [        S 5        UeS nAf[:         a  nUS:  a  [        SU[        U5      5        [!        S!UR                   S3S5       nUR#                  UR                  R%                  5       5        S S S 5        O! , (       d  f       O= f['        S"UR                  < S#U S$UR                   35      UeS nAff = f)&N   zTEST:   z  check runtimer   z  runr   r5  c                 2    [         R                  " / UQU P76 $ r|   )r
  r   )r  argss     r;   r&  QTestOnnxBackEndWithReferenceEvaluator.common_test_onnx_test_run.<locals>.<lambda>h  s     3X3a3a 44&4rM   z%[runtime=ReferenceEvaluator, verbose=])r   r   r   r   z[runtime=ReferenceEvaluator]z	  end run   z  missed_r   wb	mismatch_onnxruntimezCHECK RUNTIME onnxruntime)InferenceSession)r   zai.onnxzLSkip test because of IR or opset version is not supported by onnxruntime yetc                 0   > T" U R                  5       S/S9$ )NCPUExecutionProvider)	providers)SerializeToString)rx   rV  s    r;   r&  rO    s     0--/<R;S!rM   c                  2    [         R                  " U SS0UD6$ )Nr  rT   )r
  r   )r   r   s     r;   r&  rO    s#    $I$Q$Q%$%%)*%rM   r   MbP?z[runtime=onnxruntime])r   r   r   doneissue_zUnable to run test z due to r   z  end example.)r   r_   assertInreprassertGreaterr   r   r
  r   getr   r   r   r   r   rY   rh   writerZ  rG   r9   rU  rV  r   opset_importdomainversion
ir_versionr   rr   )r   r  r  r  r  r  r  r  r  r   r   r  r   ry   r^   onnx_domain_opsetopsetrV  s                    @r;   r  ?TestOnnxBackEndWithReferenceEvaluator.common_test_onnx_test_runN  s    Q;'277#Q;#$bggtBx(3r7A&R	{g{*1  :AA"''40"''40CG9AN%  	 9BB9AA"((DHHRWWd,CD"((DHHRWWd,CD:%   {k"Q;"--(r 	"hhrxx648PQRQ;"# s # 	{dAtAw'MM2'"y.5779: 6G 65G
+ (	{dAtAw'OORG$	"''%0$71779: 877"* 33128$H!]]77E||6,1MM) 8 MM,,/KK(+OOf  3  
 fG 	{dAtAw'rwwiu-t4779: 544 %bgg[2bmm_M	s   0D;H 
R-AJ*J
J
J	JR-1AO79*L,#	O7,
L:	6AO7
AO7%O77R-6R(:*Q-$	R(-
Q;	71R((R-c                 V   [        U 5      n[        U5      [        U5      [        U5      [        U5      [        U5      /nXw[        U5      -   -  n	U(       GaJ  [        R                  R	                  [
        5      n
[        S5        [        SU S[        U5       S[        U5       S[        U5       S[        U5       S[        U5       35        [        SU	S	-  S
 SU[        U5      -    S35        US:  Ga  S n[        S5        [        US S9 H(  n[        SUS   R                  SU" US   U
5      5        M*     [        US S9 H(  n[        SUS   R                  SU" US   U
5      5        M*     [        US S9 H(  n[        SUS   R                  SU" US   U
5      5        M*     [        US S9 H(  n[        SUS   R                  SU" US   U
5      5        M*     [        US S9 H(  n[        SUS   R                  SU" US   U
5      5        M*     US:  a  [        S5        [        SU S[        U5       S[        U5       S[        U5       S[        U5       S[        U5       35        [        SU	S	-  S
 SU[        U5      -    S35        [        S5        [        U5      S:  a/  US   u  p[        SUR                  < SUR                   S35      Ue[        U5      [        [        5      :  a.  [        S [        U5       S!U[        U5      -    S"U	S	-  S
 S#35      eg )$Nz-----------zsuccess=z
, skipped=z	, missed=z, load_failed=z, exec_failed=z, mismatch=z	coverage rp   z.1fz	% out of z tests   c                 x    [        U 5      R                  SS5      R                  US5      R                  SS5      $ )Nz\\r   onnxr   )r   r   )srV   s     r;   _printBTestOnnxBackEndWithReferenceEvaluator._postprocess.<locals>._print  s2    A . v. s+	rM   c                      U S   R                   $ Nr   r   ms    r;   r&  DTestOnnxBackEndWithReferenceEvaluator._postprocess.<locals>.<lambda>      1Q4::rM   )keyzloading failedr   z---c                      U S   R                   $ rs  rt  ru  s    r;   r&  rw    rx  rM   zexecution failedc                      U S   R                   $ rs  rt  ru  s    r;   r&  rw    s    !

rM   r  c                      U S   R                   $ rs  rt  ru  s    r;   r&  rw    s    adjjrM   zmissed c                      U S   R                   $ rs  rt  ru  s    r;   r&  rw    s    qtzzrM   r     zMismatch in test r   r?   zUnexpected failures. r   z# tests have failed.The coverage is z:%. New operators were added with no corresponding runtime.)r   sumrU   rV   r  	onnx_filer   sortedr   rG   r_   r   
SKIP_TESTS)r  r  r  r  r  r  r  successfailedcoveragerV   rp  r  r  ry   s                  r;   _postprocess2TestOnnxBackEndWithReferenceEvaluator._postprocess  sB    i.KLM
 F3477??9-D- 7):c'l^9S[MQ_`cdo`p_q r";/0CM?L HsN3/y3v;9N8OvV { m$1EFA*AaDJJvadD?QR G1EFA,adjj%!dAST G.BCA*adjj%!d9KL D,@AA)QqTZZqtT8JK B-ABA)QqTZZqtT8JK C R<-("7):c'l^9SQW[MYghklwhxgy z''*;'7&8CM?T #HsN3#7y3v;AV@WW]^ -(x=1QKEB #BGG;bqA v;Z( 'F}AgF6K5L M##+c>#"6 7JK  )rM   c                    SSSSSSSSSSSSSS.U l         0 SS_S	S_S
S_SS_SS_SS_SS_SS_SS_SS_SS_SS_SS_SS_SS_SS_SS_0 SS_SS_SS_SS_SS_SS_S S_S!S_S"S_S#S_S$S_S%S_S&S_S'S_S(S_S)S_S*S_ESSSSSSSSSSSSS+.EU l        [        U l        U(       a  [	        5       U l        / U l        / U l        / U l        / U l        / U l	        / U l
        g ),Ng{Gz?r   g-C6?r   r\  )test_adam_multipletest_blackmanwindow_expanded&test_blackmanwindow_symmetric_expandedtest_simple_rnn_batchwise#test__pytorch_converted_Conv1d_pad1test__pytorch_converted_Conv2d&test__pytorch_converted_Conv2d_no_bias&test__pytorch_converted_Conv2d_strided0test_layer_normalization_4d_axis1_expanded_ver18:test_layer_normalization_4d_axis_negative_1_expanded_ver18:test_layer_normalization_4d_axis_negative_3_expanded_ver18test_ConvTranspose2d'test__pytorch_converted_ConvTranspose2dtest_blackmanwindowr   r  test_blackmanwindow_symmetricr  test_Conv1dgư>test_Conv2d_depthwise_paddedtest_Conv3d_dilatedtest_gridsample_bicubictest_gru_seq_lengthtest_hammingwindow_expanded%test_hammingwindow_symmetric_expandedtest_hannwindow_expandedtest_hannwindow_symmetric"test_hannwindow_symmetric_expanded4test_layer_normalization_4d_axis_negative_1_expanded*test_layer_normalization_4d_axis1_expanded4test_layer_normalization_4d_axis_negative_3_expanded	test_mishtest_mish_expandedtest_roialign_aligned_falsetest_roialign_aligned_true/test__pytorch_converted_ConvTranspose2d_no_bias&test__pytorch_converted_Linear_no_biastest_Linear_no_biasr  r  (test__pytorch_converted_Conv2d_depthwise0test__pytorch_converted_Conv2d_depthwise_strided8test__pytorch_converted_Conv2d_depthwise_with_multiplier/test__pytorch_converted_Conv2d_depthwise_padded%test__pytorch_converted_Conv2d_groups*test__pytorch_converted_Conv2d_groups_thnnr  r  )1test__pytorch_operator_operator_symbolic_overridetest_operator_symbolic_override.test__pytorch_converted_Conv3d_dilated_strided%test__pytorch_converted_Conv3d_groupstest_affine_grid_2dtest_affine_grid_2d_expanded!test_affine_grid_2d_align_corners*test_affine_grid_2d_align_corners_expandedtest_affine_grid_3dtest_affine_grid_3d_expanded!test_affine_grid_3d_align_corners*test_affine_grid_3d_align_corners_expanded)r   r   r  r  r  r  r  r  r  r  r  )r  	all_testss     r;   
setUpClass0TestOnnxBackEndWithReferenceEvaluator.setUpClass  sK    #',-67)*37.26:6:@DJNJN$(7;
 0
!40
*D0
 ,T0
 5d	0

 40
 +D0
 "40
 &t0
 "40
 *40
 4T0
 '0
 (0
 1$0
 CD0
  9$!0
" CD#0
$ %0
& !$'0
( *4)0
* )$+0
. >t/0
0 5d10
2 "430
4 2450
6 -d70
8 790
: ?;0
< G=0
> >t?0
@ 4TA0
B 9$C0
D 5dE0
F 5dG0
H BF/3>B59#',015:>#',015:>_0
d #ECM
rM   c           	         [        U R                  5      S:X  al  U R                  U R                  -   U R                  -   U R
                  -   n[        U5      S:  a  [        S[        U5       S35      US   S   e[        S5      eU R                  U R                  U R                  U R                  U R                  U R
                  U R                  S5        g )Nr   zNo test was successful, z failed.rT   zNo test was successful.
   )	r   r  r  r  r  r  rs   r  r  )r  r  s     r;   tearDownClass3TestOnnxBackEndWithReferenceEvaluator.tearDownClassT  s    s}}"\\CJJ.@3??RF6{Q".s6{m8Day|$ 899MMJJKKOOOOLL	
rM   r   )r   )r   NNNF)F)r   r   r   r   rU   rV   r}   abspathr  r   r~   classmethodr   r,  r2  r   r   r   r  r  r  r  r   r   rM   r;   r
  r
    s    WW\\
124LF Q Q>'' 8 8 &' * *j  j$X D DL P Pd 
 
rM   r
  __main__   )	verbosityr|   )1r   rU   r   sysunittestr   numpyrD   version_utilsr   r   numpy.testingr   rn  r   r   r   r	   r
   onnx.backend.testr   r  onnx.helperr  onnx.numpy_helperr   r   r   onnx.referencer   onnx.reference.op_runr   onnx.reference.ops.op_castr   rZ   r   r   MIN_PASSING_TESTSr  numpy_older_thanplatformrL   rO   r  TestCaser
  r   r   mainr   rM   r;   <module>r     s6  " 
  
     ) ) I I 8 - G G - 3 .  #6*H##NO '*
148( $
  
 !!(++  J
 <<7  J !!(++  
 
JG2F FR
*@{
H,=,= {
| & 6 6 8 zMMA rM   