
    hoA                     `   S SK r S SKrS SKrS SKJr  S SKrS SKrS SKrS SKJ	r	  S SK
Jr  S SKJr  S SKJr  S SKJr  S SKJrJr  S S	KJr  \ R.                  " \5      r " S
 S\R4                  R6                  5      r " S S\R4                  R6                  5      r " S S5      r " S S5      rg)    N)Path)
TypeHelper)	OnnxModel)PastKeyValuesHelper)T5EncoderInputs)torch_onnx_export)	MT5ConfigT5Config)InferenceSessionc            
          ^  \ rS rSrSr SS\R                  R                  S\R                  R                  S\\	-  S\
S-  4U 4S jjjrS	\R                  S
\R                  S\R                  4S jrSrU =r$ )T5DecoderInit   zvA T5 decoder with LM head to create initial past key values.
This model is only called once during starting decoding.
Ndecoderlm_headconfigdecoder_start_token_idc                   > [         TU ]  5         Xl        X l        X0l        Ub  UOU R                  R
                  U l        [        U R                  S5      (       a  U R                  R                  U l        g SU l        g Ntie_word_embeddingsT)super__init__r   r   r   r   hasattrr   )selfr   r   r   r   	__class__s        g/var/www/fran/franai/venv/lib/python3.13/site-packages/onnxruntime/transformers/models/t5/t5_decoder.pyr   T5DecoderInit.__init__    so     	&<&H"dkkNpNp 	# 07t{{DY/Z/ZDKK++ 	 `d 	     decoder_input_idsencoder_attention_maskencoder_hidden_statesc                    UcL  UR                   S   n[        R                  " US4[        R                  UR                  S9U R
                  -  nU R                  UUUSSS9nUR                  nUR                  nU R                  (       a  X`R                  R                  S-  -  nU R                  U5      n[        R                  " U5      u  pXU
4$ )Nr      dtypedeviceT)	input_idsr    r   	use_cachereturn_dict      )shapetorchoneslongr%   r   r   last_hidden_statepast_key_valuesr   r   d_modelr   r   group_by_self_or_cross)r   r   r   r    
batch_sizedecoder_outputssequence_outputpresent_key_values	lm_logits	past_self
past_crosss              r   forwardT5DecoderInit.forward2   s     $/55a8J

O**188
 --.  ,,'"7#9 ' 
 *;;,<<##-1D1Dd1JKOLL1	 3 J JK] ^	Z//r   )r   r   r   r   r   N)__name__
__module____qualname____firstlineno____doc__r+   nnModuler
   r	   intr   TensorFloatTensorr9   __static_attributes____classcell__r   s   @r   r   r      s     .2

 
 9$	

 !$d

 
$!0 <<!0 !&!0  %00	!0 !0r   r   c                   2   ^  \ rS rSrSrU 4S jrS rSrU =r$ )	T5DecoderV   z-A T5 decoder with LM head and past key valuesc                    > [         TU ]  5         Xl        X l        X0l        [        U R                  S5      (       a  U R                  R                  U l        g SU l        g r   )r   r   r   r   r   r   r   )r   r   r   r   r   s       r   r   T5Decoder.__init__Y   sN    /6t{{DY/Z/ZDKK++ 	 `d 	 r   c           	         U R                   R                  n[        R                  " X45      nUR	                  S5      nU R                  UUUUSSS9nUR                  nUR                  n	U R                  (       a  XR                   R                  S-  -  nU R                  U5      n
[        R                  " U	5      u  pX4$ )N   T)r&   r/   r    r   r'   r(   r)   )r   num_decoder_layersr   group_by_layer	unsqueezer   r.   r/   r   r0   r   r1   )r   r   r   pastrP   r/   dummy_encoder_hidden_statesr3   r4   r5   r6   present_self_s                r   r9   T5Decoder.forwardb   s    ![[;;-<<TV '=&F&Fq&I#,,'+"=#9 ' 
 *;;,<<##-1D1Dd1JKOLL1	-DDEWX &&r   )r   r   r   r   )	r<   r=   r>   r?   r@   r   r9   rF   rG   rH   s   @r   rJ   rJ   V   s    7
' 'r   rJ   c                       \ rS rSr SS jr\  SS\\-  S\S\S\S\	R                  S	\S
\4S jj5       rS\4S jrS rSrg)T5DecoderInputs~   Nc                 (    Xl         X l        X0l        g r;   r   r   r/   )r   r   r   r/   s       r   r   T5DecoderInputs.__init__   s     4E8N#Xgr   r   r2   encode_sequence_lengthpast_decode_sequence_lengthr%   float16use_int32_inputsc           	         U R                   nU R                  nU R                  n	U R                  n
Sn[        R
                  " SU	S-
  X4U(       a  [        R                  O[        R                  US9n[        R                  " UUU	UUS9nU(       a  [        R                  O[        R                  nUS:  a  UUUU
/nUUUU
/n/ n[        SU-  5       H'  nUR                  [        R                  " XUS95        M)     [        SU-  5       H'  nUR                  [        R                  " UXS95        M)     OSn[        XR                   U5      $ )a  Create dummy inputs for T5Decoder.

Args:
    decoder: decoder
    batch_size (int): batch size
    encode_sequence_length (int): sequence length of input_ids for encoder
    past_decode_sequence_length (int): past sequence length of input_ids for decoder
    device (torch.device): device of output tensors
    float16 (bool): whether the model uses float32 or float16 in input
    use_int32_inputs(bool): whether use int32 instead of int64 for some inputs

Returns:
    T5DecoderInputs: dummy inputs for decoder
r"   r   )lowhighsizer$   r%   )ra   rO   r#   N)	num_headsrP   
vocab_sized_kvr+   randintint32int64r   create_dummyr`   float32rangeappendrandrY   attention_mask)r   r2   r^   r_   r%   r`   ra   num_attention_heads
num_layersrg   	head_sizesequence_lengthr   encoder_inputs
float_typeself_attention_past_shapecross_attention_past_shaperS   rV   s                      r   rl   T5DecoderInputs.create_dummy   sI   0 $*#3#3 33
 ++
  	 !MMa."25;;
 )55"-
 '.U]]5==
&*#+	)% #&	*& D1z>*EJJ'@[abc + 1z>*EJJ'Acd + D02O2OQUVVr   returnc                     U R                   U R                  /nU R                  (       a  UR                  U R                  5        U$ r;   )r   r   r/   extend)r   
input_lists     r   to_listT5DecoderInputs.to_list   s@    ""''

 d223r   c                    U R                   (       a7  U R                    Vs/ s H  oR                  [        R                  S9PM!     snOS n[	        U R
                  R                  5       U R                  R                  5       U5      $ s  snf )N)r$   )r/   tor+   rm   rY   r   cloner   )r   prS   s      r   to_fp32T5DecoderInputs.to_fp32   sp    LPL`L`43G3GH3Ga5==)3GHfj""((*''--/
 	
 Is   &Br\   r;   )FF)r<   r=   r>   r?   r   staticmethodr
   r	   rC   r+   r%   boolrl   listr   r   rF    r   r   rY   rY   ~   s    
 	h  !&IW9$IWIW !$IW &)	IW
 IW IW IW IWV 
r   rY   c                       \ rS rSr\   SS\\-  S\R                  S\	S\
S\
S\
4S jj5       r\S	\4S
 j5       r\ SS\\-  S\S\R                  S\
S\4
S jj5       rSrg)T5DecoderHelper   r   r%   onnx_model_pathverboseuse_external_data_formatra   c                    [        U [        [        45      (       d   e[        R	                  U R
                  SS[        U [        5      (       a  SOSUUS9nUR                  5       nU R
                  R                  n[        R                  " USS9n	[        R                  " USS9n
U
S	SU-   n[        U [        5      (       a  U	O/ n[        U [        5      (       a  UOU
nS
/UQnS/nUR                  S5        UR                  U5        SS0SSS.SSS.SS0S.nU H  nSSU;   a  SOSS.UU'   M     U H9  nSU;   a
  SSS.UU'   M  [        U [        5      (       a
  SSS.UU'   M2  SS0UU'   M;     [        U5      R                  R                  SSS9  [        R                   " 5        n["        R$                  R'                  US5      n[        U5      R                  R                  SSS9  [)        U [+        U5      U(       a  UOUSUUUSSUUS9  U(       a,  [,        R.                  " USS9n[0        R2                  " UUSSS9  S	S	S	5        g	! , (       d  f       g	= f)a  Export decoder to ONNX

Args:
    decoder (Union[T5Decoder, T5DecoderNoPastState]): decoder object
    device (torch.device): device of decoder object
    onnx_model_path (str): onnx path
    verbose (bool, optional): print verbose information. Defaults to True.
    use_external_data_format (bool, optional): use external data format or not. Defaults to False.
    use_int32_inputs (bool, optional): use int32 inputs
rO         r   )r2   r^   r_   r%   ra   F)presentTNlogitsr&   r   r2   r^   )r   r"   )r&   r   r    r   r   r_   )r   rO   crosszpast_decode_sequence_length + 1)parentsexist_okzdecoder.onnx   )
argsfexport_paramsinput_namesoutput_namesdynamic_axesopset_versiondo_constant_foldingr   r   )load_external_data)save_as_external_dataall_tensors_to_one_file)
isinstancerJ   r   rY   rl   r   r   rP   r   get_past_namesro   r}   r   parentmkdirtempfileTemporaryDirectoryospathjoinr   tupleonnx
load_modelr   save)r   r%   r   r   r   ra   inputsr~   rP   
past_namespresent_namespresent_self_namesinput_past_namesoutput_present_namesr   r   r   nametmp_dir_nametemp_onnx_model_pathmodels                        r   export_onnxT5DecoderHelper.export_onnx   s   & 'I}#=>>>> --NN#$-7-K-KQR- . 
 ^^%
$^^>>(778JTYZ
+::;MW[\*+CQ1C-CD)3GY)G)G:R5?5S5S1Yf 8#78 #m34+, < +7;S&T)5:R%S<
 %D4:dN0H`"L % )D$)5:R%ST"gy11'<*L& <*L& ) 	_$$**4$*G((*l#%77<<n#M %&--33D43P:&*B&"')) $()A ((<QUV#*.,0	' +**s   ,BI
Ir   c                    [         R                  S5        [        R                  " UR                  R                  5       R                  5       5      [        R                  " UR                  R                  5       R                  5       5      S.nUR                  (       a  [        UR                  5      S-  S:X  d   e[        [        UR                  5      S-  5      n[        R                  " U5      n[        UR                  5       H<  u  pV[        R                  " UR                  5       R                  5       5      X$U   '   M>     U R                  SU5      nU$ )zRun inference of ONNX model.zstart onnxruntime_inference)r&   r      r   N)loggerdebugnumpyascontiguousarrayr   cpur   r/   lenrC   r   r   	enumeraterun)ort_sessionr   
ort_inputsrs   r   ipast_tensorort_outputss           r   onnxruntime_inference%T5DecoderHelper.onnxruntime_inference^  s    	23 001I1I1M1M1O1U1U1WX&+&=&=f>[>[>_>_>a>g>g>i&j


 !!v--.2a777S!7!781<=J,;;JGJ"+F,B,B"C,1,C,CKOODUD[D[D],^
a=) #D "oodJ7r   r   r   	max_casesc                 z   [         R                  " US5      S:H  n/ SQn/ nUSU  GH~  u  nn	n
[        U [        5      (       a  Sn
[        R                  U R                  UU	U
UUUS9nUR                  5       R                  5       n[        R                  " 5          U " U6 nSSS5        [        R                  X5      nU R                  R                  n[        R                  " [        R                   " WS   R#                  5       R                  5       US   -
  5      5      nUn[$        R'                  SU 35        [)        SU-  5       H  n[        R                  " [        R                   " US	   U   R#                  5       R                  5       US	U-      -
  5      5      n[$        R'                  S
U SU 35        [+        UU5      nM     [        U [        5      (       a  [)        SU-  5       H  n[        R                  " [        R                   " US   U   R#                  5       R                  5       US	SU-  -   U-      -
  5      5      n[$        R'                  SU SU 35        [+        UU5      nM     UR-                  U5        [$        R/                  SUU	U
U5        GM     W$ ! , (       d  f       GN= f)zQCompare the result from PyTorch and OnnxRuntime to verify the ONNX model is good.past_key_self_0ztensor(float16)))r      r   )r"   rO   r   )r   r"   r"   )   r   rO   Nr   )r%   r`   ra   zlogits max_diff=rO   r"   zself attention past state z
 max_diff=zcross attention past state zUbatch_size=%s, encode_sequence_length=%s, past_decode_sequence_length=%s, max_diff=%s)r   get_input_typer   r   rY   rl   r   r   r   r+   no_gradr   r   rP   r   amaxabsr   r   r   rn   maxro   info)r   r   r%   ra   r   r`   
test_casestest_cases_max_diffr2   r^   r_   r   r~   torch_outputsr   rP   max_diffmax_diff_allr   s                      r   verify_onnxT5DecoderHelper.verify_onnxr  s    #11+?PQUffB
 
 
#	
"'%//./+$11&+!1 2 F  )113J  %z 2 ! *??TK!&!@!@zz%))M!,<,@,@,B,H,H,J[YZ^,["\]H#LLL+H:671112 ::eiia0@0C0G0G0I0O0O0QT_`ade`eTf0f&gh9!JxjQR"<: 3
 %//q#556A$zz		-"21"5"9"9";"A"A"CkRSVWZlVlRlopRpFq"qr H LL#>qcH:!VW#&|X#>L 7  &&|4KKg&+U $d A !s   J++
J:	r   N)TFF)r   )r<   r=   r>   r?   r   rJ   r   r+   r%   strr   r   rY   r   r   rC   r   rF   r   r   r   r   r      s    
 ).!&s]*ss s 	s
 #'s s sj ?  &  B=(B%B B 	B
 B Br   r   ) loggingr   r   pathlibr   r   r   r+   io_binding_helperr   
onnx_modelr   past_helperr   
t5_encoderr   torch_onnx_export_helperr   transformersr	   r
   onnxruntimer   	getLoggerr<   r   rA   rB   r   rJ   rY   r   r   r   r   <module>r      s     	      (   + & 6 , (			8	$80EHHOO 80v%' %'Pf
 f
RN Nr   