
    hN                       S SK Jr  S SKrS SKrS SKJrJr  S SKJ	r	  S SK
JrJr  SS jr  S           SS jjr    S               SS jjr     S                     SS	 jjr                SS
 jrSSS jjrSS jr   S       SS jjrS S jrS!S jr            S"S jr          S#S jr                S$S jrg)%    )annotationsN)
AutoConfigAutoTokenizer)DynamicCache)InferenceSessionOrtValuec                    U R                  5       R                  S5      S-
  nUR                  U S:H  S5        U(       a  US S 2S4   R                  S5      nU$ )N   r   )longcumsummasked_fill_	unsqueeze)attention_maskuse_past_kvposition_idss      l/var/www/fran/franai/venv/lib/python3.13/site-packages/onnxruntime/transformers/models/llama/llama_inputs.pyget_position_idsr      sY    !&&(//3a7Ln115#ArE*44R8     c                   [         R                  " SU R                  X#4[         R                  S9n[         R                  " X#[         R                  S9n[        USS9nUS:X  a  UR                  5       OUR                  U5      nUS:X  a  UR                  5       OUR                  U5      nUS:X  a  UR                  5       OUR                  U5      nU(       d  XgU4$ UUUS.n	U	$ )Nr   lowhighsizedtyper   Fr   ort	input_idsr   r   )torchrandint
vocab_sizeint64onesr   numpyto)
configdevice
batch_sizeseq_lenenginereturn_dictr    r   r   inputss
             r   get_sample_inputsr/       s     !&*;*;:BW_d_j_jkIZZ
5;;GN#NFL &,u_	!),,v:NI/5^))+NDUDUV\D]N+1U?<%%'PV@WL<88 ($F
 Mr   c                J   [         R                  " SU R                  US4[         R                  S9n[         R                  " X#S-   [         R                  S9n	[        U	SS9n
[        XX4US9nUS:X  a  UR                  5       OUR                  U5      nUS:X  a  U	R                  5       OU	R                  U5      n	US:X  a  U
R                  5       OU
R                  U5      n
US:X  a  [        U5      O7U Vs/ s H*  oS   R                  U5      US   R                  U5      4PM,     snnU(       d  [        U[        5      (       d   eXX4$ UU	U
S	.nUS:X  a*  [        U[        5      (       d   eUR                  U5        U$ [        U[        5      (       d   eXS
'   U$ s  snf )Nr   r   r   r   Tr   
world_sizer   r   past_key_values)r!   r"   r#   r$   r%   r   get_past_kv_inputsr&   r'   flatten_past_kv_inputs
isinstancelistdictupdate)r(   r)   r*   past_seq_lenuse_fp16r,   r-   r2   r    r   r   past_kvkvr.   s                 r   get_sample_with_past_kv_inputsr>   C   s    !&*;*;:q/Y^YdYdeIZZ
1,<EKKPN#NEL \XbcG &,u_	!),,v:NI/5^))+NDUDUV\D]N+1U?<%%'PV@WL+1U?w'pw@xpwjlQ%((6BRTVWXTYT\T\]cTdAepw@x  '4((((<AA ($F
 '4((((g
 M '4(((($+ !M) Ays   71F c                t   [         R                  " SU R                  X#4[         R                  S9n[         R                  " X$U-   [         R                  S9n[        XS:g  S9n[        XXFU
S9nUS:X  a  UR                  5       OUR                  U5      nUS:X  a  UR                  5       OUR                  U5      nUS:X  a  UR                  5       OUR                  U5      nUS:X  a  [        U5      O7U Vs/ s H*  oS   R                  U5      US   R                  U5      4PM,     snnU	(       d  [        U[        5      (       d   eXX4$ UUUS.nUS:X  a=  [        U[        5      (       d   eUR                  U5        U(       a  [        UXE5      nU$ [        U[        5      (       d   eUUS	'   U$ s  snf )
Nr   r   r   r   r1   r   r   r   r3   )r!   r"   r#   r$   r%   r   r4   r&   r'   r5   r6   r7   r8   r9    enable_past_present_share_buffer)r(   r)   r*   r+   r:   max_seq_lenr;   use_buffer_sharer,   r-   r2   r    r   r   r<   r=   r.   s                    r   %get_merged_sample_with_past_kv_inputsrC   y   s    !&*;*;:BW_d_j_jkIZZ
7,B%++VN#NQRARTL \XbcG &,u_	!),,v:NI/5^))+NDUDUV\D]N+1U?<%%'PV@WL+1U?w'pw@xpwjlQ%((6BRTVWXTYT\T\]cTdAepw@x  '4((((<AA ($F
 '4((((g5flXF M '4(((($+ !M1 Ays   81F5c                   U(       a  [         R                  O[         R                  nU R                  U R                  -  n	U(       Gd!  [         R
                  R                  XU R                  5      R                  U5      S[         R                  " [         R                  " XU45      SS9-  R                  U5      [         R
                  R                  XR                  X R                  U	5      R                  U5      [         R
                  R                  XR                  X R                  U	5      R                  U5      [         R                  " U[         R                  S9S.n
U
$ [         R
                  R                  XU R                  5      R                  U5      [         R                  " [         R                  " XU4[         R                  S9SS9S-
  R                  [         R                  5      [         R                  " U[         R                  S9S.n
[        U R                  5       H  nU
R                  SU S3[         R
                  R                  XR                  X)5      R                  U5      S	U S3[         R
                  R                  XR                  X)5      R                  U5      05        M     U(       a  [!        XU5      n
U
$ )
Ng     r   )kr   )x	attn_maskk_cachev_cachepos)rF   rG   rJ   k__cachev_)npfloat16float32hidden_sizenum_attention_headsrandomrandastypetriur%   num_hidden_layersarrayr$   int32ranger9   r@   )r(   r*   r:   r+   rA   r;   rB   split_kvnp_dtype	head_size
ort_inputsis               r   get_msft_sample_inputsr`      s    &rzz2::H""f&@&@@I
V5G5GHOOPXY"RWWRWWj{5[-\`a%bbjjkstyy~~44lD^D^`ifXyy~~44lD^D^`ifX88L9


D + 
V5G5GHOOPXY''"'':K*PXZX`X`"aefgjkkss 88L9

 v//0A6NBIINN"$>$>%fX&6NBIINN"$>$>%fX&	 1 9*T_`Jr   c                   U R                   U-  n[        U S5      (       a  U R                  OU R                  U R                  -  nU(       a  [
        R                  O[
        R                  n[        U R                  5       Vs/ s H/  n[
        R                  " XX&US9[
        R                  " XX&US94PM1     n	nU	$ s  snf )Nhead_dimr   )num_key_value_headshasattrrb   rQ   rR   r!   rO   rP   rZ   rW   rT   )
r(   r*   r:   r;   r2   	num_headsr]   torch_dtype_r<   s
             r   r4   r4      s    **j8I#*6:#>#>FDVDVZ`ZtZtDtI#+%--K v//0
 1A JJzl[YJJzl[Y	
 1   Ns   6B>c                   0 n[        U 5       H  u  nu  p4[        U [        5      (       af  UR                  5       R	                  5       R                  5       USU 3'   UR                  5       R	                  5       R                  5       USU 3'   M  UR                  5       R	                  5       R                  5       USU S3'   UR                  5       R	                  5       R                  5       USU S3'   M     U$ )Npast_key_values_key_cache_past_key_values_value_cache_past_key_values..key.value)	enumerater6   r   detachcpur&   )r3   r<   r_   past_kpast_vs        r   r5   r5      s    G(9Fo|448>8K8K8M8S8S8UG045:@--/:M:M:O:U:U:WG21#6728--/2E2E2G2M2M2OG&qc./4:MMO4G4G4I4O4O4QG&qc01  : Nr   c                L   0 nU R                  5        Hz  u  pV[        U[        R                  5      (       a  XdU'   M*  US:X  a  UR	                  [        U5      5        ML  UR                  5       R                  5       R                  5       XE'   M|     U(       a  [        XBU5      nU$ )Nr3   )
itemsr6   rN   ndarrayr9   r5   ro   rp   r&   r@   )	pt_inputsrB   r:   rA   r^   rE   vs          r   convert_inputs_for_ortrx      s     J!a$$qM##4Q78HHJNN,224JM " 5jP[\
r   c                    U R                  5        HW  u  p4SU;   d  SU;   d  M  UR                  u  pVpx[        R                  " XVX(4UR                  S9n	XIS U2S U2S U2S U24'   XU'   MY     U $ )Ncacher3   r   )rt   shaperN   zerosr   )
r^   r:   rA   rE   rw   r*   re   rg   r]   new_vs
             r   r@   r@     s      "a<,123''/J1HHj[LTUT[T[\EHI+:+z	z=L=*9*DE!qM # r   c                ,   U R                  5        Vs1 s H  o"R                  iM     nn[        UR                  5       5      nX4-
  n[	        U5      (       a  [        SU 35        [        S5      eXC-
  n[	        U5      (       a  U H  nX	 M     U$ s  snf )Nz(The following model inputs are missing: zEThere are missing inputs to the model. Please add them and try again.)
get_inputsnamesetkeyslenprint	Exception)modelr^   model_inputmodel_inputsuser_inputsmissing_inputsunnecessary_inputsunnecessary_inputs           r   verify_ort_inputsr   #  s    8=8H8H8JK8J$$8JLKjoo'(K!/N
>88HIJ_`` %3
!3- "4  Ls   Bc                   U R                  5       nU R                  5        Vs1 s H  owR                  iM     nnUR                  5        H  u  pX;  a  M  U(       af  SU	;   d  SU	;   aZ  X;  a,  [        R
                  " XUS9nUR                  X5        XU	'   MP  XY   R                  U
5        UR                  XU	   5        My  [        R
                  " XUS9nUR                  X5        M     U R                  5        Hj  nUR                  nU(       aD  SU;   d  SU;   a8  UR                  SS5      R                  SS5      nUR                  XU   5        MZ  UR                  XUS9  Ml     Xe4$ s  snf )Nrz   r3   )device_type	device_idoutpresent)
io_bindingr   r   rt   r   ortvalue_from_numpybind_ortvalue_inputupdate_inplaceget_outputsreplacebind_ortvalue_outputbind_output)r   r^   r)   r   rB   kv_cache_ortvaluesr   r_   r   rE   rw   v_deviceoutputr   
input_names                  r   add_io_bindings_as_ortvaluesr   7  s\    !!#J$)$4$4$67$6qFF$6L7  "   A1Ba1G*#77Ybc..q;(01%"%44Q7..qQ2GH33AU^_H**17% #( ##%{{$)t2CeW5==iIZ[J++DZ2PQ""4y"Q & ))= 8s   E.c           
        [        X5      nS n[        R                  [        R                  [        R                  [        R
                  S.nU R                  5       nUR                  5        H  u  pUR                  UU	R                  R                  U	R                  R                  S:X  a  SOU	R                  R                  U[        U	R                  5         [        U	R                  5      U	R!                  5       S9  U	R                  nM     U R#                  5        H  n
U
R$                  nU(       a  SU;   a  XR'                  SS5         OX+   n	UR)                  UUR                  UR                  S:X  a  SOUR                  U(       a  [        R                  O[        R
                  [        U	R                  5      U	R!                  5       S9  M     U$ )N)ztorch.int32ztorch.int64ztorch.float16ztorch.float32rp   r   )r   r   r   element_typer{   
buffer_ptrr   r3   )r   rN   rY   r$   rO   rP   r   rt   
bind_inputr)   typeindexreprr   tupler{   data_ptrr   r   r   r   )r   r.   outputsr;   rB   r)   pt_to_npr   rE   rw   r   r   s               r   add_io_bindings_as_tensorsr   d  sj    u-FFxxxx	H !!#J88==E1aqxx~~!$qww-0.zz| 	 	
   ##%{{  I$5 <<	+<=> 	

 	!;;%/aV\\(0"**bjj.zz| 	 	
 &" r   c           
     f   UR                   Ul        UR                  USS9nU(       a  [        R                  O[        R
                  n	[        R                  " US   U[        R                  S9n
[        R                  " US   U[        R                  S9n[        USS9nU
R                  S   nX:  a!  U
S S 2S U24   n
US S 2S U24   n[        USS9nOX:  a  U
S S 2S	4   R                  S	5      R                  nUS S 2S	4   R                  S	5      R                  n[        X--
  5       H1  n[        R                  " X45      n
[        R                  " X45      nM3     [        USS9nU
R                  S   nX:X  d   eUS
:X  a  U
R                  5       OU
US
:X  a  UR                  5       OUUS
:X  a  UR                  5       OUS.nUS
:w  a  / US'   U
R                  u  nnU R                  nU R                   n[#        U S5      (       a  U R$                  OU R&                  U R(                  -  n[        U R*                  5       H  n[        R,                  " UUU(       a  UOS	UUU	S9n[        R,                  " UUU(       a  UOS	UUU	S9nUS
:X  a;  UR/                  SU S3UR                  5       SU S3UR                  5       05        M  US   R1                  UU45        M     S nUS
:X  a  [        R,                  " UUU R2                  XIS9nSUR                  5       0nU(       d  [        U R*                  5       Hl  n[        R,                  " UUUUXIS9n[        R,                  " UUUUXIS9nUR/                  SU S3UR                  5       SU S3UR                  5       05        Mn     UU4$ )NT)paddingr    )r)   r   r   Fr   r
   r   r   r   r3   rb   rk   rl   rm   logitszpresent.)	eos_token	pad_tokenbatch_encode_plusr!   rO   rP   tensorr$   r   r{   r   TrZ   hstack
contiguousmax_position_embeddingsrc   rd   rb   rQ   rR   rW   r|   r9   appendr#   )r(   	tokenizerrequested_lengthpromptr)   r;   rB   r,   encodings_dictrf   r    r   r   tokenized_lengthinput_ids_first_colattention_mask_first_colrg   r.   r*   sequence_lengthmax_sequence_lengthre   r]   r_   past_key
past_valuer   r   present_keypresent_values                                 r   get_initial_inputs_and_outputsr     s    $--I000FN#+%--K
 ^K8u{{[I\\.1A"B6Y^YdYdeN#NFL !r**a!2"2!223	'+<,<+<(<='EJ		,'1o77:<<#1!Q$#7#A#A!#D#F#F ':;A&9%EFI"\\+C*TUN < (EJ r*/// 06Y))+I9?5.335n5;u_//1,F
 $& ! #,//J 88**I#*6:#>#>FDVDVZ`ZtZtDtI 6++,;;#3
 [[#3

 U?MM&qc.0C0C0E&qc0*2G2G2I $%,,h
-CD3 -6 GZ&:K:KTZnV..016334#kk	?If !&	?If! s$')?)?)AXaSPVCWYfYqYqYst 5 7?r   )r   ztorch.Tensorr   bool)ptF)r(   r   r)   torch.devicer*   intr+   r   r,   strr-   r   )Fr   Fr   )r(   r   r)   r   r*   r   r:   r   r;   r   r,   r   r-   r   r2   r   )FFr   Fr   )r(   r   r)   r   r*   r   r+   r   r:   r   rA   r   r;   r   rB   r   r,   r   r-   r   r2   r   )r(   r   r*   r   r:   r   r+   r   rA   r   r;   r   rB   r   r[   r   )r   )
r(   r   r*   r   r:   r   r;   r   r2   r   )r3   z'list[tuple[torch.Tensor, torch.Tensor]])Fr   i   )rv   r8   rB   r   r:   r   rA   r   )r^   r8   r:   r   rA   r   )r   r   r^   r8   )r   r   r^   r8   r)   r   r   r   rB   r   r   r8   )
r   r   r.   r8   r   r8   r;   r   rB   r   )r(   r   r   r   r   r   r   z	list[str]r)   r   r;   r   rB   r   r,   r   )
__future__r   r&   rN   r!   transformersr   r   transformers.cache_utilsr   onnxruntimer   r   r   r/   r>   rC   r`   r4   r5   rx   r@   r   r   r   r    r   r   <module>r      s   #   2 1 2(   	
  P ))) ) 	)
 ) ) ) )z "000 0 	0
 0 0 0 0 0 0 0h000 0 	0
 0 0 0 0j	 #	  	0	((*(*(* (* 	(*
 (* (*Z,,%),48,DH,\`,`ddd d 	d
 d d d dr   