U hI@sddlmZddlmZmZmZmZddlZddl m Z ddl m Z ddl mZmZddlmZmZmZmZddlmZeeZGd d d ZGd d d e ZdS) ) getLogger)ListOptionalTupleUnionN)Fusion)AttentionMaskFormat) FusionUtils NumpyHelper) NodeProto TensorProtohelper numpy_helper) OnnxModelc@sLeZdZdZedddZedddZdd Zd d Z e e d d dZ dS) AttentionMask: Fuse Attention subgraph into one Attention node. )modelcCs2||_i|_i|_t||_tj|_||_ dSN) r mask_indice mask_castedr utilsrZ MaskIndexEnd mask_formatZget_opset_version opset_version)selfrrM/tmp/pip-unpacked-wheel-socb9apf/onnxruntime/transformers/fusion_attention.py__init__s  zAttentionMask.__init__rcCs ||_dSrr)rrrrrset_mask_format!szAttentionMask.set_mask_formatcCs*||jkr||j|kst||j|<dSr)rAssertionError)rmask mask_indexrrrset_mask_indice$s zAttentionMask.set_mask_indicecCs t|jdksttt|jSNr)lenrrnextiter)rrrrget_first_mask)szAttentionMask.get_first_mask)inputreturnc Csb|jtjkrdS||jkr$|j|S|j|rB|j|\}}n|j|\}}d}|rd||j |<|jtj kr~||j|<|S|j d}|j dkrt jd|g|g|j ddd}|jt ddgt d d gnpd }|j|dkr|jt j|tjdgdgd d t jd||g|g|j ddd}|jt d d g|j|||j|<|S)NTr! Z ReduceSumZ MaskReduceSuminputsoutputsnameaxesZkeepdimsrZort_const_1_reduce_sum_axesFr. data_typedimsvalsraw)rrZNoMaskrrZfind_graph_inputrZcast_graph_input_to_int32Zcast_input_to_int32rrcreate_node_namerr make_node attributeextendmake_attributeget_initializeradd_initializerZ make_tensorr INT64add_node)rr(ZcastedZ input_nameZ cast_nodeZ output_nameZmask_index_nodeZ axes_namerrr process_mask-sV          $    zAttentionMask.process_maskN) __name__ __module__ __qualname____doc__rrrrr"r'strr?rrrrrs  rcseZdZdZdddddgfeeeeeeee e dfdd Z e e eefd d d Ze e eefd d dZe dddZe dddZe e e dddZe e e e fdddZe e e dddZe e dddZe ee dfee dfe ee dfdd d!Ze e e e ee dfee dfeee dfd"d#d$Zd.e ee e dfee e dfe ee dfee dfeee e e e e e e eee dfd&d'd(Zd/e e e e e e e eee e e e e e e eeeee dfd)d*d+Zd,d-ZZS)0FusionAttentionrNFSkipLayerNormalizationLayerNormalization)r hidden_size num_headsattention_maskuse_multi_head_attention!disable_multi_head_attention_biassearch_op_typesc s\|rdnd}t|||||_||_|r0|nt||_||_||_d|_d|_ d|_ dS)NMultiHeadAttention AttentionT) superrrHrIrrJrKrLmask_filter_valuenum_heads_warninghidden_size_warning) rrrHrIrJrKrLrMZattention_op_name __class__rrrms zFusionAttention.__init__)concatr)cCst|jdkrv|j|jd}|j|jd}t|tjrv|jdkrvt|tjrv|jdkrv|d|d|dfS|j|j fS)aU Detect num_heads and hidden_size from Concat node in the following subgraph: SkipLayerNormalization or EmbedLayerNormalization / | MatMul Shape | | Add Gather(indices=0) | | | Unsqueeze | | | Concat (*, -1, 12, 64) | / Reshape | Transpose r0r) r$r(rZget_constant_value isinstancenpZndarraysizerIrH)rrVrI head_sizerrr)get_num_heads_and_hidden_size_from_concats  z9FusionAttention.get_num_heads_and_hidden_size_from_concat) reshape_qr)cCsR|j|jd}|dkrf|j|d}|dk rD|jdkrD||St|jdd|j|j fSt |}t |dks|ddks|ddkrtd |d |j|j fS|d}|d}||}|jdkr ||jkr |j r td |jd |d d|_ |j dkrJ||j krJ|jrJtd|j d |d d|_||fS)zDetect num_heads and hidden_size from a reshape node. Args: reshape_q (NodeProto): reshape node for Q Returns: Tuple[int, int]: num_heads and hidden_size r0NConcatz is not initializer.rWrXrrYzq_shape_value=z7. Expected value are like [0, 0, num_heads, head_size].z--num_heads is z. Detected value is z. Using detected value.Fz--hidden_size is )rr;r(Z get_parentop_typer^loggerdebugrIrHr to_arrayr$rRwarningrS)rr_Zq_shaperVZ q_shape_valuerIr]rHrrrget_num_heads_and_hidden_sizes2    $ z-FusionAttention.get_num_heads_and_hidden_sizeadd_qkcCs|jjdd}|dkrdS||jd}||jd}|dksJ|dkr`td|ddS||kr~td|ddS|jdS) NT)updaterr0zone of the inputs of z is Nonezthe shape of two inputs of z is not same)rZinfer_runtime_shapeZget_edge_shaper(rbrc)rrhZ shape_inferZ input_0_shapeZ input_1_shaperrrget_add_qk_strszFusionAttention.get_add_qk_strcsdttfdd|j}t|dkr0St|dks@t|jd}tjdfddt |j Dg|dd }|j ||j |j |<S) NZ_maskcs|jdkSr#)output)node)mask_output_namerrz0FusionAttention.reshape_add_qk..r0rr`csg|]}qSrr).0_rgrr sz2FusionAttention.reshape_add_qk..r,r-r.axis)listfilter nodes_to_addr$rrr6r r7rangerIappendthis_graph_namenode_name_to_graph_name)rrhZ concat_nodeconcat_node_nameZconcat_add_qk_fp32r)rhrmrreshape_add_qks     zFusionAttention.reshape_add_qk)past_kpast_vr)c Cs|jd}|jd}|ddd}|ddd}tjd|g|g|dgd}tjd|g|g|dgd}|j||j||j|j|<|j|j|<|jd} |dd ddd d } tjd||g| g| dd } |j| |j|j| <| S) zConcatenate past_k and past_v inputs to create past_kv input. Args: past_k (str): name of past K value past_v (str): name of past V value Returns: kv_output_name (str): name of past KV value UnsqueezeZ_5d.rqr)r,r-r.r/r`z.valuez.kv_valueZ_kvrs) rr6replacer r7rwryrzr{) rr~rZunsqueeze_k_nameZunsqueeze_v_nameZ k_5d_nameZ v_5d_nameZk_5dZv_5dr|Zkv_output_name concat_kvrrrrsD         zFusionAttention.concat_kvc Csd}|j|}|dkrJtjtjdd|jjgdd|d}|j||j|j d}|j d}|d  d d }|d  d d }t j d||g|g|d } t j d||g|g|d } |j | |j | |j|j|<|j|j|<||fS) ahReshape past_k and past_v from 4D to 3D to use as inputs for multihead attention node. Args: past_k (str): name of past K value of shape 4D past_v (str): name of past V value of shape 4D Returns: k_3d (str): name of past K value of shape 3D v_3d (str): name of past V value of shape 3D Z kv_4d_to_3dNrint64Zdtyper.ReshapeZ_3drrqr+)rr;r from_arrayr[arrayrHr<rzr6rr r7rwryr{) rr~rZ new_dims_nameZnew_dimsZreshape_k_nameZreshape_v_nameZ k_3d_nameZ v_3d_nameZk_3dZv_3drrr reshape_kv.s:       zFusionAttention.reshape_kv)present_k_namepresent_v_namekv_nodec Csd\}}|j|}|j|}|dkrPtjtjddd|d}|j||j|dkrtjtjddd|d}|j||j|jd}|jd} t j d||g|g|dd } t j d||g|g| dd } |j | |j | |j|j |<|j|j | <dS) a?Split kv_node containing present KV values into separate present K and present V values. Args: present_k_name (str): name of output to store present K value in present_v_name (str): name of output to store present V value in kv_node (str): name of present KV values )Zindex_0Zindex_1Nrrrrr0ZGatherrs)rr;rrr[rr<rzr6r r7rwryr{) rrrrZk_indexZv_indexZk_dimZv_dimZ gather_k_nameZ gather_v_name present_k present_vrrrsplit_kv_s:        zFusionAttention.split_kv)r~rc Cs|ddd}|ddd}|jd}|jd}tjd|g|g|ddddgd }tjd|g|g|ddddgd }|j||j||j|j|<|j|j|<||fS) a}Transpose past_k and past_v from (B,N,P,H) to (B,P,N,H) Args: past_k (str): name of past K value of shape (B,N,P,H) past_v (str): name of past V value of shape (B,N,P,H) Returns: past_k_transpose (str): name of past K value of shape (B,P,N,H) past_v_transpose (str): name of past V value of shape (B,P,N,H) Z _transposedrrq TransposerrXr0rY)r,r-r.perm) rrr6r r7rwryrzr{) rr~rZpast_k_transposeZpast_v_transposeZtranspose_k_nameZtranspose_v_nameZ transpose_kZ transpose_vrrr transpose_kvs.         zFusionAttention.transpose_kv)q_addk_addv_add name_prefixr)cCs|j|jdp"|j|jd}t|}t|}t|}|dk rx|j|jdpl|j|jd} t| }|dk r|j|jdp|j|jd} t| }tj|||fdd} dt|j } |d} |j | |j | g| d| S)Nr0rrtrY _qkv_biasr.r2r3r4) rr;r(r rdr[Z zeros_likestackprodshaper<r2)rrrrrq_biasqbkbvbk_biasv_biasqkv_bias qkv_bias_dim bias_namerrrcreate_combined_qkv_biass($   $ $ z(FusionAttention.create_combined_qkv_bias)q_matmulk_matmulv_matmulrrrrIr)c$Cs"|jd}|jd|jdkr4|jd|jdks8t|j|jd} |j|jd} |j|jd} t| } t| } t| }| j| jkr| j|jkst| jd}tj | | |fdd |d|f}|d}|j || j |jd|jdg|d|d}t jd|jd|g|g|d }|j|j|<|g}|d }|j |tjdgdgd d |d }|j |tjdg|gd d |d}|j |tjdgd|gd d |d}|j |tjdgd|gd d |d}|j |tjdgdgd d |d}t jd||||g|g|jdd }|j|j|j<|d}t jd||||g|g|jdd }|j|j|j<|d}t jd||||g|g|jdd }|j|j|j<|} |}!|}"||||g|jr |dk r,|j|jdrdnd}#tt|j|j|#r,||jd|#<|} |||j|j|j<|dk r|j|jdrNdnd}#tt|j|j|#r||jd|#<|}!|||j|j|j<|dk r |j|jdrdnd}#tt|j|j|#r ||jd|#<|}"|||j|j|j<|j|| |!|"fS)aCreate packed QKV MatMul node before MultiHeadAttention node. This is for the scenario where an Attention node should be created but cannot be created because past_key and past_value are separate inputs and not one concatenated input. Args: q_matmul (NodeProto): name of MatMul from Q path - (batch_size, sequence_length, hidden_size) k_matmul (NodeProto): name of MatMul from K path - (batch_size, sequence_length, hidden_size) v_matmul (NodeProto): name of MatMul from V path - (batch_size, sequence_length, hidden_size) q_add (NodeProto): name of Add from Q path k_add (NodeProto): name of Add from K path v_add (NodeProto): name of Add from V path num_heads (int): number of heads Returns: Union[NodeProto, None]: the node created or None if failed. MatMulrr0rrY _qkv_weightrZ_qkv_outr+Z_q_start_indexFr1Z_k_start_indexZ_v_start_indexrXZ_end_of_qkv_indexZ_qkv_last_axisrZ_q_outZSliceZ_k_outZ_v_outN)rr6r(rr;r rdrr[rZreshaper<r2r r7rzr{r r=r.r9rLanyryrw)$rrrrrrrrIZmatmul_node_nameq_weightk_weightv_weightqwkwvwd qkv_weightZqkv_weight_nameZqkv_matmul_outputZ qkv_matmul qkv_nodesZ q_slice_nameZ k_slice_nameZ v_slice_nameZend_of_qkv_nameZqkv_last_axis_nameZq_slice_outputq_sliceZk_slice_outputk_sliceZv_slice_outputv_sliceZq_outputZk_outputZv_outputZinitializer_inputrrrcreate_packed_qkv_matmul_nodes ,    "                  z-FusionAttention.create_packed_qkv_matmul_node)rrrrrrrIrHrkkey_padding_maskrhr~rrr packed_qkvr)c Cs`|dks t|dkr:||dkr:td|d|dStdd|jjD}tdd|jjD}|jd}g}|r| |||||||\}}}| |jd|jd|jdgnt |t kr.t |t kr.|j r | |jd|jd|jdgn"| |jd|jd|jdgnlt |tkrt |tkr||kr||kr|j r~| |jd||gn| |jd||gndS|j s|||||}||n |d | r| r| |kr| |kr| | | | | g| g}|r.|r.||kr.||kr.| ||gtjd |||d }d |_|j td |g|S)a[Create a MultiHeadAttention node. Args: q_matmul (NodeProto): name of MatMul from Q path - (batch_size, sequence_length, hidden_size) k_matmul (NodeProto): name of MatMul from K path - (batch_size, sequence_length, hidden_size) or (batch_size, num_heads, past_sequence_length, head_size) v_matmul (NodeProto): name of MatMul from V path - (batch_size, sequence_length, hidden_size) or (batch_size, num_heads, past_sequence_length, head_size) q_add (NodeProto): name of Add from Q path k_add (NodeProto): name of Add from K path v_add (NodeProto): name of Add from V path num_heads (int): number of attention heads. If a model is pruned, it is the number of heads after pruning. hidden_size (int): hidden dimension. If a model is pruned, it is the hidden dimension after pruning. output (str): output name of MHA key_padding_mask (str): name of key padding mask add_qk (str): name of add after Q x K' past_k (str): name of past K value - (batch_size, num_heads, past_sequence_length, head_size) past_v (str): name of past V value - (batch_size, num_heads, past_sequence_length, head_size) present_k (str): name of present K value - (batch_size, num_heads, sequence_length, head_size) present_v (str): name of present V value - (batch_size, num_heads, sequence_length, head_size) packed_qkv (bool): whether to combine MatMuls from Q, K, V paths Note: This is for the scenario where an Attention node should be created but cannot be created because past_key and past_value are separate inputs and not one concatenated input. Returns: Union[NodeProto, None]: the node created or None if failed. rinput hidden size # is not a multiple of num of heads NcSsg|] }|jqSrrrprlrrrrrszCFusionAttention.create_multihead_attention_node..cSsg|] }|jqSrrrrrrrrsrOrrNr+ com.microsoftrI)rrbrcsetrgraphr(rkr6rr9typer rLrDrryr r7domainr8r:)rrrrrrrrIrHrkrrhr~rrrrZgraph_input_namesZgraph_output_namesZ mha_node_nameZ mha_inputsrrrrZ mha_outputsZmha_noderrrcreate_multihead_attention_node`sj-   $$$      z/FusionAttention.create_multihead_attention_node)r!rrrrrrrIrHr(rk add_qk_strr~rrrscalecausalr)c6Cs||dks t| dkr:| |dkr:td| d|dSd}|dkrZ|dkrZ|dkrZd}|j|jd}|j|jd}|j|jd}d\}}}|r*|j|jdp|j|jd}|j|jdp|j|jd}|j|jdp |j|jd}|r&|r&|r&|s*dS|dkrLt|jdd dSt|}t|}t|}|j |j ks|t|j d}|j d}|j d}||kr|ksnt| dkr| |krt d | d |d d} |j |j krd} t |j dd}!t |j dd}"t |j dd}#d}$| rbt j |||fdd }%|!|"|#}$nt j|||fdd }%d|!}$|r6t|}&t|}'t|}(t |&j })t |'j }*t |(j }+|)|*kr|!ksnt|+|#kst| rt j |&|'|(fdd },|)|*|+}-nt j|&|'|(fdd },d|)}-|jd}.|jsf|j|.d|j||$g|%d|r|j|.d|j|-g|,d|jr| rtddS|jd|jd|jd|.dg}/|dk r|/|tjd|/| g|.d}0n| |.d|r|.dndg}/|dk r |/|n |/d| o2|}1|1rP|| |}2|/|2| dk r~|| }3|1st|/d|/|3| g}4|r|r|dddddd}5|4|5||||5tjd|/|4|.d}0d|0_|0jtd|g|r|0jtddg|dk r.|0jtd|g| rP|0jtd|!|"|#gg|jdk rx|0jtd t |jg|0S)!a+Create an Attention node. Args: mask_index (str): mask input q_matmul (NodeProto): MatMul node in fully connection for Q k_matmul (NodeProto): MatMul node in fully connection for K v_matmul (NodeProto): MatMul node in fully connection for V q_add (NodeProto): Add bias node in fully connection for Q k_add (NodeProto): Add bias node in fully connection for K v_add (NodeProto): Add bias node in fully connection for V num_heads (int): number of attention heads. If a model is pruned, it is the number of heads after pruning. hidden_size (int): hidden dimension. If a model is pruned, it is the hidden dimension after pruning. input (str): input name output (str): output name add_qk_str (str): name of Add node after Q x K' past_k (str): name of input for past K value past_v (str): name of input for past V value present_k (str): name of output to store present K value present_v (str): name of output to store present V value scale: scale before softmax causal: whether it is uni-directional mask. Returns: Union[NodeProto, None]: the node created or None if failed. rrrNTFr0)NNNzl is not an initializer. Please set do_constant_folding=True in torch.onnx.export to unblock attention fusionzInput hidden size (z3) is not same as weight matrix dimension of q,k,v (z:). Please provide a correct input hidden size or pass in 0rrYrOrrrzVMultiHeadAttention does not support relative_position_bias: cannot fuse the attention.rNr+rz.key_keyrrqrrIZunidirectionalrZqkv_hidden_sizesrQ)!rrbrcrr;r(printr rdrrer[rZ concatenaterr6rKr<r2rkryr r7rr}rrrr8r9r:rQfloat)6rr!rrrrrrrIrHr(rkrr~rrrrrZhas_biasrrrrrrrrrZ qw_in_sizeZ kw_in_sizeZ vw_in_sizeZis_qkv_diff_dimsZ qw_out_sizeZ kw_out_sizeZ vw_out_sizeZqkv_weight_dimrrrrZ q_bias_shapeZ k_bias_shapeZ v_bias_shaperrZattention_node_nameZattention_inputsZattention_nodeZ past_existsZpast_kvrmZattention_outputsZ present_kvrrrcreate_attention_nodes.  $$&                               z%FusionAttention.create_attention_nodec6Cs&|}|jdkr0|j|dd}|dk r,|}ndS|j|dddddgdddddg}d}|dk rp|\}}} } } n:|j|ddddgddddg}|dk r|\}}} } ndSg} t|jD]0\} }||krq||djdkrq| |qt| dkrdS| d}|j|d d}|dk r||jd}|dk rht|d krh|d}|jdkrb|jd}ndSn(|dk rt|d kr|jd}ndSn6|jdkr||}|D]}|jdkr|jd}q||}|jd krt|jd kr|jd}||}dd|D}| ddkr dS|j| ddddgddddg}|dkrXt ddS|\}}}}d}d}d}ddddgddddgfddd dgddddgfddddgddd dgfddddgdddd gfdddgdddgfd}d}| D]`\}} |j| | d| d}|dkrq|dkr(d}|dkr6d}|dkrDd}qNq|dkrft ddSd}!d}"d}#|r|\}}#}"}n2|r|\}}!}#}"n|r|\}}}"n |\}}!}}"|j|"ddddgddddg}$|$dkr|j|"dddddgdddddg}$|$dkrt ddS|$d}%|$d}&|$d}'|j|"ddddgddddg}(|(dkr|j|"dddddgdddddg}(|(dkrt d dS|(d})|(d}*d}+d},|r |j |#d!dd"gdddgfd"d#d#gdddgfd$d!dd"gddddgfg|\}}+}n|r|j |#d$d"d#d#gddddgfd"d#d#gdddgfg|\}}+}|!dk r||!},|,dkrt d%|!dSnN|rnF|j |!d d&d$d#d#gdddddgfd d&d#d#gddddgfg|\}}+}|s|+dkrt d'dS|s8t|+dkr8|+djd kr8|j|+d\}}-|-d(kr8|-|_|jd|kr"|'jd|kr"|*jd|kr"|s|j|+djdnd}.|dkr| n| }/||%\}0}1|0dks|1dkrt d)dS||.|'|*||&|)||0|1||/jd|, }2|2dkrdS|j|2|j|j|2j<|dk r|jd}3d*|3}4|jd+|3tjd gtdd|0t|1|0gdd,}5|j t!"d|/jd|5jg|4gd-|3|j|4|jd<|j#$|/| | g|j#$||j#$|j%s|$n |$dd|j#$|j%s|(n |(dd|j#$|j%s|n |ddd|_&dS).NrGZAddrrrrZEinsumr0ZMulrXrFrWcSsg|] }|jqSr)ra)rpchildrrrrrsz(FusionAttention.fuse..rYz&fuse_attention: failed to match v pathFZSoftmaxZDivZWhere)path1path2path3path4path5rTrrz'fuse_attention: failed to match qk pathz&fuse_attention: failed to match q pathrz&fuse_attention: failed to match k pathZExpandZEqualrZCastz4fuse_attention: failed to verify shape inference of ZSubz)fuse_attention: failed to match mask pathizmFailed to detect num_heads and hidden_size for Attention fusion. Please specify those parameters in argument.Zedge_modified_Zshape_modified_tensorr1Zreshape_modified_)'rarZ match_parentZmatch_parent_path enumerater(rkryr$countrbrcitemsZmatch_parent_pathsrjZget_constant_inputrQrJr?rfrerrwrzr{r.r<r r=r[rintr>r r7Znodes_to_remover9rKZ prune_graph)6rZnormalize_nodeZinput_name_to_nodesZoutput_name_to_nodeZ start_nodeZadd_before_layernormrZ einsum_noderqZ reshape_qkvZ transpose_qkvZ matmul_qkvZ other_inputsZ_ir(Z root_inputZmul_before_layernormZ mul_childrenZlayernorm_nodechildrenrZ parent_nodeZchildren_typesZv_nodesZadd_vZmatmul_vZ is_distillZis_distill_addZis_no_mask_attentionZqk_pathsZqk_nodeskvrhZ matmul_qkZwhere_qkZq_nodesr_Zadd_qZmatmul_qZk_nodesZadd_kZmatmul_kZ mask_nodesrZmul_valr!Zattention_last_nodeZ q_num_headsZ q_hidden_sizeZnew_nodeZ unique_indexZnew_edgeZ shape_tensorrrrfuses                                                $ 0          zFusionAttention.fuse)rrrrrrF)rrrrrNF)r@rArBrCrrrrboolrrDrr rr^rfrjr}rrrrrrrrrrr __classcell__rrrTrrEhs )71-)           w frE)loggingrtypingrrrrZnumpyr[Z fusion_baserZfusion_optionsrZ fusion_utilsr r Zonnxr r r rZ onnx_modelrr@rbrrErrrrs    V