U hI@sZddlZddlZddlmZmZddlmZmZddl m Z e e Z GdddeZdS)N) AttentionMaskFusionAttention) TensorProtohelper) OnnxModelcs<eZdZdZeeeedfdd ZddZddZ Z S) FusionBartAttentionz? Fuse Bart Attention subgraph into one Attention node. )model hidden_size num_headsattention_maskcst||||dS)N)super__init__)selfrr r r  __class__R/tmp/pip-unpacked-wheel-socb9apf/onnxruntime/transformers/fusion_bart_attention.pyr szFusionBartAttention.__init__cCs|j|dgdg}|dkr dS|d}|j|dddgdddg} |j|dddgdddg} | dksp| dkrtdS| \} } } | \} }}| jd|ks|jd|krdS|j|dddgdddg}|j|dddgdddg}|dks|dkrdS|d j| jks|d j|jkrdS|j|ddd gdddg}|j|ddd gdddg}|j|ddd gdddg}|dks|dks|dkrdS|d }|d }|d }| jd}|jd|ks|jd|ks|jd|krdSd S) NConcatFr UnsqueezeZGatherZShapeMulT)rmatch_parent_pathinputnameoutput)r reshape_qkv_2 reshape_qkv_1 reshape_q_2 reshape_k_2 reshape_v_2 root_inputZconcat_qkv_2_pathZ concat_qkv_2Zreshape_qkv_2_path_1Zreshape_qkv_2_path_2_Zgather_1Zshape_1Zgather_2Zshape_2Zreshape_qkv_1_path_1Zreshape_qkv_1_path_2Zreshape_q_2_pathZreshape_k_2_pathZreshape_v_2_pathmul_qZmul_kZmul_vZ gather_1_outrrrcheck_runtime_shape_paths<   $ 0z,FusionBartAttention.check_runtime_shape_pathcOCs |j|ddddddgddddddg}|dk rB|\}}}}} } ndSg} |jD],} | |kr^qP| |djdkrrqP| | qPt| dkrdS| d} || }|jdkr|j|d}|jD]6}|sq||}dd|D}|ddkr|} qqt d d|j jD}t d d|j jD}|j| dddddgdddddg}|j| dd ddddgddddddg}|j| dgdg}d \}}d \}}|dk r|\}}}}}|jd}n|dk r|\}}}}}}|}|jd}|jd}n|dk rx|djd|krx|}|djd}|djd}||krt t dd|j |}t|dkrr|djdnd}ntddS||kr|nd}||kr|nd}|j| ddgddg} |j| dddddgdddddg}!| dk r| \}"}#| }$n"|!dk r|!\}"}"}%}"}#|!}$ndS|j|#ddddddgddddddg}&|&dk rd|&\}'}(})}*}+},ndS|j|#ddddddgddddddg}-|j|#dddddgdddddg}.|j|#ddd dddgddddddg}/|j|#ddgddg}0d \}1}2d\}3}4}5|-dk r&|-\}"}3}6}4}7}5|-}8n|.dk rN|.\}"}3}6}4}5|.}8|6jd}2n|/dk r|/\}"}3}9}"}4}5|/}8|9jd}1|9jd}2n|0dk r|0djd|kr|0}8|8djd}1|8djd}2|2|kr t t dd|j |1}:t|:dkr|:djdnd}2ndS|1|kr|1nd}1|2|kr,|2nd}2|8|.|/fkr|j|jdjd};d}<|j|<}=|=dkr|j|td|<|5jdg|4jg|>}7|1s||| |'|3|| sdS|1o|o|5dkodtk}?|? o6|5jd| ko6|,jd| ko6|jd| k}@|? ox|,jd| kox|5jd|jdkox|5jd|,jdk}A|@o|$| k}B|@o|$|!k}C|Bo|1o|}D|Ao|$| k}E|?o|$| k}Fd}G|Cr2|j|%dgdg}H|j|%ddddgddddg}I|Idk r|Idjd}Gn|Hdk r2|Hdjd}G|BsP|CsP|DsP|EsP|F r|}J||)\}K}L|Kdks|Ldks|L|KdkrtddSd}M|Ds|Es|Fr6|jr0|j |,|Es|Dr|5n|1|Es|Dr|n||+|Es|Dr|7nd|Es|Dr|nd|K|L|Jjd|Dr|1nd|Dr"|nd|2||Dd nd}MnL|j}Nd!|_|j!d|,|5||+|7||K|L| |Jjd|Crl|Gnd|1||2|d"}M|N|_|MdkrdS|j"|M|j#|j$|Mj<|j%&|J|| g|j%&|$|Ds|Es|F r~|&djdkr|&'|8djdk r |8'|djdk r"|'|j( r~|E s6|F r~|&djdk rN|&'|8djdk rf|8'|djdk r~|'|j%&|&|j%&|8|j%&|d#|_)dS)$NZAddZMatMulZReshapeZ TransposerrcSsg|] }|jqSrop_type).0childrrr sz,FusionBartAttention.fuse..cSsg|] }|jqSrrr(noderrrr*scSsg|] }|jqSrr+r,rrrr*sr)r.)NNrcSs |jdkSNZIdentityr&r-rrrz*FusionBartAttention.fuse..r.z&fuse_attention: failed to match v pathZSoftmaxr)NNNcSs |jdkSr/r&r0rrrr1r2Z empty_biasg)Zdtype)dimsvalsmatmul_vZWhereZExpandrz9fuse_attention: failed to detect num_heads or hidden_size)past_kpast_v present_k present_vZ packed_qkvF)Z add_qk_strr6r7r8r9T)*rrrrappendlenr'Z get_childrencountsetgraphlistfilterinput_name_to_nodesloggerdebugZget_initializerr3Zadd_initializerrFLOATnparrayZfloat32Zcreate_node_namerZ make_noderr%localsZget_num_heads_and_hidden_sizeZuse_multi_head_attentionZcreate_multihead_attention_nodeZcreate_attention_nodeZ nodes_to_addZthis_graph_nameZnode_name_to_graph_nameZnodes_to_removeextendpopZ!disable_multi_head_attention_biasZ prune_graph)OrZnormalize_noderAZoutput_name_to_nodeZ qkv_nodesZadd_outZ matmul_outrZ transpose_qkvrZ matmul_qkvZ other_inputsrr"Zskip_layernormrchildrenZchildren_typesZgraph_input_namesZgraph_output_namesZv_nodesZv_nodes_with_past_self_attnZv_nodes_with_past_cross_attnr7r9r!Zadd_vZ transpose_vZ reshape_v_1r5Zconcat_vZidentity_node_vZ qk_nodes_1Z qk_nodes_2r#Z matmul_qkZqk_nodesZadd_qkZq_nodesrZ transpose_qZ reshape_q_1r$Zadd_qZmatmul_qZk_nodes_with_biasZk_nodes_no_biasZ#k_nodes_no_bias_with_past_self_attnZ$k_nodes_no_bias_with_past_cross_attnr6r8r Z reshape_k_1Zmatmul_kZ transpose_k_1Zadd_kZk_nodesZconcat_kZidentity_node_kZbias_dimZempty_bias_nameZ empty_tensorZadd_nameZthree_root_inputsZone_root_inputZtwo_root_inputsZencoder_attentionZdecoder_attentionZdecoder_attention_with_pastZdecoder_cross_attentionZ!decoder_cross_attention_with_pastZ mask_indexZmask_nodes_bartZmask_nodes_whisperZattention_last_noder r Znew_nodeZ%use_multi_head_attention_ground_truthrrrfuseMsJ               "               "              "          zFusionBartAttention.fuse) __name__ __module__ __qualname____doc__rintrr r%rK __classcell__rrrrrs 0r)loggingZnumpyrEZfusion_attentionrrZonnxrrZ onnx_modelr getLoggerrLrBrrrrrs