U
    h.                     @   sh   d dl mZ d dlmZ d dlmZ d dlmZ d dlm	Z	 ee
ZG dd deZG dd	 d	eZd
S )    )	getLogger)Dict)Fusion)helper)	OnnxModelc                       s2   e Zd Zed fddZeedddZ  ZS )FusionLayerNormalizationmodelc                    s   t  |dd d S )NLayerNormalization
ReduceMeansuper__init__selfr	   	__class__ M/tmp/pip-unpacked-wheel-socb9apf/onnxruntime/transformers/fusion_layernorm.pyr      s    z!FusionLayerNormalization.__init__input_name_to_nodesoutput_name_to_nodec                 C   s  | j ||}t|dks&t|dkr*dS |jd }|d jdksT|d jd |krXdS t|dkr|d jdks|d jd |krdS d}|D ]$}| j j|d|dd}|dk	r qq|dkrdS | j |d	d
dddgdddddgfd	d
ddddgddddddgfg|\}}	}
|dk rdS |	d }||kr4dS |	d }| j |\}}|dksj|dksj|dkr~t	d|  dS |	d }| j 
|ddkrdS ||jd  d }|jdkrdS ||jd  d }|jd
krdS |g}|| ||	dd  ||||g | j ||j||s8t	d dS |jd| j |jd |  }| j |ddsldS |jd| j |jd |  }| j |ddsdS | j| tjd|jd ||g|jd g| j jdddd}|jtdt|g | j| | j| j|j< dS )a  
        Fuse Layer Normalization subgraph into one node LayerNormalization:
              +----------------------+
              |                      |
              |                      v
          [Root] --> ReduceMean -->  Sub  --> Pow --> ReduceMean --> Add --> Sqrt --> Div --> Mul --> Add
                     (axis=2 or -1)  |      (Y=2)   (axis=2 or -1)  (E-6 or E-12 or 0)    ^
                                     |                                               |
                                     +-----------------------------------------------+

         It also handles cases of duplicated sub nodes exported from older version of PyTorch:
              +----------------------+
              |                      v
              |           +-------> Sub-----------------------------------------------+
              |           |                                                           |
              |           |                                                           v
          [Root] --> ReduceMean -->  Sub  --> Pow --> ReduceMean --> Add --> Sqrt --> Div  --> Mul --> Add
              |                      ^
              |                      |
              +----------------------+
        r      NSub   ZDivF)	recursiveSqrtAddr   ZPowCastg-C6?zHskip SkipLayerNormalization fusion since epsilon value is not expected:    g       @Mulz4It is not safe to fuse LayerNormalization node. Skipzlayernorm weightzlayernorm biasr
   	LayerNormZname_prefixinputsoutputsnameepsilon)r	   Zget_childrenleninputop_typeZfind_first_child_by_typematch_parent_pathsget_constant_inputloggerdebugZfind_constant_inputoutputextendis_safe_to_fuse_nodesZinput_indexZ$is_constant_with_specified_dimensionnodes_to_remover   	make_nodecreate_node_name	attributemake_attributefloatnodes_to_addappendthis_graph_namenode_name_to_graph_namer'   )r   noder   r   childrenZ
root_inputZdiv_nodechildZpath_idparent_nodes_Zsub_nodeZsecond_add_nodeiZ
add_weightZpow_nodeZmul_nodeZlast_add_nodesubgraph_nodesweight_input
bias_inputZnormalize_noder   r   r   fuse   s    
  





zFusionLayerNormalization.fuse__name__
__module____qualname__r   r   r   rF   __classcell__r   r   r   r   r      s   r   c                       s2   e Zd Zed fddZeedddZ  ZS )FusionLayerNormalizationTFr   c                    s   t  |ddd d S )Nr
   r   ZTFr   r   r   r   r   r      s    z#FusionLayerNormalizationTF.__init__r   c                 C   s  g }| j |ddddddddddg
dddd	d	d	dd	d	dg
fddddddd
ddddgdddd	d	d	d	dd	d	dgfg|\}}}|dkrdS t|dkst|d	 dkr|d dkr|d dkstd dS |dd \}}}	}
}}|dd \}}}}d}t|dkr|d }|jd
kst| j |dd	|}|dkrHtd dS | j |d	|}|dkrf|n| j |d	|}|dkrtd dS | j 	|\}}|dks|d	ks|dkr|dkrtd dS |dkr|j
d	 |j
ks|j
d	 |j
krtd dS |dk	rL|j
d	 |j
ks>|j
d	 |j
krLtd dS |j
d	 |j
d krptd dS ||||	|
|||||||g}|dk	r| j |d
d	|}|dkrtd dS ||||g | j ||j| j  | j  s td dS | j| |	j
d }|j
d	 }tjd|j
d	 ||g|jd	 g| j jdddd}|jtdt|g | j| | j| j|j< dS )aU  
         Layer Norm from Tensorflow model(using keras2onnx or tf2onnx):
          +------------------------------------+
          |                                    |
          |                                    |
        (Cast_1)                               |
          |                                    |
          |                                    v                                           (B)                             (B)             (A)
         Add --> (Cast_1) --> ReduceMean -->  Sub  --> Mul --> ReduceMean --> (Cast_3) --> Add --> Sqrt --> Reciprocol --> Mul --> Mul --> Sub --> Add
          |                       |                                                                                         |       ^              ^
          |                       |                                                                                         |       |              |
          |                       +--------------------------------------------------(Cast_2)-------------------------------|-------+              |
          |                                                                                                                 v                      |
          +---------------------------------------------------------------------------------------------------------------> Mul--------------------+
        r   r!   Z
Reciprocalr   r   r   r   Nr   r   r    )r   r   r   z=return indice is exepected in [0, 1], but got {return_indice}      zmul_node_3 not foundzroot node is nonegh㈵>zepsilon is not matchedz;reduce_mean_node_1 and mul_node_3 shall link from root nodez%mul_node_2 shall have two same inputszcast_node_2 not foundz$not safe to fuse layer normalizationr
   r"   r#   r$   r(   )r	   r,   r)   AssertionErrorr.   r/   r+   Zmatch_parentZ
get_parentr-   r*   r1   r2   r0   r   r   r3   r   r4   r5   r6   r7   r8   r9   r:   r;   r<   r'   )r   r=   r   r   Zreturn_indicerA   r@   Z
sub_node_0Z
mul_node_0Z
mul_node_1Zreciprocol_nodeZ	sqrt_nodeZ
add_node_0Zreduce_mean_node_0Z
mul_node_2Z
sub_node_1Zreduce_mean_node_1Zcast_node_3Z
mul_node_3Znode_before_reduceZ	root_noderB   r(   rC   Zcast_node_2rD   rE   Z
fused_noder   r   r   rF      s    !
&$
	




(












zFusionLayerNormalizationTF.fuserG   r   r   r   r   rL      s   rL   N)loggingr   typingr   Zfusion_baser   Zonnxr   Z
onnx_modelr   rH   r.   r   rL   r   r   r   r   <module>   s   v