o

    hhW                     @   s   d dl Z d dlZd dlZd dlmZ d dlZd dlmZ d dlm  m	Z
 ddlmZm
Z
mZmZmZ ddlmZ ddlmZmZmZ d dlmZ G d	d
 d
ejZG dd deZG d
d deZG dd deZG dd deZdS )    N)Sequence   )build_activation_layer
build_dropoutr   build_conv_layerbuild_norm_layer)Linear   )
BaseModule
ModuleList
Sequential)	to_2tuplec                       s2   e Zd ZdZd
 fdd	Zdd Zdd	 Z  ZS )AdaptivePaddinga  Applies padding adaptively to the input.

    This module can make input get fully covered by filter
    you specified. It support two modes "same" and "corner". The
    "same" mode is same with "SAME" padding mode in TensorFlow, pad
    zero around input. The "corner"  mode would pad zero
    to bottom right.

    Args:
        kernel_size (int | tuple): Size of the kernel. Default: 1.
        stride (int | tuple): Stride of the filter. Default: 1.
        dilation (int | tuple): Spacing between kernel elements.
            Default: 1.
        padding (str): Support "same" and "corner", "corner" mode
            would pad zero to bottom right, and "same" mode would
            pad zero around input. Default: "corner".

    Example:
        >>> kernel_size = 16
        >>> stride = 16
        >>> dilation = 1
        >>> input = torch.rand(1, 1, 15, 17)
        >>> adap_pad = AdaptivePadding(
        >>>     kernel_size=kernel_size,
        >>>     stride=stride,
        >>>     dilation=dilation,
        >>>     padding="corner")
        >>> out = adap_pad(input)
        >>> assert (out.shape[2], out.shape[3]) == (16, 32)
        >>> input = torch.rand(1, 1, 16, 17)
        >>> out = adap_pad(input)
        >>> assert (out.shape[2], out.shape[3]) == (16, 32)
    r	   cornerc                    sN   t t|   |dv s
J t|}t|}t|}|| _|| _|| _|| _d S )N)samer   )superr   __init__r
   paddingkernel_sizestridedilation)selfr   r   r   r   	__class__ 5/root/Awesome-Backbones/configs/common/transformer.pyr   3   s   
zAdaptivePadding.__init__c                 C   s   |\}}| j \}}| j\}}t|| }t|| }	t|d | |d | jd   d | d}
t|	d | |d | jd   d | d}|
|fS )zCalculate the padding size of input.

        Args:
            input_shape (:obj:`torch.Size`): arrange as (H, W).

        Returns:
            Tuple[int]: The padding size along the
            original H and W directions
        r	   r   )r   r   mathceilmaxr   )r   input_shapeinput_hinput_wkernel_hkernel_wstride_hstride_woutput_houtput_wpad_hpad_wr   r   r   
get_pad_shape@   s0   


zAdaptivePadding.get_pad_shapec              	   C   s   |  | dd \}}|dks|dkrA| jdkr&t|d|d|g}|S | jdkrAt||d ||d  |d ||d  g}|S )zAdd padding to `x`

        Args:
            x (Tensor): Input tensor has shape (B, C, H, W).

        Returns:
            Tensor: The tensor with adaptive padding
        Nr   r   r   r   )r*   sizer   Fpad)r   xr(   r)   r   r   r   forwardU   s   	


zAdaptivePadding.forward)r	   r	   r	   r   )__name__
__module____qualname____doc__r   r*   r0   
__classcell__r   r   r   r   r      s
    "
r   c                       s@   e Zd ZdZ														d fd
d	Zdd
 Z  ZS )
PatchEmbeda  Image to Patch Embedding.

    We use a conv layer to implement PatchEmbed.

    Args:
        in_channels (int): The num of input channels. Default: 3
        embed_dims (int): The dimensions of embedding. Default: 768
        conv_type (str): The type of convolution
            to generate patch embedding. Default: "Conv2d".
        kernel_size (int): The kernel_size of embedding conv. Default: 16.
        stride (int): The slide stride of embedding conv.
            Default: 16.
        padding (int | tuple | string): The padding length of
            embedding conv. When it is a string, it means the mode
            of adaptive padding, support "same" and "corner" now.
            Default: "corner".
        dilation (int): The dilation rate of embedding conv. Default: 1.
        bias (bool): Bias of embed conv. Default: True.
        norm_cfg (dict, optional): Config dict for normalization layer.
            Default: None.
        input_size (int | tuple | None): The size of input, which will be
            used to calculate the out size. Only works when `dynamic_size`
            is False. Default: None.
        init_cfg (`mmcv.ConfigDict`, optional): The Config for initialization.
            Default: None.
          Conv2d   r   r	   TNc              
      s  t t| j|d || _|d u r|}t|}t|}t|}t|tr/t||||d| _d}nd | _t|}t	t
|d|||||||d| _|	d urSt|	|d | _
nd | _
|
rt|
}
|
| _| jrz| j|
\}}
|
\}}|| }||
 }||f}
|
d d|d   |d |d d   d |d  d }|
d d|d   |d |d d   d |d  d }||f| _d S d | _d | _d S )Ninit_cfgr   r   r   r   r   type)in_channelsout_channelsr   r   r   r   biasr	   r   )r   r6   r   
embed_dimsr
   
isinstancestrr   adaptive_paddingr   dict
projectionr   normZinit_input_sizer*   
init_out_size)r   r@   rC   	conv_typer   r   r   r   rB   norm_cfg
input_sizer<   r(   r)   r    r!   h_outw_outr   r   r   r      sx   




zPatchEmbed.__init__c                 C   s\   | j r|  |}| |}|jd |jd f}|ddd}| jdur*| |}||fS )aW  
        Args:
            x (Tensor): Has shape (B, C, H, W). In most case, C is 3.

        Returns:
            tuple: Contains merged results and its spatial shape.

            - x (Tensor): Has shape (B, out_h * out_w, embed_dims)
            - out_size (tuple[int]): Spatial shape of x, arrange as
              (out_h, out_w).
        r   r7   r	   N)rF   rH   shapeflatten	transposerI   )r   r/   out_sizer   r   r   r0      s   



zPatchEmbed.forward)r7   r8   r9   r:   r:   r   r	   TNNN)r1   r2   r3   r4   r   r0   r5   r   r   r   r   r6   i   s    Hr6   c                       s@   e Zd ZdZdddddeddddf fd	d
	Zdd Z  ZS )
PatchMerginga  Merge patch feature map. Modified from mmcv, which uses pre-norm layer
    whereas Swin V2 uses post-norm here. Therefore, add extra parameter to
    decide whether use post-norm or not.

    This layer groups feature map by kernel_size, and applies norm and linear
    layers to the grouped feature map ((used in Swin Transformer)).
    Our implementation uses `nn.Unfold` to
    merge patches, which is about 25% faster than the original
    implementation. However, we need to modify pretrained
    models for compatibility.

    Args:
        in_channels (int): The num of input channels.
            to gets fully covered by filter and stride you specified.
        out_channels (int): The num of output channels.
        kernel_size (int | tuple, optional): the kernel size in the unfold
            layer. Defaults to 2.
        stride (int | tuple, optional): the stride of the sliding blocks in the
            unfold layer. Defaults to None. (Would be set as `kernel_size`)
        padding (int | tuple | string ): The padding length of
            embedding conv. When it is a string, it means the mode
            of adaptive padding, support "same" and "corner" now.
            Defaults to "corner".
        dilation (int | tuple, optional): dilation parameter in the unfold
            layer. Default: 1.
        bias (bool, optional): Whether to add bias in linear layer or not.
            Defaults to False.
        norm_cfg (dict, optional): Config dict for normalization layer.
            Defaults to dict(type='LN').
        is_post_norm (bool): Whether to use post normalization here.
            Defaults to False.
        init_cfg (dict, optional): The extra config for initialization.
            Defaults to None.
    r   Nr   r	   FLNr>   c                    s   t  j|
d || _|| _|	| _|r|}n|}t|}t|}t|}t|tr4t||||d| _	d}nd | _	t|}t
j||||d| _|d |d  | }t
j
|||d| _|d urs| jrit||d | _d S t||d | _d S d | _d S )Nr;   r=   r   )r   r   r   r   r	   )rB   )r   r   r@   rA   is_post_normr
   rD   rE   r   rF   nnUnfoldsamplerr   	reductionr   rI   )r   r@   rA   r   r   r   r   rB   rL   rV   r<   
sample_dimr   r   r   r     sB   

zPatchMerging.__init__c                 C   st  |j \}}}t|tsJ d| |\}}||| ks J d|||||g d}| jr>| |}|j dd \}}| |}|d| jjd   | jjd | jj	d d   d | jj
d  d }|d| jjd   | jjd | jj	d d   d | jj
d  d }	||	f}
|dd}| jr| 
|}| jr| |n|}||
fS | jr| |n|}| 
|}||
fS )	a  
        Args:
            x (Tensor): Has shape (B, H*W, C_in).
            input_size (tuple[int]): The spatial shape of x, arrange as (H, W).
                Default: None.

        Returns:
            tuple: Contains merged results and its spatial shape.

            - x (Tensor): Has shape (B, Merged_H * Merged_W, C_out)
            - out_size (tuple[int]): Spatial shape of x, arrange as
              (Merged_H, Merged_W).
        z(Expect input_size is `Sequence` but get zinput feature has wrong size)r   r7   r	   r   r+   Nr   r   r	   )rP   rD   r   viewpermuterF   rY   r   r   r   r   rR   rV   rZ   rI   )r   r/   rM   BLCHWout_hout_woutput_sizer   r   r   r0   @  sL   





zPatchMerging.forwardr1   r2   r3   r4   rG   r   r0   r5   r   r   r   r   rT      s    &8rT   c                       sL   e Zd ZdZddedddddf fdd	Z							dd	d
Z  ZS )MultiheadAttentionab  A wrapper for ``torch.nn.MultiheadAttention``.

    This module implements MultiheadAttention with identity connection,
    and positional encoding  is also passed as input.

    Args:
        embed_dims (int): The embedding dimension.
        num_heads (int): Parallel attention heads.
        attn_drop (float): A Dropout layer on attn_output_weights.
            Default: 0.0.
        proj_drop (float): A Dropout layer after `nn.MultiheadAttention`.
            Default: 0.0.
        dropout_layer (obj:`ConfigDict`): The dropout_layer used
            when adding the shortcut.
        init_cfg (obj:`mmcv.ConfigDict`): The Config for initialization.
            Default: None.
        batch_first (bool): When it is True,  Key, Query and Value are shape of
            (batch, n, embed_dim), otherwise (n, batch, embed_dim).
             Default to False.
            Dropout)r?   	drop_probNFc           	         s   t t| | d|v rtdt |d }|d|d< || _|| _|| _	t
j|||fi || _t
|| _
|rAt|| _d S t
 | _d S )NdropoutzThe arguments `dropout` in MultiheadAttention has been deprecated, now you can separately set `attn_drop`(float), proj_drop(float), and `dropout_layer`(dict) rj   )r   rg   r   warningswarnDeprecationWarningpoprC   	num_headsbatch_firstrW   attnri   	proj_dropr   Identity
dropout_layer)	r   rC   rp   	attn_droprs   ru   r<   rq   kwargsr   r   r   r     s*   	
zMultiheadAttention.__init__c	                 K   s   |du r|}|du r|}|du r|}|du r.|dur.|j |j kr#|}ntd| jj d |dur6|| }|dur>|| }| jrS|dd}|dd}|dd}| j|||||dd }
| jrh|
dd}
|| | 	|
 S )a  Forward function for `MultiheadAttention`.

        **kwargs allow passing a more general data flow when combining
        with other operations in `transformerlayer`.

        Args:
            query (Tensor): The input query with shape [num_queries, bs,
                embed_dims] if self.batch_first is False, else
                [bs, num_queries embed_dims].
            key (Tensor): The key tensor with shape [num_keys, bs,
                embed_dims] if self.batch_first is False, else
                [bs, num_keys, embed_dims] .
                If None, the ``query`` will be used. Defaults to None.
            value (Tensor): The value tensor with same shape as `key`.
                Same in `nn.MultiheadAttention.forward`. Defaults to None.
                If None, the `key` will be used.
            identity (Tensor): This tensor, with the same shape as x,
                will be used for the identity link.
                If None, `x` will be used. Defaults to None.
            query_pos (Tensor): The positional encoding for query, with
                the same shape as `x`. If not None, it will
                be added to `x` before forward function. Defaults to None.
            key_pos (Tensor): The positional encoding for `key`, with the
                same shape as `key`. Defaults to None. If not None, it will
                be added to `key` before forward function. If None, and
                `query_pos` has the same shape as `key`, then `query_pos`
                will be used for `key_pos`. Defaults to None.
            attn_mask (Tensor): ByteTensor mask with shape [num_queries,
                num_keys]. Same in `nn.MultiheadAttention.forward`.
                Defaults to None.
            key_padding_mask (Tensor): ByteTensor with shape [bs, num_keys].
                Defaults to None.

        Returns:
            Tensor: forwarded results with shape
            [num_queries, bs, embed_dims]
            if self.batch_first is False, else
            [bs, num_queries embed_dims].
        Nz&position encoding of key ismissing in .r   r	   )querykeyvalue	attn_maskkey_padding_mask)
rP   rl   rm   r   r1   rq   rR   rr   ru   rs   )r   ry   rz   r{   identityZ	query_posZkey_posr|   r}   rw   outr   r   r   r0     sB   2
zMultiheadAttention.forward)NNNNNNNrf   r   r   r   r   rg   v  s     
"rg   c                       sD   e Zd ZdZdddeddddd	dd	f fd
d	Zddd
Z  ZS )FFNa  Implements feed-forward networks (FFNs) with identity connection.

    Args:
        embed_dims (int): The feature dimension. Same as
            `MultiheadAttention`. Defaults: 256.
        feedforward_channels (int): The hidden dimension of FFNs.
            Defaults: 1024.
        num_fcs (int, optional): The number of fully-connected layers in
            FFNs. Default: 2.
        act_cfg (dict, optional): The activation config for FFNs.
            Default: dict(type='ReLU')
        ffn_drop (float, optional): Probability of an element to be
            zeroed in FFN. Default 0.0.
        add_identity (bool, optional): Whether to add the
            identity connection. Default: `True`.
        dropout_layer (obj:`ConfigDict`): The dropout_layer used
            when adding the shortcut.
        init_cfg (obj:`mmcv.ConfigDict`): The Config for initialization.
            Default: None.
       i   r   ReLUT)r?   inplacerh   Nc	           
   	      s   t t| | |dksJ d| d|| _|| _|| _|| _t|| _g }
|}t	|d D ]}|

tt||| jt
| |}q/|

t|| |

t
| t|
 | _|r_t|ntj
 | _|| _d S )Nr   z&num_fcs should be no less than 2. got rx   r	   )r   r   r   rC   feedforward_channelsnum_fcsact_cfgr   activaterangeappendr   r   rW   ri   layersr   torchrt   ru   add_identity)
r   rC   r   r   r   ffn_dropru   r   r<   rw   r   r@   _r   r   r   r     s:   




zFFN.__init__c                 C   s4   |  |}| js
| |S |du r|}|| | S )zoForward function for `FFN`.

        The function would add x to the output tensor if residue is None.
        N)r   r   ru   )r   r/   r~   r   r   r   r   r0   @  s   

zFFN.forward)Nrf   r   r   r   r   r     s    
"r   )copyr   rl   typingr   r   torch.nnrW   Ztorch.nn.functional
functionalr-   Zbasic.build_layerr   r   r   r   Zbasic.wrappersr   base_moduler
   r   r   
utils.miscr
   Moduler   r6   rT   rg   r   r   r   r   r   <module>   s$   Y{  