
     `i              	       @   d Z ddlZddlZddlZddlmZ ddlmZm	Z	 ddl
Z
ddl
mZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZmZ ddlmZmZmZmZ ddlmZ ddlm Z   ej!        e"          Z#e ed           G d de                                  Z$e ed           G d de                                  Z%e ed           G d de                                  Z&e ed           G d de                                  Z'd Z(d Z) G d d ej*                  Z+ G d! d"ej*                  Z, G d# d$ej*                  Z-dLd'e
j.        d(e/d)e0d*e
j.        fd+Z1 G d, d-ej*                  Z2 G d. d/ej*                  Z3 G d0 d1ej*                  Z4 G d2 d3ej*                  Z5 G d4 d5ej*                  Z6 G d6 d7ej*                  Z7 G d8 d9ej*                  Z8 G d: d;e          Z9 G d< d=ej*                  Z:e G d> d?e                      Z;e G d@ dAe;                      Z< edB           G dC dDe;                      Z= edE           G dF dGe;                      Z> edH           G dI dJe;e                      Z?g dKZ@dS )MzPyTorch Swin Transformer model.    N)	dataclass)OptionalUnion)nn   )ACT2FN)GradientCheckpointingLayer)BackboneOutput)PreTrainedModel) find_pruneable_heads_and_indicesmeshgridprune_linear_layer)ModelOutputauto_docstringlogging	torch_int)BackboneMixin   )
SwinConfigzN
    Swin encoder's outputs, with potential hidden states and attentions.
    )custom_introc                       e Zd ZU dZdZeej                 ed<   dZ	ee
ej        df                  ed<   dZee
ej        df                  ed<   dZee
ej        df                  ed<   dS )SwinEncoderOutputa  
    reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
        shape `(batch_size, hidden_size, height, width)`.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
        include the spatial dimensions.
    Nlast_hidden_state.hidden_states
attentionsreshaped_hidden_states)__name__
__module____qualname____doc__r   r   torchFloatTensor__annotations__r   tupler   r        z/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/transformers/models/swin/modeling_swin.pyr   r   *   s           6:x 12999=AM8E%"3S"89:AAA:>Ju0#567>>>FJHU5+<c+A%BCJJJJJr&   r   zV
    Swin model's outputs that also contains a pooling of the last hidden states.
    c                       e Zd ZU dZdZeej                 ed<   dZ	eej                 ed<   dZ
eeej        df                  ed<   dZeeej        df                  ed<   dZeeej        df                  ed<   dS )	SwinModelOutputa  
    pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`, *optional*, returned when `add_pooling_layer=True` is passed):
        Average pooling of the last layer hidden-state.
    reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
        shape `(batch_size, hidden_size, height, width)`.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
        include the spatial dimensions.
    Nr   pooler_output.r   r   r   )r   r   r   r    r   r   r!   r"   r#   r*   r   r$   r   r   r%   r&   r'   r)   r)   @   s         	 	 6:x 1299915M8E-.555=AM8E%"3S"89:AAA:>Ju0#567>>>FJHU5+<c+A%BCJJJJJr&   r)   z*
    Swin masked image model outputs.
    c                      e Zd ZU dZdZeej                 ed<   dZ	eej                 ed<   dZ
eeej        df                  ed<   dZeeej        df                  ed<   dZeeej        df                  ed<   ed	             ZdS )
SwinMaskedImageModelingOutputa  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `bool_masked_pos` is provided):
        Masked image modeling (MLM) loss.
    reconstruction (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
        Reconstructed pixel values.
    reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
        shape `(batch_size, hidden_size, height, width)`.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
        include the spatial dimensions.
    Nlossreconstruction.r   r   r   c                 D    t          j        dt                     | j        S )Nzlogits attribute is deprecated and will be removed in version 5 of Transformers. Please use the reconstruction attribute to retrieve the final output instead.)warningswarnFutureWarningr.   selfs    r'   logitsz$SwinMaskedImageModelingOutput.logitss   s*    ]	
 	
 	

 ""r&   )r   r   r   r    r-   r   r!   r"   r#   r.   r   r$   r   r   propertyr5   r%   r&   r'   r,   r,   Y   s           )-D(5$
%,,,26NHU./666=AM8E%"3S"89:AAA:>Ju0#567>>>FJHU5+<c+A%BCJJJ# # X# # #r&   r,   z0
    Swin outputs for image classification.
    c                       e Zd ZU dZdZeej                 ed<   dZ	eej                 ed<   dZ
eeej        df                  ed<   dZeeej        df                  ed<   dZeeej        df                  ed<   dS )	SwinImageClassifierOutputa7  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Classification (or regression if config.num_labels==1) loss.
    logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
        Classification (or regression if config.num_labels==1) scores (before SoftMax).
    reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
        shape `(batch_size, hidden_size, height, width)`.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
        include the spatial dimensions.
    Nr-   r5   .r   r   r   )r   r   r   r    r-   r   r!   r"   r#   r5   r   r$   r   r   r%   r&   r'   r8   r8   }   s           )-D(5$
%,,,*.FHU&'...=AM8E%"3S"89:AAA:>Ju0#567>>>FJHU5+<c+A%BCJJJJJr&   r8   c                     | j         \  }}}}|                     |||z  |||z  ||          } |                     dddddd                                                              d|||          }|S )z2
    Partitions the given input into windows.
    r   r   r            shapeviewpermute
contiguous)input_featurewindow_size
batch_sizeheightwidthnum_channelswindowss          r'   window_partitionrJ      s     /<.A+J|!&&Fk);8Lk[g M ##Aq!Q155@@BBGGKYdfrssGNr&   c                     | j         d         }|                     d||z  ||z  |||          } |                     dddddd                                                              d|||          } | S )z?
    Merges windows to produce higher resolution features.
    r=   r   r   r   r:   r;   r<   r>   )rI   rD   rF   rG   rH   s        r'   window_reverserL      sx     =$Lll2v4e{6JKYdfrssGooaAq!Q//::<<AA"feUabbGNr&   c            
            e Zd ZdZd fd	Zdej        dededej        fdZ	 	 dd
e	ej
                 de	ej                 dedeej                 fdZ xZS )SwinEmbeddingszW
    Construct the patch and position embeddings. Optionally, also the mask token.
    Fc                 <   t                                                       t          |          | _        | j        j        }| j        j        | _        |r-t          j        t          j
        dd|j                            nd | _        |j        r6t          j        t          j
        d|dz   |j                            | _        nd | _        t          j        |j                  | _        t          j        |j                  | _        |j        | _        || _        d S )Nr   )super__init__SwinPatchEmbeddingspatch_embeddingsnum_patches	grid_size
patch_gridr   	Parameterr!   zeros	embed_dim
mask_tokenuse_absolute_embeddingsposition_embeddings	LayerNormnormDropouthidden_dropout_probdropout
patch_sizeconfig)r4   rc   use_mask_tokenrT   	__class__s       r'   rQ   zSwinEmbeddings.__init__   s     3F ; ;+7/9O]g",u{1a9I'J'JKKKcg) 	,')|EK;QR?TZTd4e4e'f'fD$$'+D$L!122	z&"<== +r&   
embeddingsrF   rG   returnc                    |j         d         dz
  }| j        j         d         dz
  }t          j                                        s||k    r||k    r| j        S | j        ddddf         }| j        ddddf         }|j         d         }|| j        z  }	|| j        z  }
t          |dz            }|                    d|||          }|                    dddd          }t          j
                            ||	|
fdd	
          }|                    dddd                              dd|          }t          j        ||fd          S )a   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   Nr=         ?r   r   r:   bicubicF)sizemodealign_cornersdim)r?   r\   r!   jit
is_tracingrb   r   reshaperA   r   
functionalinterpolater@   cat)r4   rf   rF   rG   rT   num_positionsclass_pos_embedpatch_pos_embedro   
new_height	new_widthsqrt_num_positionss               r'   interpolate_pos_encodingz'SwinEmbeddings.interpolate_pos_encoding   sr    !&q)A-06q9A= y##%% 	,+*F*F6UZ??++2111bqb592111abb59r"t.
T_,	&}c'9::)11!5GI[]`aa)11!Q1==-33i(	 4 
 
 *11!Q1==BB1b#NNy/?;CCCCr&   Npixel_valuesbool_masked_posr|   c                    |j         \  }}}}|                     |          \  }}	|                     |          }|                                \  }
}}|R| j                            |
|d          }|                    d                              |          }|d|z
  z  ||z  z   }| j        '|r|| 	                    |||          z   }n
|| j        z   }| 
                    |          }||	fS )Nr=         ?)r?   rS   r^   rk   rZ   expand	unsqueezetype_asr\   r|   ra   )r4   r}   r~   r|   _rH   rF   rG   rf   output_dimensionsrE   seq_lenmask_tokensmasks                 r'   forwardzSwinEmbeddings.forward   s	    *6);&<(,(=(=l(K(K%
%YYz**
!+!2!2
GQ&/00WbIIK",,R0088EED#sTz2[45GGJ#/' C'$*G*G
TZ\a*b*bb

'$*BB
\\*--
,,,r&   )F)NF)r   r   r   r    rQ   r!   Tensorintr|   r   r"   
BoolTensorboolr$   r   __classcell__re   s   @r'   rN   rN      s              &&D5< &D &DUX &D]b]i &D &D &D &DV 7;).	- -u01- "%"23- #'	-
 
u|	- - - - - - - -r&   rN   c                   t     e Zd ZdZ fdZd Zdeej                 de	ej
        e	e         f         fdZ xZS )rR   z
    This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
    `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
    Transformer.
    c                    t                                                       |j        |j        }}|j        |j        }}t          |t          j        j	                  r|n||f}t          |t          j        j	                  r|n||f}|d         |d         z  |d         |d         z  z  }|| _        || _        || _        || _
        |d         |d         z  |d         |d         z  f| _        t          j        ||||          | _        d S )Nr   r   )kernel_sizestride)rP   rQ   
image_sizerb   rH   rY   
isinstancecollectionsabcIterablerT   rU   r   Conv2d
projection)r4   rc   r   rb   rH   hidden_sizerT   re   s          r'   rQ   zSwinPatchEmbeddings.__init__  s   !'!2F4EJ
$*$79Ik#-j+/:R#S#SqZZZdfpYq
#-j+/:R#S#SqZZZdfpYq
!!}
15*Q-:VW=:XY$$(&$Q-:a=8*Q-:VW=:XY)L+:^hiiir&   c                 Z   || j         d         z  dk    r@d| j         d         || j         d         z  z
  f}t          j                            ||          }|| j         d         z  dk    rBddd| j         d         || j         d         z  z
  f}t          j                            ||          }|S )Nr   r   )rb   r   rs   pad)r4   r}   rF   rG   
pad_valuess        r'   	maybe_padzSwinPatchEmbeddings.maybe_pad   s    4?1%%**T_Q/%$/!:L2LLMJ=,,\:FFLDOA&&!++Q4?1#5QRAS8S#STJ=,,\:FFLr&   r}   rg   c                     |j         \  }}}}|                     |||          }|                     |          }|j         \  }}}}||f}|                    d                              dd          }||fS )Nr:   r   )r?   r   r   flatten	transpose)r4   r}   r   rH   rF   rG   rf   r   s           r'   r   zSwinPatchEmbeddings.forward)  s    )5);&<~~lFEBB__\22
(.1fe#UO''**44Q::
,,,r&   )r   r   r   r    rQ   r   r   r!   r"   r$   r   r   r   r   r   s   @r'   rR   rR   
  s         j j j j j  	-HU->$? 	-E%,X]^aXbJbDc 	- 	- 	- 	- 	- 	- 	- 	-r&   rR   c            	            e Zd ZdZej        fdee         dedej        ddf fdZ	d Z
d	ej        d
eeef         dej        fdZ xZS )SwinPatchMerginga'  
    Patch Merging Layer.

    Args:
        input_resolution (`tuple[int]`):
            Resolution of input feature.
        dim (`int`):
            Number of input channels.
        norm_layer (`nn.Module`, *optional*, defaults to `nn.LayerNorm`):
            Normalization layer class.
    input_resolutionro   
norm_layerrg   Nc                     t                                                       || _        || _        t	          j        d|z  d|z  d          | _         |d|z            | _        d S )Nr;   r:   Fbias)rP   rQ   r   ro   r   Linear	reductionr^   )r4   r   ro   r   re   s       r'   rQ   zSwinPatchMerging.__init__B  sa     01s7AG%@@@Jq3w''			r&   c                     |dz  dk    p|dz  dk    }|r.ddd|dz  d|dz  f}t           j                            ||          }|S )Nr:   r   r   )r   rs   r   )r4   rC   rF   rG   
should_padr   s         r'   r   zSwinPatchMerging.maybe_padI  s\    qjAo:519>
 	IQ519a!<JM--mZHHMr&   rC   input_dimensionsc                    |\  }}|j         \  }}}|                    ||||          }|                     |||          }|d d dd ddd dd d f         }|d d dd ddd dd d f         }	|d d dd ddd dd d f         }
|d d dd ddd dd d f         }t          j        ||	|
|gd          }|                    |dd|z            }|                     |          }|                     |          }|S )Nr   r:   r   r=   r;   )r?   r@   r   r!   ru   r^   r   )r4   rC   r   rF   rG   rE   ro   rH   input_feature_0input_feature_1input_feature_2input_feature_3s               r'   r   zSwinPatchMerging.forwardQ  sD   ((5(;%
C%**:vulSS}feDD'14a4Aqqq(89'14a4Aqqq(89'14a4Aqqq(89'14a4Aqqq(89	?O_Ve"fhjkk%**:r1|;KLL		-00}55r&   )r   r   r   r    r   r]   r$   r   ModulerQ   r   r!   r   r   r   r   s   @r'   r   r   5  s        
 
 XZWc ( (s (# (29 (hl ( ( ( ( ( (  U\ U3PS8_ Y^Ye        r&   r           Finput	drop_probtrainingrg   c                     |dk    s|s| S d|z
  }| j         d         fd| j        dz
  z  z   }|t          j        || j        | j                  z   }|                                 |                     |          |z  }|S )aF  
    Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).

    Comment by Ross Wightman: This is the same as the DropConnect impl I created for EfficientNet, etc networks,
    however, the original name is misleading as 'Drop Connect' is a different form of dropout in a separate paper...
    See discussion: https://github.com/tensorflow/tpu/issues/494#issuecomment-532968956 ... I've opted for changing the
    layer and argument names to 'drop path' rather than mix DropConnect as a layer name and use 'survival rate' as the
    argument.
    r   r   r   )r   dtypedevice)r?   ndimr!   randr   r   floor_div)r   r   r   	keep_probr?   random_tensoroutputs          r'   	drop_pathr   l  s     CxII[^
Q 77E
5EL Y Y YYMYYy!!M1FMr&   c                   j     e Zd ZdZd	dee         ddf fdZdej        dej        fdZ	de
fdZ xZS )
SwinDropPathzXDrop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).Nr   rg   c                 V    t                                                       || _        d S N)rP   rQ   r   )r4   r   re   s     r'   rQ   zSwinDropPath.__init__  s$    "r&   r   c                 8    t          || j        | j                  S r   )r   r   r   r4   r   s     r'   r   zSwinDropPath.forward  s    FFFr&   c                     d| j          S )Nzp=)r   r3   s    r'   
extra_reprzSwinDropPath.extra_repr  s    $DN$$$r&   r   )r   r   r   r    r   floatrQ   r!   r   r   strr   r   r   s   @r'   r   r     s        bb# #(5/ #T # # # # # #GU\ Gel G G G G%C % % % % % % % %r&   r   c                        e Zd Z fdZ	 	 	 d
dej        deej                 deej                 dee         de	ej                 f
d	Z
 xZS )SwinSelfAttentionc                    t                                                       ||z  dk    rt          d| d| d          || _        t	          ||z            | _        | j        | j        z  | _        t          |t          j	        j
                  r|n||f| _        t          j        t          j        d| j        d         z  dz
  d| j        d         z  dz
  z  |                    | _        t          j        | j        d                   }t          j        | j        d                   }t          j        t'          ||gd                    }t          j        |d          }|d d d d d f         |d d d d d f         z
  }	|	                    ddd                                          }	|	d d d d dfxx         | j        d         dz
  z  cc<   |	d d d d dfxx         | j        d         dz
  z  cc<   |	d d d d dfxx         d| j        d         z  dz
  z  cc<   |	                    d	          }
|                     d
|
           t          j        | j        | j        |j                  | _        t          j        | j        | j        |j                  | _        t          j        | j        | j        |j                  | _        t          j        |j                  | _         d S )Nr   zThe hidden size (z6) is not a multiple of the number of attention heads ()r:   r   ij)indexingr=   relative_position_indexr   )!rP   rQ   
ValueErrornum_attention_headsr   attention_head_sizeall_head_sizer   r   r   r   rD   r   rW   r!   rX   relative_position_bias_tablearangestackr   r   rA   rB   sumregister_bufferr   qkv_biasquerykeyvaluer_   attention_probs_dropout_probra   )r4   rc   ro   	num_headsrD   coords_hcoords_wcoordscoords_flattenrelative_coordsr   re   s              r'   rQ   zSwinSelfAttention.__init__  s   ?akCkk_hkkk   $- #&sY#7#7 !58PP%k;?3KLLlKKS^`kRl 	 -/LKT-a0014T=Ma=P9PST9TUW`aa-
 -
)
 < 0 344< 0 344Xx&:TJJJKKvq11(AAAt4~aaaqqqj7QQ)11!Q::EEGG111a   D$4Q$7!$;;   111a   D$4Q$7!$;;   111a   A(8(;$;a$??   "1"5"5b"9"968OPPPYt143EFO\\\
9T/1C&/ZZZYt143EFO\\\
z&"EFFr&   NFr   attention_mask	head_maskoutput_attentionsrg   c                    |j         \  }}}||d| j        f}|                     |                              |                              dd          }	|                     |                              |                              dd          }
|                     |                              |                              dd          }t          j        |	|
                    dd                    }|t          j
        | j                  z  }| j        | j                            d                   }|                    | j        d         | j        d         z  | j        d         | j        d         z  d          }|                    ddd                                          }||                    d          z   }|v|j         d         }|                    ||z  || j        ||          }||                    d                              d          z   }|                    d| j        ||          }t$          j                            |d          }|                     |          }|||z  }t          j        ||          }|                    dddd                                          }|                                d d         | j        fz   }|                    |          }|r||fn|f}|S )Nr=   r   r:   r   rn   r   )r?   r   r   r@   r   r   r   r!   matmulmathsqrtr   r   rD   rA   rB   r   r   r   rs   softmaxra   rk   r   )r4   r   r   r   r   rE   ro   rH   hidden_shapequery_layer	key_layervalue_layerattention_scoresrelative_position_bias
mask_shapeattention_probscontext_layernew_context_layer_shapeoutputss                      r'   r   zSwinSelfAttention.forward  s    )6(;%
C"CT-EFjj//44\BBLLQPQRRHH]++00>>HHANN	jj//44\BBLLQPQRR !<Y5H5HR5P5PQQ+di8P.Q.QQ!%!B4C_CdCdegChCh!i!7!<!<Q$"21"55t7G7JTM]^_M`7`bd"
 "
 "8!?!?1a!H!H!S!S!U!U+.D.N.Nq.Q.QQ%'-a0J/44j(*d6NPSUX     0.2J2J12M2M2W2WXY2Z2ZZ/44R9QSVX[\\ -//0@b/II ,,77  -	9O_kBB%--aAq99DDFF"/"4"4"6"6ss";t?Q>S"S%**+BCC6G]=/22mM]r&   NNF)r   r   r   rQ   r!   r   r   r"   r   r$   r   r   r   s   @r'   r   r     s        #G #G #G #G #GP 7;15,16 6|6 !!236 E-.	6
 $D>6 
u|	6 6 6 6 6 6 6 6r&   r   c                   P     e Zd Z fdZdej        dej        dej        fdZ xZS )SwinSelfOutputc                     t                                                       t          j        ||          | _        t          j        |j                  | _        d S r   )rP   rQ   r   r   denser_   r   ra   r4   rc   ro   re   s      r'   rQ   zSwinSelfOutput.__init__  sD    YsC((
z&"EFFr&   r   input_tensorrg   c                 Z    |                      |          }|                     |          }|S r   r  ra   )r4   r   r  s      r'   r   zSwinSelfOutput.forward  s*    

=11]33r&   r   r   r   rQ   r!   r   r   r   r   s   @r'   r   r     sn        G G G G G
U\  RWR^        r&   r   c                        e Zd Z fdZd Z	 	 	 ddej        deej                 deej                 dee	         d	e
ej                 f
d
Z xZS )SwinAttentionc                     t                                                       t          ||||          | _        t	          ||          | _        t                      | _        d S r   )rP   rQ   r   r4   r   r   setpruned_heads)r4   rc   ro   r   rD   re   s        r'   rQ   zSwinAttention.__init__  sQ    %fc9kJJ	$VS11EEr&   c                    t          |          dk    rd S t          || j        j        | j        j        | j                  \  }}t          | j        j        |          | j        _        t          | j        j        |          | j        _        t          | j        j	        |          | j        _	        t          | j
        j        |d          | j
        _        | j        j        t          |          z
  | j        _        | j        j        | j        j        z  | j        _        | j                            |          | _        d S )Nr   r   rn   )lenr   r4   r   r   r  r   r   r   r   r   r  r   union)r4   headsindexs      r'   prune_headszSwinAttention.prune_heads  s    u::??F7490$)2OQUQb
 
u
 -TY_eDD	*49=%@@	,TY_eDD	.t{/@%QOOO )-	(EE

(R	%"&)"?$)B_"_	 -33E::r&   NFr   r   r   r   rg   c                     |                      ||||          }|                     |d         |          }|f|dd          z   }|S )Nr   r   )r4   r   )r4   r   r   r   r   self_outputsattention_outputr   s           r'   r   zSwinAttention.forward  sO     yy	K\]];;|AFF#%QRR(88r&   r   )r   r   r   rQ   r  r!   r   r   r"   r   r$   r   r   r   s   @r'   r	  r	    s        " " " " "; ; ;* 7;15,1
 
|
 !!23
 E-.	

 $D>
 
u|	
 
 
 
 
 
 
 
r&   r	  c                   B     e Zd Z fdZdej        dej        fdZ xZS )SwinIntermediatec                 $   t                                                       t          j        |t	          |j        |z                      | _        t          |j        t                    rt          |j                 | _        d S |j        | _        d S r   )rP   rQ   r   r   r   	mlp_ratior  r   
hidden_actr   r   intermediate_act_fnr  s      r'   rQ   zSwinIntermediate.__init__"  sx    YsC(83(>$?$?@@
f'-- 	9'-f.?'@D$$$'-'8D$$$r&   r   rg   c                 Z    |                      |          }|                     |          }|S r   )r  r  r   s     r'   r   zSwinIntermediate.forward*  s,    

=1100??r&   r  r   s   @r'   r  r  !  s^        9 9 9 9 9U\ el        r&   r  c                   B     e Zd Z fdZdej        dej        fdZ xZS )
SwinOutputc                     t                                                       t          j        t	          |j        |z            |          | _        t          j        |j                  | _	        d S r   )
rP   rQ   r   r   r   r  r  r_   r`   ra   r  s      r'   rQ   zSwinOutput.__init__1  sT    Ys6#3c#9::C@@
z&"<==r&   r   rg   c                 Z    |                      |          }|                     |          }|S r   r  r   s     r'   r   zSwinOutput.forward6  s*    

=11]33r&   r  r   s   @r'   r  r  0  s^        > > > > >
U\ el        r&   r  c                        e Zd Zd fd	Zd Zd Zd Z	 	 	 dd	ej        d
e	e
e
f         deej                 dee         dee         de	ej        ej        f         fdZ xZS )	SwinLayerr   r   c                    t                                                       |j        | _        || _        |j        | _        || _        t          j        ||j                  | _	        t          |||| j                  | _        |dk    rt          |          nt          j                    | _        t          j        ||j                  | _        t!          ||          | _        t%          ||          | _        d S )Neps)rD   r   )rP   rQ   chunk_size_feed_forward
shift_sizerD   r   r   r]   layer_norm_epslayernorm_beforer	  	attentionr   Identityr   layernorm_afterr  intermediater  r   )r4   rc   ro   r   r   drop_path_rater'  re   s          r'   rQ   zSwinLayer.__init__=  s    '-'E$$!- 0 "Sf6K L L L&vsI4K[\\\9G#9M9Mn555SUS^S`S`!|CV5JKKK,VS99 --r&   c                    t          |          | j        k    rnt          d          | _        t          j                                        r&t	          j         t	          j        |                    nt          |          | _        d S d S Nr   )minrD   r   r'  r!   rp   rq   tensor)r4   r   s     r'   set_shift_and_window_sizez#SwinLayer.set_shift_and_window_sizeJ  sv      D$444'llDO=BY=Q=Q=S=Sn	%,'788999Y\]mYnYn  54r&   c           	         | j         dk    r]t          j        d||df||          }t          d| j                   t          | j         | j                    t          | j          d           f}t          d| j                   t          | j         | j                    t          | j          d           f}d}|D ]}	|D ]}
||d d |	|
d d f<   |dz  }t          || j                  }|                    d| j        | j        z            }|                    d          |                    d          z
  }|                    |dk    d                              |dk    d          }nd }|S )Nr   r   r   r=   r:   g      Yr   )	r'  r!   rX   slicerD   rJ   r@   r   masked_fill)r4   rF   rG   r   r   img_maskheight_sliceswidth_slicescountheight_slicewidth_slicemask_windows	attn_masks                r'   get_attn_maskzSwinLayer.get_attn_maskR  s   ?Q{Avua#8fUUUHa$**++t''$/)9::t&--M a$**++t''$/)9::t&--L
 E -  #/  K@EHQQQk111<=QJEE ,Hd6FGGL',,R1ADDT1TUUL$..q11L4J4J14M4MMI!--i1nfEEQQR[_`R`beffIIIr&   c                     | j         || j         z  z
  | j         z  }| j         || j         z  z
  | j         z  }ddd|d|f}t          j                            ||          }||fS r0  )rD   r   rs   r   )r4   r   rF   rG   	pad_right
pad_bottomr   s          r'   r   zSwinLayer.maybe_padn  sp    %0@(@@DDTT	&$2B)BBdFVV
Ay!Z8
))-DDj((r&   NFr   r   r   r   always_partitionrg   c                    |s|                      |           n	 |\  }}|                                \  }}	}
|}|                     |          }|                    ||||
          }|                     |||          \  }}|j        \  }	}}}	| j        dk    r&t          j        || j         | j         fd          }n|}t          || j
                  }|                    d| j
        | j
        z  |
          }|                     |||j        |j                  }|                     ||||          }|d         }|                    d| j
        | j
        |
          }t          || j
        ||          }| j        dk    r$t          j        || j        | j        fd          }n|}|d         dk    p|d         dk    }|r&|d d d |d |d d f                                         }|                    |||z  |
          }||                     |          z   }|                     |          }|                     |          }||                     |          z   }|r
||d	         fn|f}|S )
Nr   )r   r:   )shiftsdimsr=   r   )r   r   r<   r   )r3  rk   r)  r@   r   r?   r'  r!   rollrJ   rD   r?  r   r   r*  rL   rB   r   r,  r-  r   )r4   r   r   r   r   rC  rF   rG   rE   r   channelsshortcutr   
height_pad	width_padshifted_hidden_stateshidden_states_windowsr>  attention_outputsr  attention_windowsshifted_windows
was_paddedlayer_outputlayer_outputss                            r'   r   zSwinLayer.forwardu  s      	**+;<<<<("/"4"4"6"6
Ax --m<<%**:vuhOO %)NN=&%$P$P!z&3&9#:y!?Q$)J}tFVY]YhXhEipv$w$w$w!!$1! !11FHX Y Y 5 : :2t?ORVRb?bdl m m&&	)<EZEa ' 
 
	 !NN!9iK\ + 
 
 -Q/,11"d6FHXZbcc():D<LjZcdd ?Q %
?DOUYUdCelr s s s /]Q&;*Q-!*;
 	V 1!!!WfWfufaaa2G H S S U U-22:v~xXX 4>>2C#D#DD++M::((66$t{{<'@'@@@Qf'8';<<XdWfr&   )r   r   NFF)r   r   r   rQ   r3  r?  r   r!   r   r$   r   r   r"   r   r   r   r   s   @r'   r"  r"  <  s        . . . . . .    8) ) ) 26,1+0A A|A  S/A E-.	A
 $D>A #4.A 
u|U\)	*A A A A A A A Ar&   r"  c                        e Zd Z fdZ	 	 	 ddej        deeef         deej	                 dee
         dee
         d	eej                 fd
Z xZS )	SwinStagec                 6   t                                                       | _        | _        t	          j        fdt          |          D                       | _        | |t          j                  | _	        nd | _	        d| _
        d S )Nc                 l    g | ]0}t          |         |d z  dk    rdn	j        d z            1S )r:   r   )rc   ro   r   r   r.  r'  )r"  rD   ).0irc   ro   r   r   r   s     r'   
<listcomp>z&SwinStage.__init__.<locals>.<listcomp>  sh     
 
 
  !%5'#,Q<%&UaZZqqf6HA6M  
 
 
r&   )ro   r   F)rP   rQ   rc   ro   r   
ModuleListrangeblocksr]   
downsamplepointing)	r4   rc   ro   r   depthr   r   r_  re   s	    ``` `` r'   rQ   zSwinStage.__init__  s    m
 
 
 
 
 
 
 
 u
 
 

 
 !(j)9sr|\\\DOO"DOr&   NFr   r   r   r   rC  rg   c                 *   |\  }}t          | j                  D ](\  }}	|||         nd }
 |	|||
||          }|d         })|}| j        -|dz   dz  |dz   dz  }}||||f}|                     ||          }n||||f}|||f}|r||dd          z  }|S )Nr   r   r:   )	enumerater^  r_  )r4   r   r   r   r   rC  rF   rG   rZ  layer_modulelayer_head_maskrS  !hidden_states_before_downsamplingheight_downsampledwidth_downsampledr   stage_outputss                    r'   r   zSwinStage.forward  s     )(55 	- 	-OA|.7.CillO(L/BSUe M *!,MM,9)?&5;aZA4EPQ	VWGW 1!'0BDU V OO,MO_``MM!' >&(IK\] 	/]122..Mr&   rT  )r   r   r   rQ   r!   r   r$   r   r   r"   r   r   r   r   s   @r'   rV  rV    s            < 26,1+0 |  S/ E-.	
 $D> #4. 
u|	       r&   rV  c                        e Zd Z fdZ	 	 	 	 	 	 ddej        deeef         deej	                 dee
         d	ee
         d
ee
         dee
         dee
         deeef         fdZ xZS )SwinEncoderc                     t                                                       t          j                   _         _        d t          j        dj        t          j                  d          D             t          j         fdt           j                  D                        _        d _        d S )Nc                 6    g | ]}|                                 S r%   )item)rY  xs     r'   r[  z(SwinEncoder.__init__.<locals>.<listcomp>  s     lllAqvvxxlllr&   r   cpu)r   c                 t   g | ]}t          t          j        d |z  z            d         d |z  z  d         d |z  z  fj        |         j        |         t          j        d|                   t          j        d|dz                               |j        dz
  k     rt          nd          S )r:   r   r   N)rc   ro   r   ra  r   r   r_  )rV  r   rY   depthsr   r   
num_layersr   )rY  i_layerrc   dprrU   r4   s     r'   r[  z(SwinEncoder.__init__.<locals>.<listcomp>  s         !F,q'z9::&/lq'z&BIaLUVX_U_D`%a -0$.w7!#fmHWH&=">">V]S`U\_`U`S`EaAbAb"bc4;doPQ>Q4Q4Q//X\    r&   F)rP   rQ   r  rr  rs  rc   r!   linspacer.  r   r   r\  r]  layersgradient_checkpointing)r4   rc   rU   ru  re   s   ```@r'   rQ   zSwinEncoder.__init__  s    fm,,ll63H#fmJ\J\ej!k!k!klllm        %T_55  
 
 ',###r&   NFTr   r   r   r   output_hidden_states(output_hidden_states_before_downsamplingrC  return_dictrg   c	                    |rdnd }	|rdnd }
|rdnd }|r?|j         \  }}} |j        |g||R  }|                    dddd          }|	|fz  }	|
|fz  }
t          | j                  D ]\  }}|||         nd } ||||||          }|d         }|d         }|d         }|d         |d         f}|rP|rN|j         \  }}} |j        |g|d         |d         f|R  }|                    dddd          }|	|fz  }	|
|fz  }
nC|rA|s?|j         \  }}} |j        |g||R  }|                    dddd          }|	|fz  }	|
|fz  }
|r||dd          z  }|st          d ||	|fD                       S t          ||	||
	          S )
Nr%   r   r   r   r:   r   r=   c              3      K   | ]}||V  	d S r   r%   )rY  vs     r'   	<genexpr>z&SwinEncoder.forward.<locals>.<genexpr>F  s(      mmq_`_l_l_l_l_lmmr&   )r   r   r   r   )r?   r@   rA   rc  rw  r$   r   )r4   r   r   r   r   ry  rz  rC  r{  all_hidden_statesall_reshaped_hidden_statesall_self_attentionsrE   r   r   reshaped_hidden_staterZ  rd  re  rS  rf  r   s                         r'   r   zSwinEncoder.forward  s    #7@BBD+?%IRRT"$5?bb4 	C)6)<&J;$6M$6z$bDT$bVa$b$b$b!$9$A$A!Q1$M$M!-!11&+@*BB&(55  	9  	9OA|.7.CillO(L/BSUe M *!,M0=a0@- -a 0 1" 57H7LM# G(P G-N-T*
A{ )O(I(N)"3A"68I!8L!M)OZ) ) )% )>(E(EaAq(Q(Q%!&G%II!*/D.FF**% G.V G-:-@*
A{(:(::(fHX(fZe(f(f(f%(=(E(EaAq(Q(Q%!m%55!*/D.FF*  9#}QRR'88# 	nmm]4EGZ$[mmmmmm ++*#=	
 
 
 	
r&   )NFFFFT)r   r   r   rQ   r!   r   r$   r   r   r"   r   r   r   r   r   r   s   @r'   rk  rk    s        , , , , ,4 26,1/4CH+0&*A
 A
|A
  S/A
 E-.	A

 $D>A
 'tnA
 3;4.A
 #4.A
 d^A
 
u''	(A
 A
 A
 A
 A
 A
 A
 A
r&   rk  c                   2    e Zd ZU eed<   dZdZdZdgZd Z	dS )SwinPreTrainedModelrc   swinr}   TrV  c                    t          |t          j        t          j        f          rT|j        j                            d| j        j                   |j	         |j	        j        
                                 dS dS t          |t          j                  r?|j	        j        
                                 |j        j                            d           dS t          |t                    rN|j        |j        j        
                                 |j         |j        j        
                                 dS dS t          |t                     r |j        j        
                                 dS dS )zInitialize the weightsr   )meanstdNr   )r   r   r   r   weightdatanormal_rc   initializer_ranger   zero_r]   fill_rN   rZ   r\   r   r   )r4   modules     r'   _init_weightsz!SwinPreTrainedModel._init_weightsX  sV   fry")455 	= M&&CT[5R&SSS{& &&((((( '&-- 		=K""$$$M$$S)))))// 	= ,!&,,...)5*/5577777 65 122 	=/4::<<<<<	= 	=r&   N)
r   r   r   r   r#   base_model_prefixmain_input_namesupports_gradient_checkpointing_no_split_modulesr  r%   r&   r'   r  r  P  sJ         $O&*#$= = = = =r&   r  c                        e Zd Zd fd	Zd Zd Ze	 	 	 	 	 	 	 ddeej	                 deej
                 d	eej	                 d
ee         dee         dedee         deeef         fd            Z xZS )	SwinModelTFc                    t                                          |           || _        t          |j                  | _        t          |j        d| j        dz
  z  z            | _        t          ||          | _
        t          || j
        j                  | _        t          j        | j        |j                  | _        |rt          j        d          nd| _        |                                  dS )a  
        add_pooling_layer (`bool`, *optional*, defaults to `True`):
            Whether or not to apply pooling layer.
        use_mask_token (`bool`, *optional*, defaults to `False`):
            Whether or not to create and apply mask tokens in the embedding layer.
        r:   r   )rd   r$  N)rP   rQ   rc   r  rr  rs  r   rY   num_featuresrN   rf   rk  rV   encoderr   r]   r(  	layernormAdaptiveAvgPool1dpooler	post_init)r4   rc   add_pooling_layerrd   re   s       r'   rQ   zSwinModel.__init__n  s     	   fm,, 0119L3M MNN(OOO"64?+EFFd&7V=RSSS1BLb*1--- 	r&   c                     | j         j        S r   rf   rS   r3   s    r'   get_input_embeddingszSwinModel.get_input_embeddings      //r&   c                     |                                 D ]/\  }}| j        j        |         j                            |           0dS )z
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        N)itemsr  layerr*  r  )r4   heads_to_pruner  r  s       r'   _prune_headszSwinModel._prune_heads  sU    
 +0022 	C 	CLE5Lu%/;;EBBBB	C 	Cr&   Nr}   r~   r   r   ry  r|   r{  rg   c                 ~   ||n| j         j        }||n| j         j        }||n| j         j        }|t	          d          |                     |t          | j         j                            }|                     |||          \  }}	| 	                    ||	||||          }
|
d         }| 
                    |          }d}| j        >|                     |                    dd                    }t          j        |d          }|s||f|
dd         z   }|S t          |||
j        |
j        |
j                  S )	z
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
        Nz You have to specify pixel_values)r~   r|   )r   r   ry  r{  r   r   r:   )r   r*   r   r   r   )rc   r   ry  use_return_dictr   get_head_maskr  rr  rf   r  r  r  r   r!   r   r)   r   r   r   )r4   r}   r~   r   r   ry  r|   r{  embedding_outputr   encoder_outputssequence_outputpooled_outputr   s                 r'   r   zSwinModel.forward  s    2C1N--TXT_Tq$8$D  $+Jj 	 &1%<kk$+B]?@@@ &&y#dk6H2I2IJJ	-1__/Tl .= .
 .
** ,,/!5# ' 
 
 *!,..99;" KK(A(A!Q(G(GHHM!M-;;M 	%}58KKFM-')7&1#2#I
 
 
 	
r&   )TFNNNNNFN)r   r   r   rQ   r  r  r   r   r!   r"   r   r   r   r$   r)   r   r   r   s   @r'   r  r  l  s            *0 0 0C C C  596:15,0/3).&*>
 >
u01>
 "%"23>
 E-.	>

 $D>>
 'tn>
 #'>
 d^>
 
uo%	&>
 >
 >
 ^>
 >
 >
 >
 >
r&   r  ad  
    Swin Model with a decoder on top for masked image modeling, as proposed in [SimMIM](https://huggingface.co/papers/2111.09886).

    <Tip>

    Note that we provide a script to pre-train this model on custom data in our [examples
    directory](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-pretraining).

    </Tip>
    c                        e Zd Z fdZe	 	 	 	 	 	 	 ddeej                 deej                 deej                 dee	         dee	         d	e	d
ee	         de
eef         fd            Z xZS )SwinForMaskedImageModelingc                    t                                          |           t          |dd          | _        t	          |j        d|j        dz
  z  z            }t          j        t          j	        ||j
        dz  |j        z  d          t          j        |j
                            | _        |                                  d S )NFT)r  rd   r:   r   )in_channelsout_channelsr   )rP   rQ   r  r  r   rY   rs  r   
Sequentialr   encoder_striderH   PixelShuffledecoderr  )r4   rc   r  re   s      r'   rQ   z#SwinForMaskedImageModeling.__init__  s       fdSSS	6+aF4E4I.JJKK}I(v7La7ORXRe7est   OF122	
 
 	r&   NFr}   r~   r   r   ry  r|   r{  rg   c           	         ||n| j         j        }|                     |||||||          }|d         }	|	                    dd          }	|	j        \  }
}}t          j        |dz            x}}|	                    |
|||          }	|                     |	          }d}|| j         j	        | j         j
        z  }|                    d||          }|                    | j         j
        d                              | j         j
        d                              d                                          }t          j                            ||d	          }||z                                  |                                d
z   z  | j         j        z  }|s|f|dd         z   }||f|z   n|S t'          |||j        |j        |j                  S )a7  
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).

        Examples:
        ```python
        >>> from transformers import AutoImageProcessor, SwinForMaskedImageModeling
        >>> import torch
        >>> from PIL import Image
        >>> import requests

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> image_processor = AutoImageProcessor.from_pretrained("microsoft/swin-base-simmim-window6-192")
        >>> model = SwinForMaskedImageModeling.from_pretrained("microsoft/swin-base-simmim-window6-192")

        >>> num_patches = (model.config.image_size // model.config.patch_size) ** 2
        >>> pixel_values = image_processor(images=image, return_tensors="pt").pixel_values
        >>> # create random boolean mask of shape (batch_size, num_patches)
        >>> bool_masked_pos = torch.randint(low=0, high=2, size=(1, num_patches)).bool()

        >>> outputs = model(pixel_values, bool_masked_pos=bool_masked_pos)
        >>> loss, reconstructed_pixel_values = outputs.loss, outputs.reconstruction
        >>> list(reconstructed_pixel_values.shape)
        [1, 3, 192, 192]
        ```N)r~   r   r   ry  r|   r{  r   r   r:   ri   r=   none)r   gh㈵>)r-   r.   r   r   r   )rc   r  r  r   r?   r   floorrr   r  r   rb   repeat_interleaver   rB   r   rs   l1_lossr   rH   r,   r   r   r   )r4   r}   r~   r   r   ry  r|   r{  r   r  rE   rH   sequence_lengthrF   rG   reconstructed_pixel_valuesmasked_im_lossrk   r   reconstruction_lossr   s                        r'   r   z"SwinForMaskedImageModeling.forward  s   L &1%<kk$+B]))+/!5%=#  
 
 "!*)33Aq994C4I1
L/OS$8999)11*lFTYZZ &*\\/%B%B"&;)T[-CCD-55b$EEO11$+2H!LL""4;#91==1	  #%-"7"7F`lr"7"s"s1D8==??488::PTCTUX\XcXppN 	Z02WQRR[@F3A3M^%..SYY,5!/)#*#A
 
 
 	
r&   r  )r   r   r   rQ   r   r   r!   r"   r   r   r   r$   r,   r   r   r   s   @r'   r  r    s              596:15,0/3).&*R
 R
u01R
 "%"23R
 E-.	R

 $D>R
 'tnR
 #'R
 d^R
 
u33	4R
 R
 R
 ^R
 R
 R
 R
 R
r&   r  a  
    Swin Model transformer with an image classification head on top (a linear layer on top of the final hidden state of
    the [CLS] token) e.g. for ImageNet.

    <Tip>

        Note that it's possible to fine-tune Swin on higher resolution images than the ones it has been trained on, by
        setting `interpolate_pos_encoding` to `True` in the forward of the model. This will interpolate the pre-trained
        position embeddings to the higher resolution.

    </Tip>
    c                        e Zd Z fdZe	 	 	 	 	 	 	 ddeej                 deej                 deej                 dee	         dee	         d	e	d
ee	         de
eef         fd            Z xZS )SwinForImageClassificationc                 @   t                                          |           |j        | _        t          |          | _        |j        dk    r$t          j        | j        j        |j                  nt          j                    | _	        | 
                                 d S r0  )rP   rQ   
num_labelsr  r  r   r   r  r+  
classifierr  )r4   rc   re   s     r'   rQ   z#SwinForImageClassification.__init__R  s        +f%%	 EKDUXYDYDYBIdi,f.?@@@_a_j_l_l 	
 	r&   NFr}   r   labelsr   ry  r|   r{  rg   c                 L   ||n| j         j        }|                     ||||||          }|d         }	|                     |	          }
d}||                     ||
| j                   }|s|
f|dd         z   }||f|z   n|S t          ||
|j        |j        |j                  S )a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        N)r   r   ry  r|   r{  r   r:   )r-   r5   r   r   r   )	rc   r  r  r  loss_functionr8   r   r   r   )r4   r}   r   r  r   ry  r|   r{  r   r  r5   r-   r   s                r'   r   z"SwinForImageClassification.forward`  s    " &1%<kk$+B]))/!5%=#  
 
  
//%%ffdkBBD 	FY,F)-)9TGf$$vE(!/)#*#A
 
 
 	
r&   r  )r   r   r   rQ   r   r   r!   r"   
LongTensorr   r   r$   r8   r   r   r   s   @r'   r  r  C  s              5915-1,0/3).&*-
 -
u01-
 E-.-
 )*	-

 $D>-
 'tn-
 #'-
 d^-
 
u//	0-
 -
 -
 ^-
 -
 -
 -
 -
r&   r  zM
    Swin backbone, to be used with frameworks like DETR and MaskFormer.
    c                   |     e Zd Zdef fdZd Z	 	 	 ddej        dee	         dee	         dee	         d	e
f
d
Z xZS )SwinBackbonerc   c                 6   t                                                     t                                                     j        gfdt	          t          j                            D             z   | _        t                    | _	        t          | j	        j                  | _        i }t          | j        | j                  D ]\  }}t!          j        |          ||<   t!          j        |          | _        |                                  d S )Nc                 D    g | ]}t          j        d |z  z            S )r:   )r   rY   )rY  rZ  rc   s     r'   r[  z)SwinBackbone.__init__.<locals>.<listcomp>  s.    1r1r1rST#f6FA6M2N2N1r1r1rr&   )rP   rQ   _init_backbonerY   r]  r  rr  r  rN   rf   rk  rV   r  zip_out_featuresrH  r   r]   
ModuleDicthidden_states_normsr  )r4   rc   r  stagerH   re   s    `   r'   rQ   zSwinBackbone.__init__  s      v&&&#-.1r1r1r1rX]^abhbo^p^pXqXq1r1r1rr(00"64?+EFF !#&t'94=#I#I 	D 	DE<)+l)C)C&&#%=1D#E#E  	r&   c                     | j         j        S r   r  r3   s    r'   r  z!SwinBackbone.get_input_embeddings  r  r&   Nr}   ry  r   r{  rg   c           
         ||n| j         j        }||n| j         j        }||n| j         j        }|                     |          \  }}|                     ||d|dddd          }|j        }d}	t          | j        |          D ]\  }
}|
| j	        v r|j
        \  }}}}|                    dddd                                          }|                    |||z  |          } | j        |
         |          }|                    ||||          }|                    dddd                                          }|	|fz  }	|s|	f}|r||j        fz  }|S t!          |	|r|j        nd|j        	          S )
aK  
        Returns:

        Examples:

        ```python
        >>> from transformers import AutoImageProcessor, AutoBackbone
        >>> import torch
        >>> from PIL import Image
        >>> import requests

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> processor = AutoImageProcessor.from_pretrained("shi-labs/nat-mini-in1k-224")
        >>> model = AutoBackbone.from_pretrained(
        ...     "microsoft/swin-tiny-patch4-window7-224", out_features=["stage1", "stage2", "stage3", "stage4"]
        ... )

        >>> inputs = processor(image, return_tensors="pt")
        >>> outputs = model(**inputs)
        >>> feature_maps = outputs.feature_maps
        >>> list(feature_maps[-1].shape)
        [1, 768, 7, 7]
        ```NT)r   r   ry  rz  rC  r{  r%   r   r:   r   r   )feature_mapsr   r   )rc   r  ry  r   rf   r  r   r  stage_namesout_featuresr?   rA   rB   r@   r  r   r
   r   )r4   r}   ry  r   r{  r  r   r   r   r  r  hidden_staterE   rH   rF   rG   r   s                    r'   r   zSwinBackbone.forward  s   @ &1%<kk$+B]$8$D  $+Jj 	 2C1N--TXT_Tq-1__\-J-J**,,/!%59!  	
 	
  6#&t'7#G#G 	0 	0E<))):F:L7
L&%+33Aq!Q??JJLL+00Ve^\ZZ>t7>|LL+00VULYY+33Aq!Q??JJLL/ 	"_F# 37022M%3GQ'//T)
 
 
 	
r&   )NNN)r   r   r   r   rQ   r  r!   r   r   r   r
   r   r   r   s   @r'   r  r    s        z      "0 0 0 04,0&*J
 J
lJ
 'tnJ
 $D>	J

 d^J
 
J
 J
 J
 J
 J
 J
 J
 J
r&   r  )r  r  r  r  r  )r   F)Ar    collections.abcr   r   r0   dataclassesr   typingr   r   r!   r   activationsr   modeling_layersr	   modeling_outputsr
   modeling_utilsr   pytorch_utilsr   r   r   utilsr   r   r   r   utils.backbone_utilsr   configuration_swinr   
get_loggerr   loggerr   r)   r,   r8   rJ   rL   r   rN   rR   r   r   r   r   r   r   r   r   r	  r  r  r"  rV  rk  r  r  r  r  r  __all__r%   r&   r'   <module>r     s   & %       ! ! ! ! ! ! " " " " " " " "        ! ! ! ! ! ! 9 9 9 9 9 9 . . . . . . - - - - - - [ [ [ [ [ [ [ [ [ [ D D D D D D D D D D D D 1 1 1 1 1 1 * * * * * * 
	H	%	%   
K K K K K K K  K    
K K K K Kk K K  K&   
# # # # #K # #  #<   
K K K K K K K  K*	 	 	  Y- Y- Y- Y- Y-RY Y- Y- Y-x(- (- (- (- (-") (- (- (-V3 3 3 3 3ry 3 3 3n U\ e T V[Vb    *% % % % %29 % % %\ \ \ \ \	 \ \ \~
 
 
 
 
RY 
 
 
# # # # #BI # # #L    ry   	 	 	 	 	 	 	 	z z z z z	 z z zz9 9 9 9 9* 9 9 9xX
 X
 X
 X
 X
") X
 X
 X
v = = = = =/ = = =6 `
 `
 `
 `
 `
# `
 `
 `
F 	  d
 d
 d
 d
 d
!4 d
 d
 d
N   =
 =
 =
 =
 =
!4 =
 =
 =
@   
_
 _
 _
 _
 _
& _
 _
 
_
D  r&   