bijddlmZmZmZddlZddlZddlmZddlmZddl m Z ddl m Z mZmZddlmZmZddlmZmZmZmZmZdd lmZdd lmZmZej<eZ Gd d ejBZ"Gd dejBZ#GddejBZ$GddejBZ%GddejBZ&GddeZ'GddeZ(y))OptionalTupleUnionN)nn) BertTokenizer)QuickGELUActivation))BaseModelOutputWithPastAndCrossAttentionsBaseModelOutputWithPooling,BaseModelOutputWithPoolingAndCrossAttentions) Blip2ConfigBlip2VisionConfig) Blip2EncoderBlip2PreTrainedModelBlip2QFormerAttentionBlip2QFormerIntermediateBlip2QFormerOutput)apply_chunking_to_forward)loggingreplace_return_docstringsc2eZdZdZfdZ ddZxZS)Blip2TextEmbeddingsz;Construct the embeddings from word and position embeddings.cLt|tj|j|j |j |_tj|j|j |_ tj|j |j|_ tj|j|_|jdt!j"|jj%dt'|dd|_||_y)N) padding_idxeps position_ids)position_embedding_typeabsolute)super__init__r Embedding vocab_size hidden_size pad_token_idword_embeddingsmax_position_embeddingsposition_embeddings LayerNormlayer_norm_epsDropouthidden_dropout_probdropoutregister_buffertorcharangeexpandgetattrrconfigselfr4 __class__s l/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/diffusers/pipelines/blip_diffusion/modeling_blip2.pyr"zBlip2TextEmbeddings.__init__2s !||F,=,=v?Q?Q_e_r_rs#%<<0N0NPVPbPb#c f&8&8f>S>STzz&"<"<=  ^U\\&:X:X-Y-`-`ah-ij'.v7PR\']$ c||jd}nd}|&|jdd|||zfj}|t|j|}|jdk(r|j |}||z}|>|j d}|j|dd}tj||fd}n|}|j|j}|j|}|j|}|S)Nrrr dim)sizercloner'rr)shaperepeatr0cattodtyper*r.) r6 input_idsr query_embedspast_key_values_length seq_length embeddingsr) batch_sizes r8forwardzBlip2TextEmbeddings.forwardBs   ")!,JJ  ,,Q0FVlIl0l-lmssuL  --i8J++z9&*&>&>|&L#'*== ''--a0 +22:q!D "YY j'AqI %J]]<#5#56 ^^J/ \\*- r9)NNNr)__name__ __module__ __qualname____doc__r"rJ __classcell__r7s@r8rr/sE$ r9rc\eZdZdeffd Zdej dej fdZxZS)Blip2VisionEmbeddingsr4crt|||_|j|_|j |_|j |_tjtjdd|j|_ tjd|j|j |j d|_ |j |j zdz|_|jdz|_tjtjd|j|j|_y)NrF) in_channels out_channels kernel_sizestridebias)r!r"r4r% embed_dim image_size patch_sizer Parameterr0randnclass_embeddingConv2dpatch_embedding num_patches num_positionsposition_embeddingr5s r8r"zBlip2VisionEmbeddings.__init__fs  ++ ++ ++!||EKK1dnn,MN!yyDOO\`\k\krw !OOt>1D!--1"$,,u{{1d>P>PRVR`R`/a"br9 pixel_valuesreturnc|jd}|jjj}|j|j |}|j dj dd}|jj|ddj |}tj||gd}||jddd|jdddfj |z}|S)NrrCrZrrr;) r?rbweightrCrBflatten transposer`r2r0rArer=)r6rfrI target_dtype patch_embeds class_embedsrHs r8rJzBlip2VisionEmbeddings.forwardxs!''* ++2288 ++LOO,O,OP #++A.88A> ++22:q"EHHV YY l;C $"9"9!=Qzq?Q=QST:T"U"X"XYe"ff r9) rKrLrMr r"r0TensorrJrOrPs@r8rRrRes-c0c$ ELL U\\ r9rRc:eZdZfdZ ddZxZS)Blip2QFormerEncoderc t|||_tjt |j Dcgc]}t||c}|_d|_ ycc}w)NF) r!r"r4r ModuleListrangenum_hidden_layersBlip2QFormerLayerlayergradient_checkpointingr6r4 layer_idxr7s r8r"zBlip2QFormerEncoder.__init__sV  ]]CHIaIaCb ci vy 1 c  ',# dsA$c l| rdnd} |rdnd} |rdnd}|rdnd}t|jjD]}|j|}| r| |fz} |||nd}|||nd}t |jddrGt j r3|rtjdd}|j||||||||| }n||||||||| }|d}|r ||dfz }|s| |dfz} |js||dfz}| r| |fz} | std ||| | |fDSt||| | | S) NryFzZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...rrrrZc3$K|]}|| ywNr}).0vs r8 z.Blip2QFormerEncoder.forward..s = s)last_hidden_statepast_key_values hidden_states attentionscross_attentions) rur4rvrxr3r0is_grad_enabledloggerwarning_gradient_checkpointing_funchas_cross_attentiontupler )r6rattention_mask head_maskencoder_hidden_statesencoder_attention_maskr use_cacheoutput_attentionsoutput_hidden_states return_dict query_lengthall_hidden_statesall_self_attentionsall_cross_attentionsnext_decoder_cachei layer_modulelayer_head_maskpast_key_value layer_outputss r8rJzBlip2QFormerEncoder.forwards#7BD$5b4%6rD#,R$t{{445, VA::a=L#$58H$H!.7.CilO3B3N_Q/TXNt{{$encoder_hidden_states must be given for cross-attention layers)rr;) rr ValueErrorrrfeed_forward_chunk_queryrrr?feed_forward_chunkr0rA)r6rrrrrrrrself_attn_past_key_valueself_attention_outputsattention_outputoutputspresent_key_valuequery_attention_outputcross_attention_outputs layer_outputlayer_output_texts r8rJzBlip2QFormerLayer.forwards:H9S>"1#5Y] !%   /3 "0" 2!4(2.226 ! %5a,6I%J "''(0$%eff*.*=*=*")*&7 +>+'*A)C&!$;Ab$AA4--,,  & L %%a(<7$=++00$$$Q q%89 %! %yy,8I)JPQR 4'',,   L  /G+.00r9cL|j|}|j||}|Sr)rrr6rintermediate_outputrs r8rz$Blip2QFormerLayer.feed_forward_chunkDs,"//0@A{{#68HI r9cL|j|}|j||}|Sr)rrrs r8rz*Blip2QFormerLayer.feed_forward_chunk_queryIs."556FG(()<>NO r9)NNNNNFr)rKrLrMr"rJrrrOrPs@r8rwrws01,"#EN r9rwc&eZdZdfd ZdZxZS) ProjLayerct|tj|||_t |_tj|||_tj||_ tj|||_ yNr) r!r"rLineardense1 QuickGELUact_fndense2r,r.r*)r6in_dimout_dim hidden_dimdrop_prr7s r8r"zProjLayer.__init__Qs_ ii 3 k ii G4 zz&) g37r9c |}|j|}|j|j|j|j ||z}|Sr)r*r.rrr)r6xx_ins r8rJzProjLayer.forward\sG NN1  LLT[[Q%@A BT Ir9)皙?-q=rrPs@r8rrPs  8r9rceZdZdZeZdeffd Zeee d de e jde e de e de e de eeff d Zd ZxZS) Blip2VisionModelrfr4cBt||||_|j}t ||_t j||j|_ t||_ t j||j|_ |jyr)r!r"r4r%rRrHrr*r+ pre_layernormrencoderpost_layernorm post_init)r6r4r[r7s r8r"zBlip2VisionModel.__init__jsv   && /7\\)9N9NO#F+  ll9&:O:OP r9) output_type config_classrrrrgc||n|jj}||n|jj}||n|jj}| t d|j |}|j |}|j||||}|d}|j|}|dddddf}|j|}|s ||f|ddzSt|||j|jS)z Returns: Nz You have to specify pixel_values) inputs_embedsrrrrr)r pooler_outputrr) r4rruse_return_dictrrHrrrr rr) r6rfrrrrencoder_outputsr pooled_outputs r8rJzBlip2VisionModel.forwardus2C1N-TXT_T_TqTq$8$D $++JjJj &1%* 'tn * d^ * u00 1 * g* Xr9rc eZdZdZdeffd ZdZdZdZ dde jde e d e jd ed e jf d Z dd ZxZS)Blip2QFormerModelz: Querying Transformer (Q-Former), used in BLIP-2. r4c t||||_t|j|_t |j|_tjtjd|j|jj|_t!|dr |j"t%j&dd|_n&t%j&|j"d|_|j"j)ddit+|jj|jj|jjdzd d |_t/|j|_|j3y) Nr tokenizerzbert-base-uncasedright)truncation_side bos_tokenz[DEC]rr)rrrrr)r!r"r4rqformer_configrHr vision_configvisual_encoderrr^r0zerosnum_query_tokensr% query_tokenshasattrrrfrom_pretrainedadd_special_tokensr proj_layerrrrrr5s r8r"zBlip2QFormerModel.__init__s,   -f.C.CD.v/C/CDLLQ8O8OQWQfQfQrQr)stv{+v/?/?/G*::;N`ghDN*::6;K;K]deDN ));*@A#((44))55,,881<  +6+@+@A  r9c.|jjSrrHr'rs r8rz&Blip2QFormerModel.get_input_embeddingss...r9c&||j_yrr)r6values r8set_input_embeddingsz&Blip2QFormerModel.set_input_embeddingss*/'r9c|jD]7\}}|jj|jj |9y)z Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base class PreTrainedModel N)itemsrrxr prune_heads)r6heads_to_prunerxheadss r8 _prune_headszBlip2QFormerModel._prune_headssE +002 CLE5 LL  u % / / ; ;E B Cr9r input_shapedevice has_queryrgc |jdk(r|dddddddf}nF|jdk(r|ddddddf}n%tdj||j|j |j }d|z dz}|S)a= Makes broadcastable attention and causal masks so that future and masked tokens are ignored. Arguments: attention_mask (`torch.Tensor`): Mask with ones indicating tokens to attend to, zeros for tokens to ignore. input_shape (`Tuple[int]`): The shape of the input to the model. device (`torch.device`): The device of the input to the model. Returns: `torch.Tensor` The extended attention mask, with a the same dtype as `attention_mask.dtype`. rTNrZzAWrong shape for input_ids (shape {}) or attention_mask (shape {})rig?g)r<rformatr?rBrC)r6rrrrextended_attention_masks r8get_extended_attention_maskz-Blip2QFormerModel.get_extended_attention_masks.    1 $&4Qa]&C #    !Q &'5QdA5E&F #SZZ!5!5 #:"<"<4::"<"N#&)@#@H"L&&r9c |j|dd} | j|j} | j} | jd} t j | |jjdft jj|j}t j|| jgd}||n|jj}| | n|jj} | | n|jj} |,|ddjd |jj z nd}|jjd}|j#| |j| }|jdd }|\} }|j}|j%|j&}|}|t j | ||zf| }|j)|||}|t+|t,r|dj\}}}n|j\}}}||f}t+|t,r|Dcgc]}|j/|}}n?|)t j || }|j/|}n|j/|}nd}|j1||jj2j4}|j7||||||||| | | }|d}|dddddf} | s|j9|ddd|ddfSt;|| |j<|j>|j@|jBScc}w)a  encoder_hidden_states (`torch.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, `optional`): Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if the model is configured as a decoder. encoder_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, `optional`): Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. past_key_values (`tuple(tuple(torch.Tensor))` of length `config.n_layers` with each tuple having 4 tensors of: shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`. use_cache (`bool`, `optional`): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). ptT)return_tensorspaddingrrrir;NrZ)rDrErFr)r) rrrrrrrrrr)rrrrrr)"rrBrrDr?r0onesrr=longrArr4rrrrrHrrr isinstancelistinvert_attention_mask get_head_maskrrvrrr rrrr)!r6 text_input image_inputrrrrrrrrtextrDrI query_attsrrFrembedding_outputrrGrimage_embeds_frozenrencoder_batch_sizeencoder_sequence_length_encoder_hidden_shapemaskencoder_extended_attention_maskrsequence_outputrs! r8rJzBlip2QFormerModel.forwardsxB~~jt~Lwwt{{#NN __Q' ZZT->->-C-C-Ea-H IQVQ[Q[\__`d`k`kl J0C0C#D!L1B1N-TXT_T_TqTq$8$D $++JjJj &1%PP 3  !"ZZ*jCY6Y)ZdjkN#'"B"B>S^`f"g ! ,/6AVWXAYA^A^A`>"$;QAVA[A[A]>"$;Q$68O#P 0$7`v2wX\43M3Md3S2w/2w'/).4HQW)X&262L2LMc2d/262L2LMc2d/.2 +&&y$++2L2L2^2^_ ,, 2"7#B+/!5#%'  *!,'1a0 ???1m|mQ3F#GH H;-'+;;)77&11,==   G3xsM)F) NNNNNNNNNN)rKrLrMrNr r"rrrr0rprintrrrrJrOrPs@r8rrs{./0C +' +'3Z+' +'  +'  +'^"#!A r9r))typingrrrr0torch.utils.checkpointr transformersrtransformers.activationsrrtransformers.modeling_outputsr r r /transformers.models.blip_2.configuration_blip_2r r *transformers.models.blip_2.modeling_blip_2rrrrrtransformers.pytorch_utilsrtransformers.utilsrr get_loggerrKrModulerrRrrrwrrrr}r9r8r+s*) &E [A   H % 2"))2lBII@` "))` Hc cN ,=+=BX ,X r9