o wi;@sdZddlZddlZddlZddlmZmZmZddlZddl Zddlm Z ddl m Z m Z mZddlmZddlmZdd lmZdd lmZmZmZdd lmZdd lmZmZmZdd lm Z m!Z!m"Z"ddl#m$Z$e!%e&Z'ddZ(Gddde j)Z*Gddde j)Z+Gddde j)Z,GdddeZ-e GdddeZ.e Gddde.Z/e ddGd d!d!e.eZ0e d"dGd#d$d$e.Z1gd%Z2dS)&zPyTorch OpenAI ImageGPT model.N)AnyOptionalUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss)ACT2FN)GenerationMixin)GradientCheckpointingLayer))BaseModelOutputWithPastAndCrossAttentions!CausalLMOutputWithCrossAttentions SequenceClassifierOutputWithPast)PreTrainedModel)Conv1D find_pruneable_heads_and_indicesprune_conv1d_layer)auto_docstringlogging torch_float)ImageGPTConfigc Csz ddl}ddl}Wn tytdwtj|}td||j |}g}g}|D]"\} } td| d| |j || } | | | | q2t||D]\} } | dd} | d} td d | Dsy| d d vrtd d| qZ|} | d dvrt| d} | D]} |d| r|d| }n| g}|ddks|ddkrt| d} n|ddkrt| d} nw|ddks|ddkrt| |d} t| d} n^|ddvrt| d} t| d} nMt| dkr| ddkr|ddkrt| |d} t| d} n+|ddkr"t| d } t| d} n|dd!kr4t| d} t| d} nt| |d} t|d"krLt|d}| |} qt| dkr[| ddksp| d dksp| d d!ksp| d dkrqn%z | j| jks{JWnty}z |j| j| jf7_d}~wwtd#| | d d$krt| |j|jj| jddd|jf<qZ| d d%krt| |j|jj| jdd|jd"|jf<qZ| d d&krt| |j|jj| jddd"|jdf<qZt| dkr| ddkr| d"dkrt| |j|j| _qZ| d dkr-t| | _qZ| d dkrFt| | jd|j dddf<qZ| d d!krVt| | jd <qZt| | _qZ|S)'z0 Load tf checkpoints in a pytorch model rNzLoading a TensorFlow model in PyTorch, requires TensorFlow to be installed. Please see https://www.tensorflow.org/install/ for installation instructions.z&Converting TensorFlow checkpoint from zLoading TF weight z with shape /css|]}|dvVqdS))adam_vadam_mAdamWeightDecayOptimizerAdamWeightDecayOptimizer_1 global_stepN).0nr r k/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/transformers/models/imagegpt/modeling_imagegpt.py Qs  z.load_tf_weights_in_imagegpt..)_stepz Skipping {})wtet transformerz [A-Za-z]+\d+z(\d+)wgweightbbiaswpewte)q_projk_projv_projc_attnr rattnc_projr'lm_headsoszInitialize PyTorch weight r0r1r2)!re tensorflow ImportErrorloggererrorospathabspathinfotrainlist_variables load_variableappendsqueezezipsplitanyformatjoingetattr fullmatchlenintshapeAssertionErrorargstorch from_numpyreshapen_embdTdata vocab_size)modelconfigimagegpt_checkpoint_pathr9tftf_path init_varsnamesarraysnamerParraypointerm_name scope_namesnumer r r#load_tf_weights_in_imagegpt0s                   *      F.2*$ricsBeZdZd deedeffdd Zdejdejfdd Z Z S) ImageGPTLayerNormh㈵> hidden_sizeepscs&t||_tt||_dSN)super__init__rmr ParameterrSTensorr+)selfrlrm __class__r r#rps zImageGPTLayerNorm.__init__tensorreturncCs4|ttjt|ddd|j}||j}|S)Nr%T)axiskeepdim)rSsqrtmeansquarermr+)rsrvr r r#forwards& zImageGPTLayerNorm.forward)rk) __name__ __module__ __qualname__tuplerOfloatrprSrrr} __classcell__r r rtr#rjsrjcseZdZddeedeeffdd ZddZdd d Zdd d Z d dZ ddZ       dde j deedee j dee j dee j dee j deedeedefddZZS)ImageGPTAttentionFNis_cross_attention layer_idxcsFt|j}|jdttj||ftjddd||dd|jdt ddd|j |_ |j |_ |j |j |_|j |_|j|j |j krUtd|j d |j d |j|_||_|j|_||_|j|_|jr}td |j |j |_t|j |j |_n td |j |j |_t|j |j |_t|j|_t|j|_t |_!dS) Nr-dtyperF) persistent masked_biasgz=`embed_dim` must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).r8r )"rorpmax_position_embeddingsregister_bufferrStrilonesboolviewrvrl embed_dimnum_attention_heads num_headshead_dim split_size ValueErrorscale_attn_weightsrscale_attn_by_inverse_layer_idxrreorder_and_upcast_attnrr3q_attnr5rDropout attn_pdrop attn_dropout resid_pdrop resid_dropoutset pruned_heads)rsr[rr max_positionsrtr r#rpsB   zImageGPTAttention.__init__cCst|dkrdSt||j|j|j\}}t|||j|d|jg}t|j |dd|_ t|j |dd|_ |j|j|jt||_|jt||_|j ||_dS)Nrr8rdim) rNrrrrrScatrrr3r5union)rsheadsindex index_attnr r r# prune_headss  zImageGPTAttention.prune_headsc Cst||dd}|jr|t|dd}|jr$|t|jd}|j s]|d|d}}|j dddd|||d|f} t |j j } tj| |j |jd} t| || }|dure||}tjdd|}||j }||}|dur||}t||} | |fS)Nr%?rrdevicer)rSmatmul transposerrsizerrrrr-finforminrvrwhererSoftmaxtyper) rsquerykeyvalueattention_mask head_mask attn_weights query_length key_length causal_mask mask_value attn_outputr r r#_attns(&   zImageGPTAttention._attncCs|\}}}} |\} } } } tj|||| tj|jd} d} |jr.| t|dd} |jr:| t|jd} tj |jj dd1| d|| | dd d| | }}tj | ||d | d } | |||| } Wdn1swwY|js|d|d}}|jdddd|||d|f}t| jj}tj|| j| jd}t|| |} |dur| |} tjdd | } | jtjkrtd | |j} || } |dur| |} t| |}|| fS) Nr?r%rrF)enabledrr)betaalpharzDError with upcasting, attn_weights does not have dtype torch.float32)rrSemptyfloat32rrrrrautocastrrUrbaddbmmrr-rrrrvrrr RuntimeErrorrr)rsrrrrrbszr q_seq_lendk_ k_seq_lenr scale_factorqkrrrrrr r r#_upcast_and_reordered_attn s<&&    z,ImageGPTAttention._upcast_and_reordered_attncCs2|dd||f}|j|}|ddddS)zJ Splits hidden_size dim into attn_head_size and num_heads Nr%rr8rr )rrpermutersrvrattn_head_size new_shaper r r# _split_heads>s zImageGPTAttention._split_headscCs8|dddd}|dd||f}||S)zS Merges attn_head_size dim and num_attn_heads dim into hidden_size rr8rr Nr)r contiguousrrrr r r# _merge_headsFs zImageGPTAttention._merge_heads hidden_states layer_pastrrencoder_hidden_statesencoder_attention_mask use_cacheoutput_attentionsrwc CsT|dur"t|ds td||} ||j|jdd\} } |}n||j|jdd\} } } || |j|j} || |j|j} || |j|j} |dure|\} } t j | | fdd} t j | | fdd} |durn| | f}nd}|j r| | | | ||\}}n | | | | ||\}}|||j|j}||}||}||f}|r||f7}|S)NrzIf class is used as cross attention, the weights `q_attn` have to be defined. Please make sure to instantiate class with `ImageGPTAttention(..., is_cross_attention=True)`.r8rrT)hasattrrrr3rHrrrrrSrrrrrr5r)rsrrrrrrrrrrrpast_key past_valuepresentrroutputsr r r#r}Ns:      zImageGPTAttention.forward)FN)NNNNNNNFF)r~rrrrrOrprrrrrrSrrrr}rr r rtr#rsB +  &4   rcs2eZdZfddZdejdejfddZZS) ImageGPTMLPcsFt|j}t|||_t|||_t|j|_t |j |_ dSrn) rorprlrc_fcr5r activation_functionactrrrdropout)rsintermediate_sizer[rrtr r#rps    zImageGPTMLP.__init__rrwcCs,||}||}||}||}|Srn)rrr5r)rsrr r r#r}s    zImageGPTMLP.forward)r~rrrprSrrr}rr r rtr#rs rcseZdZdfdd Z       ddejdeedeejdeejd eejd eejd eed eed efddZ Z S) ImageGPTBlockNcst|j}|jdur|jnd|}t||jd|_t||d|_t||jd|_ |j r>t|d|d|_ t||jd|_ t |||_dS)NrmrT)rr)rorprln_innerrjlayer_norm_epsilonln_1rr4ln_2add_cross_attentioncrossattention ln_cross_attnrmlp)rsr[rrl inner_dimrtr r#rps zImageGPTBlock.__init__Frrrrrrrrrwc Cs|} ||}|j||||||d} | d} | dd} | | }|durSt|ds1td|d|} ||}|j||||||d} | d} | | }| | dd} |} ||}||}| |}|f|rl| } | S| dd} | S) N)rrrrrrrrz'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`)rrrrrr8)rr4rrrrrr)rsrrrrrrrrresidual attn_outputsrrcross_attn_outputsfeed_forward_hidden_statesr r r#r}sN       zImageGPTBlock.forwardrnr) r~rrrprSrrrrrr}rr r rtr#rs8  rcs>eZdZeZeZdZdZdZ dgZ fddZ ddZ Z S) ImageGPTPreTrainedModelr( input_idsTrcstj|i|dSrn)rorp)rsinputskwargsrtr r#rpsz ImageGPTPreTrainedModel.__init__cCst|tjtfr|jjjd|jjd|j dur|j j n,t|tj r?|jjjd|jjd|j dur>|jj|j  n t|t rK|jjd|D]\}}d|vrnd|vrn|jjd|jjtd|jjdqOdS)zInitialize the weights.g)r{stdNrr5r+r8) isinstancerLinearrr+rXnormal_r[initializer_ranger-zero_ Embedding padding_idxrjfill_named_parametersmathrzn_layer)rsmodulerbpr r r# _init_weightss"     &z%ImageGPTPreTrainedModel._init_weights)r~rrr config_classriload_tf_weightsbase_model_prefixmain_input_namesupports_gradient_checkpointing_no_split_modulesrprrr r rtr#rs rc"seZdZdeffdd ZddZddZdd Ze dd e e j d e e e e j d e e j de e j de e j de e j de e j de e j de e j de e de e de e de e dedee effddZZS) ImageGPTModelr[cstj|_tj|j|_tj|j|_ t j |_ t fddtjD|_t|jjd|_d|_d|_d|_|dS)Ncsg|]}t|dqS)r)r)r!ir[r r# sz*ImageGPTModel.__init__..rF)rorprlrrr rYr/rr.r embd_pdropdrop ModuleListrangenum_hidden_layershrjrln_fmodel_parallel device_mapgradient_checkpointing post_initrsr[rtrr#rp s   zImageGPTModel.__init__cC|jSrnr/rsr r r#get_input_embeddingsz"ImageGPTModel.get_input_embeddingscC ||_dSrnr)rsnew_embeddingsr r r#set_input_embeddings z"ImageGPTModel.set_input_embeddingscCs(|D] \}}|j|j|qdS)zv Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} N)itemsr!r4r)rsheads_to_prunelayerrr r r# _prune_heads"szImageGPTModel._prune_headsNrpast_key_valuesrtoken_type_ids position_idsr inputs_embedsrrrroutput_hidden_states return_dictrrwc$ sd|vrtdt|durtd|d}| dur| n|jj} | dur'| n|jj} | dur1| n|jj} | dur;| n|jj } |durK|durKtd|durg| ||| }| d|d}|j d}n|dury| dd}|j d}ntd|dur|jn|j}|dur| d|d}|durd}tdgt|j}n |dd d }|durtj||d|tj|d }|d}|dur|dkrtd | |d}|ddddddf}|j|jd }d |t|jj}|jjr|dur| \}}}||f}| durtj||d} || } nd} |||jj}|dur/||}| |}|||j|durJ||}||!| df}|j"ri|j#ri| rit$%dd} | rndnd}| rudnd}| r|jjrdnd}| rdnd}t&t'|j|D]\}\}} |j(rtj)*j| durtfdd| D} |dur|j}t+|tj,r|j}| r|f}|| ||||| | | d}!|!d| dur||!df}| r||!| rdndf}|jjr||!| r dndf}|j(r<|j-.D]!\}"}#||#dkr:dt/|"|j0kr:dt/|"dqq|1j || rP|f}| satdd||||fDSt2||||dS)a input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0][0].shape[-2]` (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as `input_ids`. Indices can be obtained using [`AutoImageProcessor`]. See [`ImageGPTImageProcessor.__call__`] for details. Examples: ```python >>> from transformers import AutoImageProcessor, ImageGPTModel >>> from PIL import Image >>> import requests >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) >>> image_processor = AutoImageProcessor.from_pretrained("openai/imagegpt-small") >>> model = ImageGPTModel.from_pretrained("openai/imagegpt-small") >>> inputs = image_processor(images=image, return_tensors="pt") >>> outputs = model(**inputs) >>> last_hidden_states = outputs.last_hidden_state ``` pixel_values`The `pixel_values` argument is deprecated and will be removed in v4.47, use `input_ids` instead.N_You cannot pass both `pixel_values` and `input_ids`. Please make sure to only pass `input_ids`.zDYou cannot specify both input_ids and inputs_embeds at the same timer%rz5You have to specify either input_ids or inputs_embedsrrz$batch_size has to be defined and > 0rr)rzZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Fr c3s|] }|jVqdSrn)torr! past_staterr r#r$sz(ImageGPTModel.forward..)rrrTrr8r zcuda:css|] }|dur|VqdSrnr )r!vr r r#r$s)last_hidden_stater6r attentionscross_attentions)3warningswarn FutureWarningrpopr[rr:ruse_return_dict%warn_if_padding_and_no_attention_maskrrrPrrrNr!rSarangelong unsqueezer?rrrrrinvert_attention_mask get_head_maskrr/r.rr%trainingr< warning_once enumeraterGr#cuda set_devicerrrr$r2str last_devicer"r )$rsrr6rr7r8rr9rrrrr:r;r input_shape batch_sizer past_lengthencoder_batch_sizeencoder_sequence_lengthrencoder_hidden_shapeposition_embedstoken_type_embeds output_shapepresentsall_self_attentionsall_cross_attentionsall_hidden_statesrblockrrrrCr rBr#r})s/                     "    zImageGPTModel.forward) NNNNNNNNNNNNN)r~rrrrpr+r0r5rrrSrrrrrrr r}rr r rtr#rsd      rz The ImageGPT Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). ) custom_introc$s0eZdZdgZdeffdd ZddZddZe dd e e j d e e e e j d e e j d e e j de e j de e j de e j de e j de e j de e j de e de e de e de e dedee eff ddZed e e e j de j de e e j fddZZS) ImageGPTForCausalImageModelingzlm_head.weightr[csHt|t||_tj|j|jddd|_d|_ d|_ | dS)NrFr-) rorprr(rrrVrYr6r#r$r&r'rtr r#rp s   z'ImageGPTForCausalImageModeling.__init__cCr(rnr6r*r r r#get_output_embeddingsr,z4ImageGPTForCausalImageModeling.get_output_embeddingscCr-rnrjr.r r r#set_output_embeddingsr1z4ImageGPTForCausalImageModeling.set_output_embeddingsNrr6rr7r8rr9rrlabelsrrr:r;rrwcKsd|vrtdt|durtd|d}|dur|n|jj}|j||||||||| | | | |d }|d}||}d}| durk|dddddf }| dd df }t }|| d| d| d}|s|f|d d}|dur|f|S|St |||j|j|j|jd S) a% input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0][0].shape[-2]` (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as `input_ids`. Indices can be obtained using [`AutoImageProcessor`]. See [`ImageGPTImageProcessor.__call__`] for details. labels (`torch.LongTensor` of shape `(batch_size, input_ids_length)`, *optional*): Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` Examples: ```python >>> from transformers import AutoImageProcessor, ImageGPTForCausalImageModeling >>> import torch >>> import matplotlib.pyplot as plt >>> import numpy as np >>> image_processor = AutoImageProcessor.from_pretrained("openai/imagegpt-small") >>> model = ImageGPTForCausalImageModeling.from_pretrained("openai/imagegpt-small") >>> device = torch.device("cuda" if torch.cuda.is_available() else "cpu") >>> model.to(device) # doctest: +IGNORE_RESULT >>> # unconditional generation of 8 images >>> batch_size = 4 >>> context = torch.full((batch_size, 1), model.config.vocab_size - 1) # initialize with SOS token >>> context = context.to(device) >>> output = model.generate( ... input_ids=context, max_length=model.config.n_positions + 1, temperature=1.0, do_sample=True, top_k=40 ... ) >>> clusters = image_processor.clusters >>> height = image_processor.size["height"] >>> width = image_processor.size["width"] >>> samples = output[:, 1:].detach().cpu().numpy() >>> samples_img = [ ... np.reshape(np.rint(127.5 * (clusters[s] + 1.0)), [height, width, 3]).astype(np.uint8) for s in samples ... ] # convert color cluster tokens back to pixels >>> f, axes = plt.subplots(1, batch_size, dpi=300) >>> for img, ax in zip(samples_img, axes): # doctest: +IGNORE_RESULT ... ax.axis("off") ... ax.imshow(img) ```r<r=Nr>) r6rr7r8rr9rrrrr:r;r.r%r)losslogitsr6rrErF)rGrHrIrrJr[rKr(r6rrrrrr6rrErF)rsrr6rr7r8rr9rrrmrrr:r;rtransformer_outputsr lm_logitsrn shift_logits shift_labelsloss_fctoutputr r r#r}sZF  z&ImageGPTForCausalImageModeling.forwardbeam_idxcstfdd|DS)a This function is used to re-order the `past_key_values` cache if [`~PreTrainedModel.beam_search`] or [`~PreTrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct beam_idx at every generation step. c3s&|]}tfdd|DVqdS)c3s$|] }|d|jVqdS)rN) index_selectr?rr@rvr r#r$s"zJImageGPTForCausalImageModeling._reorder_cache...Nr)r!rrxr r#r$s  z@ImageGPTForCausalImageModeling._reorder_cache..ry)r6rvr rxr#_reorder_caches z-ImageGPTForCausalImageModeling._reorder_cache)NNNNNNNNNNNNNN)r~rr_tied_weights_keysrrprkrlrrrSrrrrrrrr} staticmethodrzrr r rtr#rhsz        }  rhz The ImageGPT Model transformer with an image classification head on top (linear layer). [`ImageGPTForImageClassification`] average-pools the hidden states in order to do the classification. c seZdZdeffdd Ze            ddeejdee e ejdeejdeejd eejd eejd eejd eejd ee dee dee dee de de e e ffddZZS)ImageGPTForImageClassificationr[cs@t||j|_t||_tj|j|jdd|_| dS)NFri) rorp num_labelsrr(rrrVscorer&r'rtr r#rps   z'ImageGPTForImageClassification.__init__Nrr6rr7r8rr9rmrrr:r;rrwc  Ksd| vrtdt|durtd| d}| dur| n|jj} |j|||||||| | | | d }|d}|jdd}| |}d}|dur|jj durp|j dkrVd |j_ n|j dkrl|j t jksg|j t jkrld |j_ nd |j_ |jj d krt}|j dkr|||}n+|||}n%|jj d krt}||d |j |d }n|jj d krt}|||}| s|f|dd}|dur|f|S|St|||j|j|jd S)ax input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0][0].shape[-2]` (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as `input_ids`. Indices can be obtained using [`AutoImageProcessor`]. See [`ImageGPTImageProcessor.__call__`] for details. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Examples: ```python >>> from transformers import AutoImageProcessor, ImageGPTForImageClassification >>> from PIL import Image >>> import requests >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) >>> image_processor = AutoImageProcessor.from_pretrained("openai/imagegpt-small") >>> model = ImageGPTForImageClassification.from_pretrained("openai/imagegpt-small") >>> inputs = image_processor(images=image, return_tensors="pt") >>> outputs = model(**inputs) >>> logits = outputs.logits ```r<r=Nr>) r6rr7r8rr9rrr:r;rrr regressionsingle_label_classificationmulti_label_classificationr%)rnror6rrE)rGrHrIrrJr[rKr(r{r problem_typer~rrSrNrOrrFrrrrr6rrE)rsrr6rr7r8rr9rmrrr:r;rrprpooled_hidden_statesrornrtrur r r#r}sp2      "       z&ImageGPTForImageClassification.forward) NNNNNNNNNNNN)r~rrrrprrrSrrrrrrrr}rr r rtr#r}sX       r})rhr}rrri)3__doc__r r>rGtypingrrrrStorch.utils.checkpointrtorch.nnrrr activationsr generationr modeling_layersr modeling_outputsr rrmodeling_utilsr pytorch_utilsrrrutilsrrrconfiguration_imagegptr get_loggerr~r<riModulerjrrrrrrhr}__all__r r r r#sP       l \K&}"