o Ti@srddlZddlmZddlZddlZddlZddlmZddlm Z m Z ddl m Z ddl mZmZmZmZddlZddlZddlZddlmmZddlZddlmZmZddlm Z ddl!m"Z"m#Z#dd l$m%Z%dd l&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-dd l.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4d d l5m6Z6e1rddl7m8Z8ddl9m:Z:ddl;mZ>e-rddl?Z?e0rddl@mAZAmBZBe GdddZCe GdddZD  ddeEejFdeGdeHdeeGdejFf ddZIe Gdd d ZJe Gd!d"d"ZKeL #dd$ejFdeMejFejFeGffd%d&ZNd'e(deOeHePffd(d)ZQdd+ejFd,eGd-eeGePfd.eGdejFf d/d0ZRd1ejjSddfd2d3ZTdd5d6ZUGd7d8d8ZVd9d:ZWd;e6dee'fdd?ZYd;e6dd@fdAdBZZddDdEZ[ddFdGZ\dHej]ddfdIdJZ^dKZ_dLZ`e GdMdNdNe+Zae GdOdPdPe,ZbejcfdQejFdejFfdRdSZdd1ejjSdTejFdUeGdVeGdeMejFejFejFff dWdXZed1ejjSdTejFdUeGde/fdYdZZf [dd1ejjSd\eGd]egd^egdejjSf d_d`ZhdaeGdUeGdbejFdejFfdcddZideejjSdfejFdUeGdge)deMejFejFff dhdiZjeLd1ejjSdfejFdjeGdUeGdge)f dkdlZkdmeeGdneGdoeOeHeEeGfdpeGdqeOeHeEeGfdreGdseOeHeEeGffdtduZldveGdqeOeHeEeGfdseOeHeEeGffdwdxZmdyejFdaeGdUeGdeMejFejFffdzd{Zndd|d}Zod~ejFde*deEeHfddZp    ddeeHdeHdeHdeeHdeEeHdeeHdeHdeeHdeeHdeeHdeeHde"fddZqdeeHfddZrdeHdej]ddfddZsdejFdejFdeejFeMejFdfffddZtdejFdejFdeejFeMejFdfffddZudejFfddZvddeGdejFfddZw ddeEeHdeEeHdeOeHeEePfdeEePdeGdeGddfddZxdS)N)deque) dataclassfield)version)AnyLiteralOptionalUnion) Accelerator PartialState)AcceleratorState) ModelCard ModelCardData) pad_sequence)BitsAndBytesConfigEvalPredictionGenerationConfigPreTrainedTokenizerBase TrainerStateTrainingArgumentsis_comet_available) ModelOutputis_peft_availableis_rich_availableis_torch_mlu_availableis_torch_npu_availableis_torch_xpu_available) ModelConfig)Console)Panel)Table)Text) LoraConfig PeftConfigc@szeZdZUdZeed<dZeed<dZeed<dZ e ed<d Z e ed <d d Z d e ee efdee ejffddZdS)DataCollatorForChatMLz3 Data collator for ChatML format datasets. tokenizer ignore_indexN max_lengthprompt prompt_keymessages messages_keycCs6|jjdur td|jdurt|jjd|_dSdS)NzTThe tokenizer does not have a pad token. Please set `pad_token_id` in the tokenizer.i)r& pad_token_id ValueErrorr)minmodel_max_lengthselfr4E/home/ubuntu/.local/lib/python3.10/site-packages/trl/trainer/utils.py __post_init__Ps  z#DataCollatorForChatML.__post_init__examplesreturnc Cs4g}g}g}g}g}|D]}||jd}|dur+||jdd} |jj| ddd}d|vri||j} |jj| ddd} |j| d|jdddd} || dd|vr\|| dn,|dgt| dn||dd|vr|||dn |dgt|d|j|dt|ddddd} || d|| d|jgt|d}t| d}|d|d||d<||q d d |D}d d |D}d d |D}t |d |jj d}t |d dd}t |d |jd}dd |D}dd |D}t |d |jj d}t |d dd}|||||dS)NFT)tokenizeadd_generation_prompt input_ids) truncationr)paddingreturn_tensorsadd_special_tokensattention_maskcSg|] }tj|tjdqSdtypetorchtensorlong.0idsr4r4r5 z2DataCollatorForChatML.__call__..cSrCrDrGrLmaskr4r4r5rNrOcSrCrDrG)rLlabelr4r4r5rNrOleft) padding_side padding_valuercSrCrDrGrKr4r4r5rNrOcSrCrDrGrPr4r4r5rNrO)r<rAlabelspromptsprompt_attention_mask) getr+r-r&apply_chat_templater)appendlenr(padr.)r3r7r<rAprompts_input_idsrXrVexampleformatted_promptr*messageformatted_messagetokenized_messagetokenized_promptrRcompletion_start_idxr4r4r5__call__Ws~     zDataCollatorForChatML.__call__)__name__ __module__ __qualname____doc__r__annotations__r(intr)r+strr-r6listdictrrHTensorrfr4r4r4r5r%Ds     ,r%c@speZdZUdZeed<dZeee fed<dZ e e ed<dZ e ed<d eee efd ee effd d ZdS) RewardDataCollatorWithPaddinga\ Reward DataCollator class that pads the inputs to the maximum length of the batch. Args: tokenizer (`PreTrainedTokenizerBase`): The tokenizer used for encoding the data. padding (`Union[bool, str, `PaddingStrategy`]`, `optional`, defaults to `True`): padding_strategy to pass to the tokenizer. pad_to_multiple_of (`int` or `None`, `optional`, defaults to `None`): If set will pad the sequence to a multiple of the provided value. return_tensors (`str`, `optional`, defaults to `"pt"`): The tensor type to use. r&Tr>Npad_to_multiple_ofptr?featuresr8c Csg}g}g}d|dv}|D]7}d|vs d|vs d|vs d|vr$td||d|dd||d|dd|rE||dq|jj||j|j|jd }|jj||j|j|jd }|d |d |d |d d d } |r~tj|tj d}|| d<| S)Nmarginrinput_ids_choseninput_ids_rejectedattention_mask_chosenattention_mask_rejectedz{The features should include `input_ids_chosen`, `attention_mask_chosen`, `input_ids_rejected` and `attention_mask_rejected`)r<rA)r>rrr?r<rAT)rvrxrwry return_lossrE) r/r[r&r]r>rrr?rHrIfloat) r3rtfeatures_chosenfeatures_rejectedru has_marginfeature batch_chosenbatch_rejectedbatchr4r4r5rfs^ z&RewardDataCollatorWithPadding.__call__)rgrhrirjrrkr>r boolrmrrrrlr?rnrorrfr4r4r4r5rqs  *rqrighttensorsrUrTrrr8c Cstdd|Dd}|dur%|d|}|dkr%|d||7<tjt|g|R||dj|djd}t|D]>\}}|dkrP|d|j d} n |dkrWd} nt dt | | |j d} | ft d d |j d dD} |||| <q>|S) a Pads a list of tensors to the same shape along the first dimension. Args: tensors (`list[torch.Tensor]`): List of input tensors to pad. padding_value (`int`): Value to use for padding. Default is 0. padding_side (`str`): Side on which to add padding. Must be 'left' or 'right'. Default is 'right'. pad_to_multiple_of (`int`, *optional*, defaults to `None`): If set will pad the sequence to a multiple of the provided value. Returns: `torch.Tensor`: A single tensor containing the padded tensors. Examples: ```python >>> import torch >>> pad([torch.tensor([1, 2, 3]), torch.tensor([4, 5])]) tensor([[1, 2, 3], [4, 5, 0]]) >>> pad([torch.tensor([[1, 2], [3, 4]]), torch.tensor([[5, 6]])]) tensor([[[1, 2], [3, 4]], [[5, 6], [0, 0]]]) ``` cSsg|]}|jqSr4)shaperLtr4r4r5rNszpad..rNrFdevicerSrz&padding_side must be 'left' or 'right'css|]}td|VqdS)rN)slice)rLsr4r4r5 1szpad..rB) npmaxtolistrHfullr\rFr enumeraterr/rtuple) rrUrTrr output_shape remainderoutputir seq_start seq_sliceslicesr4r4r5r]s ' *"r]c@s`eZdZUdZdZeed<dZeed<dZe e ed<de e e efd e e effd d Zd S) DPODataCollatorWithPaddinga DPO DataCollator class that pads the tokenized inputs to the maximum length of the batch. Args: pad_token_id (`int` defaults to 0): The tokenizer's pad_token_id. label_pad_token_id (`int`, defaults to -100): The label used for masking. is_encoder_decoder (`bool` or `None`, `optional`, defaults to `None`): Whether you model has an encoder_decoder architecture. rr.r'label_pad_token_idFis_encoder_decoderrtr8csi}|dD]̉dr|jrYfdd|D}dr2dr2|jdur.td|j}ndr:d}nd sCd vrG|j}ntd d t|d |d|<qdrk|jdurgtd|j}n!drt|j}ndr|d}ndrd}ntd d dvrd}nd}drtj ntj fdd|D}t |||d|<qdrt fdd|D|<qfdd|D|<q|S)Nr) _input_ids_attention_mask_labels _pixel_valuescsg|] }t|qSr4)rH LongTensorrLexkr4r5rNOz7DPODataCollatorWithPadding.__call__..r*r<zPadding is enabled, but the tokenizer is not configured with a padding token. Explicitly set `tokenizer.pad_token` (e.g. `tokenizer.pad_token = tokenizer.eos_token`) before calling the trainer.r)chosenrejected completiondecoderzUnexpected key in batch ''T) batch_firstrUrrr)prompt_input_idsrXrSrcsg|] }tj|dqSrD)rHrIrrFrr4r5rNrUrT_logpscg|]}|qSr4r4rrr4r5rNcrr4r4rrr4r5rNr) keysendswithr startswithr.r/rrrHfloat32int64r]rI)r3rt padded_batchto_padrUrTr4rr5rfIsT          z#DPODataCollatorWithPadding.__call__N)rgrhrirjr.rlrkrrrrrnrormrrfr4r4r4r5r7s  *rc@seZdZUdZeed<dZeed<dZeed<dZ eed<dZ eed <e d e j d eeeffd d ZdefddZededefddZdS)RunningMomentsz Calculates the running mean and standard deviation of a data stream. Reference: https://github.com/OpenLMLab/MOSS-RLHF/blob/40b91eb2f2b71b16919addede0341d2bef70825d/utils.py#L75 acceleratorrmeanrBstdvargW:countxsr8c Cs|jjrt|j|\}}}n |}tj|dd\}}||}}||j}|j|}||}|j |j|d|j||}||} |j||| 7_| |} | ||d |_ | |_ ||_| |||d fS)zT Updates running moments from batch's moments computed across ranks F)unbiasedrrB) ruse_distributedget_global_statisticsnumelrHvar_meanr{rrritemsqrtr) r3rxs_meanxs_varxs_countdelta tot_countnew_sumold_sumtot_sumnew_varr4r4r5updates   " $zRunningMoments.update json_pathcCsp|jjr6tj|ddd}tj|dddd}t|dd d }||Wd d S1s/wYd Sd S) zDSave the content of this instance in JSON format inside `json_path`.cSsdd|DS)NcSsi|] \}}|dkr||qS)rr4rLrvr4r4r5 rzARunningMoments.save_to_json....r4)xr4r4r5sz-RunningMoments.save_to_json..) dict_factoryrT)indent sort_keys wutf-8encodingN)ris_main_process dataclassesasdictjsondumpsopenwrite)r3r save_dict json_stringfr4r4r5 save_to_jsons "zRunningMoments.save_to_jsoncCsLt|dd }|}Wdn1swY|dd|it|S)z3Create an instance from the content of `json_path`.rrNrr4)rreadrloads)clsrrrtextr4r4r5load_from_jsons zRunningMoments.load_from_jsonN)rgrhrirjr rkrr{rrrrHno_gradrprrrmr classmethodrr4r4r4r5rs      rcpurc Cs||j}tj||dur|n|g|jd}||}|\}}||}t||d|dur7dn|}||}||} ||| ||fS)z Computes element-wise mean and variance of the tensor across processes. Reference: https://github.com/OpenLMLab/MOSS-RLHF/blob/40b91eb2f2b71b16919addede0341d2bef70825d/utils.py#L57C1-L73C75 NrrrB) torrHrIsumrreducemulr) rrrQr sum_and_count global_sumr global_meansum_var global_varr4r4r5rs , $ r eval_predc Cs|\}}|jdkr(tj|dd}tddt||D}tdd|D}n@|dddf|dddfk}t|}|dkrWtd |d t |dddfd t ||}||}tj|dd}tj||kt d  }d |iS)Nr)axiscSs0g|]\}}t||D] \}}|dkr |q qSr')zip)rL predictionrRplblr4r4r5rNs0z$compute_accuracy..cSs"g|] }|D]}|dkr|qqSrr4)rLrRrr4r4r5rNs"rrBz There are z out of zu instances where the predictions for both options are equal. These instances are ignored in the accuracy computation.rEaccuracy)ndimrargmaxarrayrrlrwarningswarnr\ UserWarningr{rr)r predictionsrV equal_maskequal_predictions_countrr4r4r5compute_accuracys&      rr9rIlength pad_valuedimcCsV|||kr |St|j}|||||<tj||tj||j|jdg|dS)Nrr)sizernrrHcatonesrFr)rIrrrpad_sizer4r4r5 pad_to_lengths rmodelcCs&|D] }t|tjjrd|_qdS)Nr)modules isinstancerHnnDropoutrrmoduler4r4r5disable_dropout_in_models r"c Cs:||}|||krt|d|d|d|||S)Nz, inexact division: z / z = )r/)abcustom_error_messageqr4r4r5 exact_divs "r(c@s(eZdZdZddZddZddZdS) PerPromptStatTrackeraI Class for tracking statistics per prompt. Mainly used to calculate advantage for the DPPO algorithm Args: buffer_size (`int`): Size of the buffer to keep for each prompt. min_count (`int`): Minimum number of samples to keep in the buffer before calculating the mean and std. cCs||_||_i|_dSN) buffer_size min_countstats)r3r+r,r4r4r5__init__(s zPerPromptStatTracker.__init__c Cst|}t|}t|}t|}|D]Q}|||k}||jvr,t|jd|j|<|j||t|j||j krKt |}t |d}nt |j|}t |j|d}||||||k<q|S)N)maxlengư>) rr unique empty_liker-rr+extendr\r,rr) r3rWrewardsr0 advantagesr*prompt_rewardsrrr4r4r5r-s       zPerPromptStatTracker.updatecCsdd|jDS)NcSs.i|]\}}|t|t|t|dqS))rrr)rrrr\rr4r4r5rCs.z2PerPromptStatTracker.get_stats..)r-itemsr2r4r4r5 get_statsBszPerPromptStatTracker.get_statsN)rgrhrirjr.rr7r4r4r4r5r)s   r)cst|D]3\}t|tjjsdvr|tj}qtfdddDr7t|dr7|j j tjkr7|tj }qdS)Nnormc3s|]}|vVqdSr*r4)rLrnamer4r5rJsz.peft_module_casting_to_bf16..)lm_head embed_tokenswtewpeweight) named_modulesrrHr LayerNormrranyhasattrr?rFbfloat16r r4r9r5peft_module_casting_to_bf16Fs  rE model_argscCs@|jrtd|j|j|j|jd}|S|jrtdd}|Sd}|S)NT) load_in_4bitbnb_4bit_compute_dtypebnb_4bit_quant_typebnb_4bit_use_double_quantbnb_4bit_quant_storage) load_in_8bit)rGr torch_dtyperIuse_bnb_nested_quantrL)rFquantization_configr4r4r5get_quantization_configPs rPcCs tjstrdtjiSdS)Nr#)rHcuda is_availablerr local_process_indexr4r4r4r5get_kbit_device_mapcs rTzOptional[PeftConfig]c CsJ|jdurdStstdt|j|j|j|j|jd|j |j |j d }|S)NFzYou need to have PEFT library installed in your environment, make sure to install `peft`. Make sure to run `pip install -U peft`.none) task_typertarget_modules lora_alpha lora_dropoutbias use_rslorause_doramodules_to_save) use_peftrr/r#lora_task_typelora_rlora_target_modulesrYrZr\r]lora_modules_to_save)rF peft_configr4r4r5get_peft_configjs$  recCsZtdg|jt|jj}t||j}|dkr+t|d|d|S|S)a1 Get the exponent cap of a value. This is used to cap the exponent of a value to avoid overflow. The formula is : log(value.dtype.max) E.g. For float32 data type, the maximum exponent value is 88.7228 to 4 decimal points. Args: value (`torch.Tensor`): The input tensor to obtain the data type decimal (`int`): The number of decimal points of the output exponent cap. eg: direct calling exp(log(torch.float32.max)) will result in inf so we cap the exponent to 88.7228 to avoid overflow. rBr ) rHzerosrrFfinforlogrfloor)valuedecimal vdtype_maxvdtype_log_maxr4r4r5 get_exp_caps" &rpcCs(|dkrt|n|}ttj||dS)Nr)r)rprHexpclamp)rlcapr4r4r5cap_expsrtdfcCshtstdt}tdd}|jD]}||q|D]\}}|j|t  q| |dS)NzgThe function `print_rich_table` requires the `rich` library. Please install it with `pip install rich`.T) show_lines) r ImportErrorrr!columns add_columniterrowsadd_rowastypermrprint)ruconsoletablecolumn_rowr4r4r5print_rich_tables   rzT{% for message in messages %}{{' ' + message['content']}}{% endfor %}{{ eos_token }}z{% for message in messages %}{{message['role'].capitalize() + ': ' + message['content'] + ' '}}{% endfor %}{% if add_generation_prompt %}{{ 'Assistant:' }}{% endif %}c@seZdZUdZeed<dS)OnlineTrainerStaterepisodeN)rgrhrirrlrkr4r4r4r5rs rcsneZdZUdZedddidZeed<edddidZe e ed <eddd idZ e e ed <eddd idZ e eed <edddidZeed<edddidZe eed<edddidZeed<edddidZeed<edddidZeed<edddidZe eded<edddidZe eed<ed dd!idZeed"<eddd#idZe eed$<ed%dd&idZe ed'<eddd(idZe eed)<eddd*idZe eed+<eddd,idZe eed-<eddd.idZe eed/<eddd0idZe eed1<eddd2idZe eed3<eddd4idZ e eed5<ed6dd7idZ!e ed8<fd9d:Z"Z#S);OnPolicyConfiga Base configuration class for on-policy trainers. This class includes only the parameters that are specific to some on-policy training. For a full list of training arguments, please refer to the [`~transformers.TrainingArguments`] documentation. Note that default values in this class may differ from those in [`~transformers.TrainingArguments`]. Using [`~transformers.HfArgumentParser`] we can turn this class into [argparse](https://docs.python.org/3/library/argparse#module-argparse) arguments that can be specified on the command line. Parameters: run_name (`str` or `None`, *optional*, defaults to `None`): Name of the run. dataset_num_proc (`int` or `None`, *optional*, defaults to `None`): Number of processes to use for processing the dataset. num_mini_batches (`int`, *optional*, defaults to `1`): Number of minibatches to split a batch into. total_episodes (`int` or `None`, *optional*, defaults to `None`): Total number of episodes in the dataset. local_rollout_forward_batch_size (`int`, *optional*, defaults to `64`): Per rank no grad forward pass in the rollout phase. num_sample_generations (`int`, *optional*, defaults to `10`): Number of debugging samples generations (i.e., `generate_completions` calls) throughout training. response_length (`int`, *optional*, defaults to `53`): Length of the response. stop_token (`str` or `None`, *optional*, defaults to `None`): Specifies the stop token to use for text generation. This parameter is mutually exclusive with `stop_token_id`. - `None`: No stop token is applied, unless `stop_token_id` is specified. - `'eos'`: Uses the tokenizer's `eos_token`. stop_token_id (`int` or `None`, *optional*, defaults to `None`): Specifies the ID of the stop token to use for text generation. If `None`, no stop token ID is applied, unless `stop_token` is specified. This parameter is mutually exclusive with `stop_token`. temperature (`float`, *optional*, defaults to `0.7`): Sampling temperature. missing_eos_penalty (`float` or `None`, *optional*, defaults to `None`): Penalty applied to the score when the model fails to generate an EOS token. This is useful to encourage to generate completions shorter than the maximum length (`max_new_tokens`). The penalty must be a positive value. sft_model_path (`str`, *optional*, defaults to `"EleutherAI/pythia-160m"`): Path to the SFT model. world_size (`int` or `None`, *optional*, defaults to `None`): Number of processes (GPUs) to use for the training. num_total_batches (`int` or `None`, *optional*, defaults to `None`): Number of total batches to train. micro_batch_size (`int` or `None`, *optional*, defaults to `None`): Micro batch size across devices (HF's `per_device_train_batch_size` * `world_size`). local_batch_size (`int` or `None`, *optional*, defaults to `None`): Batch size per GPU (HF's `per_device_train_batch_size` * `gradient_accumulation_steps`). batch_size (`int` or `None`, *optional*, defaults to `None`): Batch size across devices (HF's `per_device_train_batch_size` * `world_size` * `gradient_accumulation_steps`). local_mini_batch_size (`int` or `None`, *optional*, defaults to `None`): Mini batch size per GPU. mini_batch_size (`int` or `None`, *optional*, defaults to `None`): Mini batch size across GPUs. push_to_hub (`bool`, *optional*, defaults to `False`): Whether to push the model to the Hub after training. rghelpzLog every X updates steps. Should be an integer or a float in range `[0,1)`. If smaller than 1, will be interpreted as ratio of total training steps.)defaultmetadata logging_stepsNzWhether to use bf16 (mixed) precision instead of 32-bit. Requires Ampere or higher NVIDIA architecture or Intel XPU or using CPU (use_cpu) or Ascend NPU. If not set, it defaults to `True` if `fp16` is not set.bf16zName of the run.run_namez6Number of processes to use for processing the dataset.dataset_num_procrBz,Number of minibatches to split a batch into.num_mini_batchesz(Total number of episodes in the dataset.total_episodes@z3Per rank no grad forward pass in the rollout phase. local_rollout_forward_batch_sizezaNumber of debugging samples generations (i.e., `generate_completions` calls) throughout training.num_sample_generations5zLength of the response.response_lengthzoSpecifies the stop token to use for text generation. This parameter is mutually exclusive with `stop_token_id`.eos stop_tokenzSpecifies the ID of the stop token to use for text generation. If `None`, no stop token ID is applied, unless `stop_token` is specified. This parameter is mutually exclusive with `stop_token`. stop_token_idgffffff?zSampling temperature. temperaturezPenalty applied to the score when the model fails to generate an EOS token. This is useful to encourage to generate completions shorter than the maximum length (`max_new_tokens`). The penalty must be a positive value.missing_eos_penaltyzEleutherAI/pythia-160mzPath to the SFT model.sft_model_pathz3Number of processes (GPUs) to use for the training. world_sizez!Number of total batches to train.num_total_batcheszTMicro batch size across devices (HF's `per_device_train_batch_size` * `world_size`).micro_batch_sizezXBatch size per GPU (HF's `per_device_train_batch_size` * `gradient_accumulation_steps`).local_batch_sizeznBatch size across devices (HF's `per_device_train_batch_size` * `world_size` * `gradient_accumulation_steps`). batch_sizezMini batch size per GPU.local_mini_batch_sizezMini batch size across GPUs.mini_batch_sizeFz4Whether to push the model to the Hub after training. push_to_hubcs(|jdur |j n|j|_tdSr*)rfp16superr6r2 __class__r4r5r6fszOnPolicyConfig.__post_init__)$rgrhrirjrrr{rkrrrrrmrrlrrrrrrrrrrrrrrrrrrrr6 __classcell__r4r4rr5rs @ rboolscCs<|d}|||tj|||jd}tj|ddjS)a Takes an N-dimensional bool tensor and returns an (N-1)-dimensional tensor of integers giving the position of the first True in each "row". Returns the length of the rows (bools.size(-1)) if no element is True in a given row. Args: bools (`torch.Tensor`): An N-dimensional boolean tensor. dtype (`torch.dtype`, optional): The desired data type of the output tensor. Defaults to `torch.long`. Returns: `torch.Tensor`: An (N-1)-dimensional tensor of integers indicating the position of the first True in each row. If no True value is found in a row, returns the length of the row. r9rr)rtyperHarangerr0values)rrFrow_len zero_or_indexr4r4r5first_true_indicesls "rquery_responsesr.context_lengthc Cs||k}|d|}t||j}t||d}||||dddd}||jd} t|dd|df|kd|} | | tj | d| j d| f d| fS) a5 Computes the reward logits and the rewards for a given model and query responses. Args: model (`torch.nn.Module`): The model used to compute the reward logits. query_responses (`torch.Tensor`): The tensor containing the query responses. pad_token_id (`int`): The token ID representing the pad token. context_length (`int`): The length of the context in the query responses. Returns: tuple: - `reward_logits` (`torch.Tensor`): The logits for the reward model. - `final_rewards` (`torch.Tensor`): The final rewards for each query response. - `sequence_lengths` (`torch.Tensor`): The lengths of the sequences in the query responses. rBrTF)r<rA position_ids return_dictoutput_hidden_states use_cacher9Nr) cumsumrJgetattrbase_model_prefixrH masked_fillscore hidden_statesrrrrsqueeze) rrr.rrAr lm_backboner<r reward_logitssequence_lengthsr4r4r5 get_rewards. $rcCs<||k}|d|}t||d}||||dddS)a Performs a forward pass through the model with the given query responses and pad token ID. Args: model (`torch.nn.Module`): The model to perform the forward pass. query_responses (`torch.Tensor`): The tensor containing the query responses. pad_token_id (`int`): The token ID representing the pad token. Returns: `ModelOutput`: The output of the model, including hidden states. rBrT)r<rArrr)rrJrHr)rrr.rArr<r4r4r5forwardsrFper_device_train_batch_sizerrc Csddl}tj}|j}|dddkr1||d<|dddd}|r(d d i|d <n;|r0d d i|d <n2t|d rct|jddrCt|jjnt|jdd}|durc|dddkrc| ||d|dd|j ||d^}}| |S)a Prepares the model for training with DeepSpeed (both for stage 2 and 3), configuring the appropriate settings based on the model and batch size. Args: model (`torch.nn.Module`): The model to be prepared for DeepSpeed training. per_device_train_batch_size (`int`): The training batch size per device. Returns: `torch.nn.Module`: The model initialized and configured with DeepSpeed for training. rNzero_optimizationstagertrain_micro_batch_size_per_gpuF)rprescale_gradientswall_clock_breakdownenabledTrrconfig hidden_sizes hidden_sizerg)z$zero_optimization.reduce_bucket_sizez4zero_optimization.stage3_param_persistence_thresholdz-zero_optimization.stage3_prefetch_bucket_size)rr) deepspeedr deepspeed_plugindeepspeed_configrCrrrrr initializeeval) rrrrrr config_kwargsrrr4r4r5prepare_deepspeeds:    rr responsescCsft||kd}dgt|d|jdg}tj|jd|jdj|}t |||k|}|S)aA Truncates the responses at the first occurrence of the stop token, filling the rest with pad tokens. Args: stop_token_id (`int`): The token ID representing the stop token where truncation occurs. pad_token_id (`int`): The token ID representing the pad token used to fill the truncated responses. responses (`torch.Tensor`): The tensor containing the responses to be truncated. Returns: `torch.Tensor`: The truncated responses tensor with pad tokens filled after the stop token. r9rBr) r unsqueezer\rrrHrrviewr)rr.r trunc_idxsnew_sizeidxspostprocessed_responsesr4r4r5truncate_response s "rrqueriesgeneration_configc Csl|jd}||k}t||d}|j|||ddd}t|jd}tj||jdd|dffdd|fS)a Generates sequences from the language model backbone in a way that does not affect padding tokens. Args: lm_backbone (`torch.nn.Module`): The language model backbone used for generation. queries (`torch.Tensor`): The tensor containing the input queries. pad_token_id (`int`): The token ID representing the pad token. generation_config (`GenerationConfig`): The configuration for the generation process. Returns: tuple: - `generated_sequences` (`torch.Tensor`): The concatenated tensor of input queries and generated sequences. - `logits` (`torch.Tensor`): The logits output from the generation process. rBrT)r<rArreturn_dict_in_generate output_scoresNr)rrHrgeneratestackscoresr sequences) rrr.rrrAr<rlogitsr4r4r5r"s  (rrcCsg}g}|jd}td||D]}||||} t|| ||\} } || || qt||dd} t|ddd} | d| jdd|} | jdg| jddRd|} | | fS)Nrrrr9r)rrangerr[r]r)rrrr.rrlogitssrrqueryquery_responserpadded_query_responsespadded_logitssr4r4r5batch_generationIs$   $r bos_token_idprompt_len_input_ids prompt_tokenschosen_prompt_len_input_ids chosen_tokensrejected_prompt_len_input_idsrejected_tokenscCs|dur^|dks||ddkr"|g|d|d<dg|d|d<|dks.||ddkr@|g|d|d<dg|d|d<|dksL||ddkr^|g|d|d<dg|d|d<|||fS)NrrrBrXr4)rrrrrrrr4r4r5add_bos_token_if_neededjs  r eos_token_idcCst|ddks||ddkr|d||ddt|ddks.||ddkr<|d||dd||fS)Nr<rr9rArB)r\r[)rrrr4r4r5add_eos_token_if_neededs  rr<cCst||kd}dgt|d|jdg}tj|jd|jdj|}t |||k|}t t |||kd}||fS)a Truncates the input tensor from the right side after the first occurrence of the stop token. Args: input_ids (`torch.Tensor`): The tensor containing the responses to be truncated stop_token_id (`int`): The token ID representing the stop token where truncation occurs pad_token_id (`int`): The token ID representing the pad token used to fill the truncated responses Returns: tuple: - `output_ids` (`torch.Tensor`): The truncated responses tensor with pad tokens filled after the stop token - `mask` (`torch.Tensor`): The mask tensor to indicate the padding tokens r9rBrr) rrr\rrrHrrrr ones_like)r<rr.rrr output_idsrQr4r4r5truncate_rights "rcCsJtr tjdStrtjdStrtjdStjdS)a3Empties the cache of the available torch device. This function checks for the availability of different torch devices (XPU, MLU, NPU, CUDA) and empties the cache of the first available device it finds. If none of the specific devices are available, it defaults to emptying the CUDA cache. N) rrHxpu empty_cachermlurnpurQr4r4r4r5rsrinputsr&cs j|dd}fdd|DS)ay Decodes the input tensor and strips the padding tokens. Args: inputs (`torch.Tensor`): The input tensor to be decoded. tokenizer (`transformers.PreTrainedTokenizerBase`): The tokenizer used to decode the input tensor. Returns: `list[str]`: The list of decoded strings with padding tokens stripped. F)skip_special_tokenscsg|] }|jdqSr#)replace pad_token)rLdr&r4r5rNrz,decode_and_strip_padding..) batch_decode)rr&decodedr4rr5decode_and_strip_paddingsr base_model model_name hub_model_id dataset_nametags wandb_url trainer_nametrainer_citation paper_titlepaper_id comet_urlc Cst||dd|dg|d} tj| fidttddd|d |d |d |d |d | d|d|d|d| dtddtddtddtddtd} | S)a Generate a `ModelCard` from a template. Args: base_model (`str` or `None`): Base model name. model_name (`str`): Model name. hub_model_id (`str`): Hub model ID as `username/model_id`. dataset_name (`str` or `None`): Dataset name. tags (`list[str]`): Tags. wandb_url (`str` or `None`): Weights & Biases run URL. comet_url (`str` or `None`): Comet experiment URL. trainer_name (`str`): Trainer name. trainer_citation (`str` or `None`, defaults to `None`): Trainer citation as a BibTeX entry. paper_title (`str` or `None`, defaults to `None`): Paper title. paper_id (`str` or `None`, defaults to `None`): ArXiv paper ID as `YYMM.NNNNN`. Returns: `ModelCard`: A ModelCard object. transformerslicensegenerated_from_trainer)rdatasets library_namelicencerr  template_pathtrlztemplates/lm_model_card.mdrrrrr rr r r r trl_versiontransformers_versionpytorch_versionrHdatasets_versionrtokenizers_version tokenizers)rr from_templaterm pkg_resourcesfilesjoinpathr) rrrrr r r r r rr card_datacardr4r4r5generate_model_cardsX,     r$cCs$tsdStdurtjSdS)zt If Comet integration is enabled, return the URL of the current Comet experiment; otherwise, return `None`. N)rcomet_mlget_running_experimenturlr4r4r4r5get_comet_experiment_urls   r(r:rcCs4tstdt}|dur|j||ddSdS)a If Comet integration is enabled logs a table to the Comet experiment if it is currently running. Args: name (`str`): Table name. table (`pd.DataFrame`): The Pandas DataFrame containing the table to log. zLThe comet-ml is not installed. Please install it first: pip install comet-mlN) tabular_datafilename)rModuleNotFoundErrorr%r& log_table)r:r experimentr4r4r5log_table_to_comet_experiment$s  r.rQ.c s|j\}}|}dd|D}|jdd}tj||jdd}||d||d}fdd|D}|jdd} | dk} | rQt | tj n||dddf} fd d|D} | sj| S| g| RS) a Shift non-zero elements in the mask and corresponding tensors to the left. This function operates on a binary mask and any number of additional tensors with the same dimensions as the mask. For each row, non-zero values are shifted to the leftmost positions. Then, columns that contain only zeros across all rows are truncated from the mask and tensors. Visually, this operation can be represented as follows: ``` [[0, 0, x, x, x, x], -> [[x, x, x, x], [0, x, x, x, 0, 0]] [x, x, x, 0]] ``` Args: mask (`torch.Tensor`): 2D tensor (binary mask) with shape `(N, M)`. *tensors (`torch.Tensor`) One or more 2D tensors with the same shape as `mask`. These tensors will be processed alongside `mask`, with non-zero values shifted and excess zero columns truncated in the same manner. Returns: `torch.Tensor`: Updated binary mask with non-zero values flushed to the left and trailing zero columns removed. `*torch.Tensor` Updated tensors, processed in the same way as the mask. Example: ```python >>> mask = torch.tensor([[0, 0, 1, 1, 1], [0, 1, 1, 0, 0]]) >>> tensor = torch.tensor([[9, 9, 2, 3, 4], [9, 5, 6, 9, 9]]) >>> new_mask, new_tensor = flush_left(mask, tensor) >>> print(new_mask) tensor([[1, 1, 1], [1, 1, 0]]) >>> print(new_tensor) tensor([[2, 3, 4], [5, 6, 0]]) ``` cSg|]}|qSr4clonerr4r4r5rNbrzflush_left..rBrrrcg|]}|dqSrBgatherridx_rollr4r5rNiNcs g|] }|dddfqSr*r4r)first_empty_colr4r5rNp ) rr1r rHrrrr5rrBrlrint8) rQrrM mask_copyfirst_non_zeropos mask_rollrolled_tensorscol_sums empty_cols flushed_maskflushed_tensorsr4)r9r7r5 flush_left6s (     rFcs|j\}}|}dd|D}t|}|jdd}tj||jdd}||d||d}fdd|D} |j dd} | dk} | rVt | tj n||dddf} fd d| D} | so| S| g| RS) zs Shift non-zero elements in the mask and corresponding tensors to the right. See `flush_left` for details. cSr/r4r0rr4r4r5rNrzflush_right..rBrrrcr2r3r4rr6r4r5rNr8Ncs g|] }|dddfqSr*r4r)first_non_empty_colr4r5rNr:)rr1rHfliplrr rrrr5rrBrlrr;)rQrrr<r= flipped_maskr>r?r@rArBnon_empty_colsrDrEr4)rGr7r5 flush_rightws"       rKc Cs|jtjtjfvr'tj|d|ddd}tdd|D}||}|Sg}t||D]\}}t j |dd}|jd|ddd}| |q.t|}|S)aw A memory-efficient implementation of the common `log_softmax -> gather` operation. This function is equivalent to the following naive implementation: ```python logps = torch.gather(logits.log_softmax(-1), dim=-1, index=index.unsqueeze(-1)).squeeze(-1) ``` Args: logits (`torch.Tensor`): Logits tensor of shape `(..., num_classes)`. index (`torch.Tensor`): Index tensor of shape `(...)`, specifying the positions to gather from the log-softmax output. Returns: `torch.Tensor`: Gathered log probabilities with the same shape as `index`. r9)rindexcSsg|] }tj|ddqS)r9r)rH logsumexp)rLlgr4r4r5rNrz)selective_log_softmax..r) rFrHrfloat64r5rrrrF log_softmaxr[) rrLselected_logitslogsumexp_valuesper_token_logps row_logits row_labels row_logpsrow_per_token_logpsr4r4r5selective_log_softmaxs   rYrB chunk_sizecCsTg}|j|ddD]}tj|dd}t||d }||q t|}|S)a Compute the Shannon entropy (in nats) for each row of *logits* without materialising the full soft-max in memory. The batch dimension is processed in chunks of size `chunk_size` so that only a subset of rows is expanded to probabilities at any one time. Args: logits (`torch.Tensor`): Logits tensor of shape `(..., num_classes)`. Entropy is taken along the last axis; all leading dimensions are preserved. chunk_size (`int`, *optional*, defaults to `1`): Number of rows to process per iteration. Returns: `torch.Tensor`: Entropy values with shape `logits.shape[:-1]`. rrr9)splitrPrQrHrqrr2r)rrZper_token_entropies logits_chunklogps chunk_entropyr4r4r5entropy_from_logitss  r`rW completionsr3r4step num_samplesc ststdt}tdddd}|jddd|jdd dD] }|j|d d d q#|jd dd d |durI|tkrCd}n|dkrIdS|dur}tt t|fddDfddDfdd DfddDt tD]*fddD} |j t t g| dR| qt|dd|dd} || dS)u Print out a sample of model completions to the console with multiple reward metrics. This function creates a nicely formatted table showing prompt-completion pairs, useful for monitoring model outputs during training. It requires the `rich` library to be installed. Args: prompts (`list[str]`): List of prompts. completions (`list[str]`): List of completions corresponding to the prompts. rewards (`dict[str, list[float]]`): Dictionary where keys are reward names and values are lists of rewards. advantages (`list[float]`): List of advantages corresponding to the prompts and completions. step (`int`): Current training step number, used in the output title. num_samples (`int` or `None`, *optional*, defaults to `None`): Number of random samples to display. If `None` (default), all items will be displayed. Example: ```python >>> from trl.trainer.utils import print_prompt_completions_sample >>> prompts = ["The sky is", "The sun is"] >>> completions = [" blue.", " in the sky."] >>> rewards = {"Correctness": [0.123, 0.456], "Format": [0.789, 0.101]} >>> advantages = [0.987, 0.654] >>> print_prompt_completions_sample(prompts, completions, rewards, advantages, 42) ╭──────────────────────────── Step 42 ─────────────────────────────╮ │ ┏━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━┳━━━━━━━━━━━┓ │ │ ┃ Prompt ┃ Completion ┃ Correctness ┃ Format ┃ Advantage ┃ │ │ ┡━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━╇━━━━━━━━━━━┩ │ │ │ The sky is │ blue. │ 0.12 │ 0.79 │ 0.99 │ │ │ ├────────────┼──────────────┼─────────────┼────────┼───────────┤ │ │ │ The sun is │ in the sky. │ 0.46 │ 0.10 │ 0.65 │ │ │ └────────────┴──────────────┴─────────────┴────────┴───────────┘ │ ╰──────────────────────────────────────────────────────────────────╯ ``` zvThe function `print_prompt_completions_sample` requires the `rich` library. Please install it with `pip install rich`.Tz bold white) show_header header_styleexpandPrompt bright_yellow)style Completion bright_greenz bold cyanr)rijustify Advantagez bold magentaNrcg|]}|qSr4r4rLr)rWr4r5rNrz3print_prompt_completions_sample..crnr4r4ro)rar4r5rNrcs$i|]\}|fddDqS)crnr4r4rovalr4r5rNrz>print_prompt_completions_sample...r4rLkey)indicesrpr5rs$z3print_prompt_completions_sample..crnr4r4ro)r4r4r5rN rcsg|] }|dqS).2fr4rr)rr3r4r5rN#rOruFzStep )rftitle border_style)rrwrr!ryrr\randomsamplerr6r{r" add_sectionr r}) rWrar3r4rbrcr~r reward_name reward_valuespanelr4)r4rarrtrWr3r5print_prompt_completions_samples80  0 r~)rrN)Nr)r9r)rf)FF)r8N)NNNNr3r*)yrimportlib.resources resourcesrrrxr  collectionsrrrimportlib.metadatartypingrrrr numpyrpandaspdrHtorch.nn.functionalr functionalrPtorch.utils.data accelerater r accelerate.stater huggingface_hubr rtorch.nn.utils.rnnrrrrrrrrrtransformers.utilsrrrrrrtrainer.model_configr rich.consoler rich.panelr rich.tabler! rich.textr"r%peftr#r$r%rqrnrprlrmr]rrrrrror{rrModuler"r(r)rErPrTrerprt DataFramerSIMPLE_SFT_CHAT_TEMPLATESIMPLE_CHAT_TEMPLATErrrJrrrrrrrrrrrrrr$r(r.rFrKrYr`r~r4r4r4r5s     $     bO BS: ,% )   8 2 ! 6 '          J ..A#"