o
    	Ti#                  	   @   s  d dl mZmZ d dlmZ d dlZd dlmZ d dlm	Z	 d dl
m
Z
 d dlmZmZmZmZmZmZ d dlmZmZmZmZ d d	lmZ e
  eG d
d dZeeZe d  Zeejejej dej!ej"ej#dZ$	d-ddZ%dZ&dZ'e%e$e&e'dZ(dd Z)ee$j* ej+e$jej,dZ-e+e-Z-ee-ddZ.e	e/dd e-0 e$jdZ1e+e$jZ2e2j3e2_4ee$e-e.e2e(e)e1dZ5dZ6e+e6Z7ej+e6ej8d9e5j:j;Z<d d!d"d#e2j=d$Z>dZ?dZ@ee?e@ZAejBZBe
eCe5jDD ]\ZEZFeFd% ZGg ZHeGD ]ZIeA ZJeJe>d&< e5jKeIfi e>ZLeHMeLN eJ d  qd'd( eHD eFd)< eFd) ZOe7eOd#d#d*d+9e5j:j;ZPe<d.i ePjQR ZQeQddd f S ZTd,d( eTD ZUe5VeGeHeUZWe5XeWeFeU eEd d krne5j:jYrne5ZeB qdS )/    )	dataclassfield)OptionalN)load_dataset)Adam)tqdm)AutoModelForCausalLMAutoTokenizerHfArgumentParser RobertaForSequenceClassificationRobertaTokenizerset_seed)!AutoModelForCausalLMWithValueHead	PPOConfig
PPOTrainercreate_reference_model)LengthSamplerc                   @   s   e Zd ZU dZedddidZee ed< edddidZ	ee ed	< ed
ddidZ
ee ed< edddidZee ed< edddidZee ed< edddidZee ed< edddidZee ed< dS )ScriptArgumentszG
    The name of the Casual LM model we wish to fine-tune with PPO
    zybelkada/gpt-j-6b-sharded-bf16helpzthe model name)defaultmetadata
model_nameNzuse 'wandb' to log with wandblog_withgM >zthe learning ratelearning_rate   zthe PPO minibatch sizemini_batch_size   zthe batch size
batch_size   z)the number of gradient accumulation stepsgradient_accumulation_stepsz3./gpt-j-6B-detoxified-long-context-26-shl-1e4-finalzthe path to save the modelmodel_save_path)__name__
__module____qualname____doc__r   r   r   str__annotations__r   r   floatr   intr   r   r     r)   r)   q/home/ubuntu/.local/lib/python3.10/site-packages/examples/research_projects/toxicity/scripts/gpt-j-6b-toxicity.pyr   <   s   
 r   d   )r   r   r   
ppo_epochsr   r   r   allenai/real-toxicity-prompts   
   c                    s~   t | jj_t|dd}dd }|j|dd}t||  fdd}|j|dd}|j	d	d
 |j
dddd }|S )ai  
    Build dataset for training. This builds the dataset from `load_dataset`, one should
    customize this function to train the model on its own dataset.

    Args:
        dataset_name (`str`):
            The name of the dataset to be loaded.

    Returns:
        dataloader (`torch.utils.data.DataLoader`):
            The dataloader for the dataset.
    train)splitc                 S   s   | d d }|d uo|dkS )Nprompttoxicityg333333?r)   )sampler3   r)   r)   r*   	filter_fnw   s   z build_dataset.<locals>.filter_fnF)batchedc                    sJ   | d d }| d d } || d    | d< | d | d< | S )Nr2   textcontinuation	input_idsquery)encodedecode)r4   r2   r8   
input_size	tokenizerr)   r*   tokenize   s
   zbuild_dataset.<locals>.tokenizetorch)typeg?)	test_sizeshuffle)r	   from_pretrainedr   	eos_token	pad_tokenr   filterr   map
set_formattrain_test_split)configdataset_nameinput_min_text_lengthinput_max_text_lengthdsr5   r@   r)   r=   r*   build_datasetc   s   
rQ      (   )rN   rO   c                    s    fdd d D S )Nc                    s    i | ]   fd dD qS )c                    s   g | ]}|  qS r)   r)   ).0dkeyr)   r*   
<listcomp>   s    z'collator.<locals>.<dictcomp>.<listcomp>r)   )rT   datarV   r*   
<dictcomp>   s     zcollator.<locals>.<dictcomp>r   r)   rY   r)   rY   r*   collator   s   r\   )torch_dtype   )num_shared_layersc                 C   s   | j S )N)requires_grad)pr)   r)   r*   <lambda>   s    rb   )lr)	ref_modelr?   datasetdata_collator	optimizerz0facebook/roberta-hate-speech-dynabench-r4-targetg        g      ?T)
min_lengthtop_ktop_p	do_samplepad_token_idr9   max_new_tokensc                 C   s   g | ]	}t | qS r)   )r?   r<   squeeze)rT   rr)   r)   r*   rX      s    rX   responsept)padding
truncationreturn_tensorsc                 C   s   g | ]}t |qS r)   )rA   tensor)rT   outputr)   r)   r*   rX      s    )r-   r.   r/   r)   )[dataclassesr   r   typingr   rA   datasetsr   torch.optimr   r   transformersr   r	   r
   r   r   r   trlr   r   r   r   trl.corer   pandasr   parserparse_args_into_dataclassesscript_argsr   r   r   r   r   r   rL   rQ   min_input_lengthmax_input_lengthre   r\   seedrE   bfloat16modelrd   rH   
parametersrg   r?   rF   rG   ppo_trainertoxicity_model_idtoxicity_tokenizerfloat16toacceleratordevicetoxicity_modeleos_token_idgeneration_kwargsoutput_min_lengthoutput_max_lengthoutput_length_samplerr    	enumerate
dataloaderepochbatchquery_tensorsresponse_tensorsr:   gen_lengeneraterq   appendro   textstoxicity_inputslogitsr'   tolisttoxicity_labelsrewardsstepstats	log_statsis_main_processsave_pretrainedr)   r)   r)   r*   <module>   s    	
-


	


