o پi3@sddlZddlmZmZmZmZmZmZmZddl Z ddl m Z ddl m Z mZer6ddlZ ddlmZmZGdddedd ZGd d d ZGd d d Z     ddedeedeedededeedefddZdS)N) TYPE_CHECKINGIteratorListOptionalTuple TypedDictUnion)Unpack)GenerationParametersSamplingParameters)ExLlamaV2DynamicGeneratorExLlamaV2Samplerc@sNeZdZUeed<eeeeefed<eeed<ded<eeed<dS)ExllamaV2Params max_tokensstop_conditionsseedzExLlamaV2Sampler.Settings gen_settingsmax_new_tokensN) __name__ __module__ __qualname__int__annotations__rrrstrrrM/home/ubuntu/.local/lib/python3.10/site-packages/outlines/models/exllamav2.pyrs  rF)totalc@s2eZdZddZddZdddeefdd Zd S) OutlinesExLlamaV2TokenizercCs.||_|j|_t|jj|_|jj|_dSN)exl2_tokenizerget_piece_to_id_dict vocabularysetextended_piece_to_idspecial_tokens eos_token_id)self tokenizerrrr__init__s z#OutlinesExLlamaV2Tokenizer.__init__cCs|Srr)r&tokenrrrconvert_token_to_stringsz2OutlinesExLlamaV2Tokenizer.convert_token_to_string token_idsztorch.LongTensorreturncCs*|jjt|dd}t|tr|gS|S)NF)decode_special_tokens)rdecodetorchtensor isinstancer)r&r+decodedrrrr. s z!OutlinesExLlamaV2Tokenizer.decodeN)rrrr(r*rrr.rrrrrsrc@seZdZdZdddddefddZd eeeefd e d e d e e d e e eeeefff ddZdeeeefd e fddZd eeeefd e d e d e e d eeeeff ddZd eeeefd e d e d e e d eeeeeff ddZdS)ExLlamaV2ModelzRepresents a `exl2` model. generatorr r'r max_seq_lencCs||_||_||_dSr)r4r'r5)r&r4r'r5rrrr(-s zExLlamaV2Model.__init__promptsgeneration_parameterssampling_parametersexllamav2_paramsr,c s|ddlm}t|tr|g}t|\}}durTsz@ExLlamaV2Model.prepare_generation_parameters..rrr)exllamav2.generatorr r1r dataclassesastupler4r'encodeshapeappendr5rangelenr%stop_atlistSettings temperaturetop_ptop_klogits_processor num_samples)r&r6r7r8structure_logits_processorr9r rIrpromptids prompt_tokensrrrr>rprepare_generation_parameters7sP                 z,ExLlamaV2Model.prepare_generation_parametersoutputcCst|tr|St|dkr|dS|jdkrVt||jkr|St||jdks*Jt||j}g}t|jD]}g}t|D] }|||||q@||q8|S|S)a The purpose of this function is to reformat the output from exllamav2's output format to outline's output format For exllamav2, it mainly accepts only a list or a string(they also do cfg sampling with tuples but we will ignore this for now) The exllamav2's logic is 1. If the prompt is a string, return a string. This is the same as outlines 2. If a prompt is a list, return a list. This is not the same as outlines output in that if the list is only one element, the string is expected to be outputted. 3. There is no such thing as num_samples, so the prompts had to be duplicated by num_samples times. Then, we had the function output a list of lists r@r)r1rrHrPrGrF)r&rVr8num_items_per_sample new_outputi curr_samplejrrrreformat_outputus"     zExLlamaV2Model.reformat_outputc KsR|||||\}} |jj||dt|ddd|dd|dd}|||S)NrrTrFr)rRrrcompletion_onlyr:radd_bosr)rUr4generateminr\)r&r6r7rQr8r9rVrrrr_s$  zExLlamaV2Model.generatec sddlm}|||\}}it|tr|g}t|}|d}t|D]4\} } jjj | ddd} || |d| d||d|d ddd } |durP|d 7}j | } | | <q&d g|d t tffdd }|S)Nr)ExLlamaV2DynamicJobrTF)r:r^rrr) input_idsrmin_new_tokensrrr token_healingr-r@r,c3sxjr:j}|D] }|d}|ddkr%|dd}||<|dr-d|<q VjsdS)Nserialstage streamingtextreeos)r4num_remaining_jobsiterategetr\)resultsridxri next_textorderr8r&rrtoken_generators       z.ExLlamaV2Model.stream..token_generator) rArarUr1rrH enumerater4r'rDenqueuer)r&r6r7rQr8r9ra batch_sizerrpprbjobrfrtrrqrstreamsB        zExLlamaV2Model.streamN)rrr__doc__rr(rrrr r r rrrUr\r_rrzrrrrr3*s^   >  "r3T model_pathdraft_model_pathr5cache_q4pagedmax_chunk_sizer,c CsFzddlm}m}m}m} m} ddlm} Wn ty!tdw| |} |dur2|| _ |d| _ | || } |dur@d}|rJ|| |dd }n|| |dd }| j |dd t d | | }|rdd nd }d}d}|dur| |}||}|r|||dd }n|||dd }| | |||||d||d }|j}t|}t|||}|S)a2 Load an ExLlamaV2 model. Parameters ---------- model_path (str) Path to the model directory. device (str) Device to load the model on. Pass in 'cuda' for GPU or 'cpu' for CPU max_seq_len (Optional[int], optional) Maximum sequence length. Defaults to None. scale_pos_emb (Optional[float], optional) Scale factor for positional embeddings. Defaults to None. scale_alpha_value (Optional[float], optional) Scale alpha value. Defaults to None. no_flash_attn (Optional[bool], optional) Disable flash attention. Defaults to None. num_experts_per_token (Optional[int], optional) Number of experts per token. Defaults to None. cache_q4 (bool, optional) Use Q4 cache. Defaults to False. tokenizer_kwargs (dict, optional) Additional keyword arguments for the tokenizer. Defaults to {}. gpu_split (str) "auto", or VRAM allocation per GPU in GB. Auto will use exllama's autosplit feature low_mem (bool, optional) Enable VRAM optimizations, potentially trading off speed verbose (bool, optional) Enable if you want debugging statements Returns ------- An `ExLlamaV2Model` instance. Raises ------ `ImportError` if the `exllamav2` library is not installed. r) ExLlamaV2ExLlamaV2CacheExLlamaV2Cache_Q4ExLlamaV2ConfigExLlamaV2Tokenizer)r a2The `exllamav2`, `transformers` and `torch` libraries needs to be installed in order to use `exllamav2` models. Please run `pip install transformers torch git+https://github.com/lapp0/exllamav2@sampler-logits-processor` Documentation: https://dottxt-ai.github.io/outlines/latest/reference/models/exllamav2/Nr;T)r5lazy)progresszLoading tokenizer...r@F) modelcache draft_model draft_cacher'max_batch_sizeuse_ngram_draftrr) exllamav2rrrrrrAr ImportError max_input_lenmax_attention_sizearch_compat_overridesload_autosplitprintr5rr3)r|r}r5r~rrrrrrrr configrrr'rrr draft_configr4outlines_tokenizeroutlines_exl2_modelrrrexl2sd/     r)NNFTN)rBtypingrrrrrrrr/typing_extensionsr outlines.generate.apir r torch.LongTensorrAr r rrr3rrboolrrrrrs@$ M