o ॵij@sZddlZddlmZmZmZmZmZddlZddlm Z ddl m Z ddl m Z ddlmZddlmZmZmZddlmZmZdd lmZdd lmZdd lmZdd lmZmZdd l m!Z!m"Z"ddl#m$Z$ddl%m&Z&ddl'm(Z(e$Z)gdZ*ej+ej,e j,dGdddee&Z-ej+ej.e j/dej+ej.e j0dej+ej.e j1dej+ej.e j.dGddde-Z2ej+ej3ddGdddeZ4ej+ej3ddGdddeZ5ej+ej3ddGdd d eZ6ej+ej,d!dGd"d#d#eZ7ej+ej,d$dGd%d&d&eZ8ej+ej,e j9dGd'd(d(e-Z:ej+ej3e j;dGd)d*d*eZ>> from modelscope.pipelines import pipeline >>> pipeline_ins = pipeline(task='text-generation', >>> model='damo/nlp_palm2.0_text-generation_chinese-base') >>> sentence1 = '本文总结了十个可穿戴产品的设计原则，而这些原则，同样也是笔者认为是这个行业最吸引人的地方：' >>> '1.为人们解决重复性问题；2.从人开始，而不是从机器开始；3.要引起注意，但不要刻意；4.提升用户能力，而不是取代' >>> print(pipeline_ins(sentence1)) >>> # Or use the dict input: >>> print(pipeline_ins({'sentence': sentence1})) To view other examples plese check tests/pipelines/test_text_generation.py. compileFcompile_options)r%r&r'r( auto_collater)r*z,please check whether model config exists in Nfirst_sequence postprocessor model_dirdecode)super__init__pop isinstancer%r r CONFIGURATIONrfrom_pretrainedr.r&evalr-hasattrrget has_logged) selfr%r&r'r(r+r,kwargscfg __class__r0e/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/pipelines/nlp/text_generation_pipeline.pyr2's<        zTextGenerationPipeline.__init__cK i|ifSNr0r;pipeline_parametersr0r0r@_sanitize_parameters^ z+TextGenerationPipeline._sanitize_parametersinputsreturnc KstGz|jj|fi|WWdStyJ}z'|js.td|dd|_|jjdi||WYd}~WdSd}~ww1sNwYdS)Nz.When inputs are passed directly, the error is z,, which can be ignored if it runs correctly.Tr0)torchno_gradr%generateAttributeErrorr:loggerwarning)r;rGforward_paramser0r0r@forwardas"  zTextGenerationPipeline.forwardcCs|jj|ddS)NTskip_special_tokensr&r/tolistr;rGr0r0r@r/oszTextGenerationPipeline.decodecCs|j|SrBrTrVr0r0r@sentence_piecessz%TextGenerationPipeline.sentence_piececCs,|j|}|ddddddS)Nzz. zz)r&r/rUreplace)r;rGdecodedr0r0r@robertavs zTextGenerationPipeline.robertacKsXt|ttfr |d}t|tst|jdkr|d}t||j|}t|}t j |iS)zprocess the prediction results Args: inputs (Dict[str, Any]): _description_ Returns: Dict[str, str]: the prediction results sequencesr) r4dictr listlenshapegetattrr-rr TEXT)r;rGpostprocess_paramsrZtextr0r0r@ postprocess{s  z"TextGenerationPipeline.postprocess)NNr#Tr$)__name__ __module__ __qualname__rr strrrr2rErrrrQr/rWr[r rf __classcell__r0r0r>r@r#s6 7   rcs|eZdZ  ddeeefdeeffdd ZddZ de ee fffd d Z d e ee fde ee ffd d Z ZS)rNr%r&c shtj||fi|||_|t|jddd|_|t|jddd|_|t|jddd|_dS)Nr.task_specific_params min_length max_length) r1r2sub_task_parse_specific_model_paramsrbr%rlrmrn)r;r%r&ror<r>r0r@r2s z!TextGenerationT5Pipeline.__init__cCsL|durdSt|}|d|}|dur$ttj|d}||}|S)Nzmodel.z config.json)rsafe_getospathjoin)r;r.keyr=paramsr0r0r@rps z5TextGenerationT5Pipeline._parse_specific_model_paramsrHc st|tstdt||jdur9|jp|jjj}||jvr9|jj |j|d|j|vr9|j|j |}t j |fi|S)NzNot supported input type: prefix) r4rj ValueErrortyperlror%pipelineconfigupdaterwr1 preprocess)r;rGpreprocess_paramsror>r0r@r}s   z#TextGenerationT5Pipeline.preprocessrGcKs~|d|j}|d|j}|dur||d<|dur||d<t|jjdi||WdS1s8wYdS)Nrmrnr0)r9rmrnrIrJr%rK)r;rGrOrmrnr0r0r@rQs $z TextGenerationT5Pipeline.forward)NN)rgrhrirr rjrrr2rprrr}rQrkr0r0r>r@rs    rzchatglm6b-text-generation) group_keyr"c~eZdZ  ddeeefffdd ZddZdeee ffd d Z d edeee ffd d Z deee ffddZ Z S)rNFr%c sddlm}t|tr'tj|st|n|}|| }t j r'| }|dur0| |}|r6|}||_|jtjdd|i|dS)Nr)ChatGLMForConditionalGenerationr%r0)-modelscope.models.nlp.chatglm.text_generationrr4rjrrrsexistsrr6halfrIcuda is_availablequantizebfloat16r%r7r1r2)r;r%quantization_bituse_bf16r<rr.r>r0r@r2s,      z(ChatGLM6bTextGenerationPipeline.__init__cKrArBr0rCr0r0r@rErFz4ChatGLM6bTextGenerationPipeline._sanitize_parametersrHcK|SrBr0r;rGr~r0r0r@r}z*ChatGLM6bTextGenerationPipeline.preprocessrGcKs|||j|SrB)r|r%chatr;rGrOr0r0r@rQs  z'ChatGLM6bTextGenerationPipeline.forwardcKrrBr0r;inputr<r0r0r@rfrz+ChatGLM6bTextGenerationPipeline.postprocessNFrgrhrirr rjr2rErrr}rQrfrkr0r0r>r@rs rzchatglm2_6b-text-generationcr)rNFr%c  s$ddlm}|dd}t|trR|dd}tj|s#t||dn|}d} | ds1| dr5ddi} |d | } d} |rBt j } |d | } t j |d | | d }n| ds\| drdt|rd||rj| |durs||}||_|j|j |jjd d |_tjdd|i|dS)Nr AutoTokenizerr(r#revision)rrrX device_map torch_dtypeT)trust_remote_coderrrr%r0) modelscoperr9r4rjrrrsrr startswithrIrr r6rrrr%r7r. tokenizerr1r2) r;r%rrr<rr(rr.default_device_maprdefault_torch_dtyperr>r0r@r2sN         z*ChatGLM6bV2TextGenerationPipeline.__init__cKrArBr0rCr0r0r@rE%rFz6ChatGLM6bV2TextGenerationPipeline._sanitize_parametersrHcKrrBr0rr0r0r@r}(rz,ChatGLM6bV2TextGenerationPipeline.preprocessrGcKs|||j||jSrB)r|r%rrrr0r0r@rQ,s z)ChatGLM6bV2TextGenerationPipeline.forwardcKrrBr0rr0r0r@rf1rz-ChatGLM6bV2TextGenerationPipeline.postprocessrrr0r0r>r@rs )rz qwen-chatcseZdZdeeefffdd ZddZdeee ffddZ d eeefdeee ffd d Z deee ffd d Z Z S)rr%c  sddlm}m}|dtj}|dd}|dd}|dd }|rDttjdd d d tj } fd dt | Dnd|dksO|tjkrRd} nd} t |t rx|j ||dd|_|j |||d| d|_tj |dd|j_tjdd|ji|d|_dS)NrAutoModelForCausalLMrrrautouse_max_memoryFmodel_revisionzv.1.0.5@GBci|]}|qSr0r0.0i max_memoryr0r@ Bz-QWenChatPipeline.__init__..bf16Trr)rrrfp16rr%r0)rrrr9rIrintr mem_get_info device_countranger4rjr6rr7r%rgeneration_configr1r2_model_prepare r;r%r<rrrrrrn_gpusrr>rr@r28s>        zQWenChatPipeline.__init__cKrArBr0rCr0r0r@rEZrFz%QWenChatPipeline._sanitize_parametersrHcKrrBr0rr0r0r@r}]rzQWenChatPipeline.preprocessrGcKstt|tr|dd}|dd}n|}|dd}|dd}|dd}|j|j||||}|d|dd S) NrehistorysystemzYou are a helpful assistant.append_historyTrr])responser)r4rr9r%rr)r;rGrOrerrrresr0r0r@rQas     zQWenChatPipeline.forwardcKrrBr0rr0r0r@rfprzQWenChatPipeline.postprocessrr0r0r>r@r5s"  rzqwen-text-generationcxeZdZdeeefffdd ZddZdeee ffddZ d edeee ffd d Z deee ffd d Z Z S)rr%c  s$ddlm}m}|dtj}|dd}|dd}|dd }|rDttjdd d d tj } fd dt | Dnd|dksO|tjkrRd} nd} t |t rw|j |||d| d|_|j ||dd|_t ||j_n ||_|dd|_tjdd|ji|d|_dS)NrrrrrrFrzv.1.0.4rrrcrr0r0rrr0r@rrz7QWenTextGenerationPipeline.__init__..rT)rrrrrrr%r0)rrrr9rIrrrrrrr4rjr6r7r%rrrr1r2rrr>rr@r2xsB        z#QWenTextGenerationPipeline.__init__cKrArBr0rCr0r0r@rErFz/QWenTextGenerationPipeline._sanitize_parametersrHcKrrBr0rr0r0r@r}rz%QWenTextGenerationPipeline.preprocessrGcKs@|j|ddd}tj|jj|jjdi|dddiS)Npt)return_tensorszcuda:0rTrRr0)rtor rcr/r%rKcpurr0r0r@rQsz"QWenTextGenerationPipeline.forwardcKrrBr0rr0r0r@rfrz&QWenTextGenerationPipeline.postprocessrr0r0r>r@rts % rseqgptcr)r r%c snddlm}t|trtj|st|n|}t |}||_ |j | ||_ t jdd|i|dS)Nrrr%r0)modelscope.utils.hf_utilrr4rjrrrsrrr r6r%r7rr1r2)r;r%r<rr.r>r0r@r2s      zSeqGPTPipeline.__init__cKrArBr0rCr0r0r@rErFz#SeqGPTPipeline._sanitize_parametersrHcKrrBr0rr0r0r@r}rzSeqGPTPipeline.preprocesspromptcKsv|j||ddddddd}|j|jj}|jj|ddd d }|jj|dd }|d }|t|d}t j |iS) N gen_tokenrXrTi)rpadding truncationrnF) num_beams do_samplemax_new_tokensrRr) rr9 input_idsrr%r(rK batch_decoder`r rc)r;rrOroutputsdecoded_sentencesdecoded_sentencer0r0r@rQs" zSeqGPTPipeline.forwardcKrrBr0rr0r0r@rfrzSeqGPTPipeline.postprocessrr0r0r>r@r s  r cseZdZ    d$deeefdededeffdd Zd eee ffd d Z d dZ        d%dede de dededede de de d eee ffd d!Zd eee ffd"d#ZZS)&r!Nr#Tr%r&r'r(c sHtj|dtjd|_ddlm}|||_tj dd|ji|dS)uUse `model` and `preprocessor` to create a generation pipeline for prediction. Args: model (str or Model): Supply either a local model dir which supported the text generation task, or a model id from the model hub, or a torch model instance. preprocessor (Preprocessor): An optional preprocessor instance, please make sure the preprocessor fits for the model if supplied. kwargs (dict, `optional`): Extra kwargs passed into the preprocessor's constructor. Examples: >>> from modelscope.utils.constant import Tasks >>> import torch >>> from modelscope.pipelines import pipeline >>> from modelscope import snapshot_download, Model >>> model_dir = snapshot_download("modelscope/Llama-2-13b-chat-ms", >>> ignore_file_pattern = [r'\w+\.safetensors']) >>> pipe = pipeline(task=Tasks.text_generation, model=model_dir, device_map='auto', >>> torch_dtype=torch.float16) >>> inputs="咖啡的作用是什么？" >>> result = pipe(inputs,max_length=200, do_sample=True, top_p=0.85, >>> temperature=1.0, repetition_penalty=1., eos_token_id=2, bos_token_id=1, pad_token_id=0) >>> print(result['text']) To view other examples plese check tests/pipelines/test_llama2_text_generation_pipeline.py. rrrrLlama2Tokenizerr%Nr0) r r6rIfloat16r%modelscope.models.nlp.llama2rrr1r2)r;r%r&r'r(r+r<rr>r0r@r2s    zLlama2TaskPipeline.__init__rHcKrrBr0rr0r0r@r} rzLlama2TaskPipeline.preprocesscKrArBr0rCr0r0r@rErFz'Llama2TaskPipeline._sanitize_parametersF?333333??rr]rrGrnrtop_p temperaturerepetition_penalty eos_token_id bos_token_id pad_token_idc  Ksdi} |j|ddd}|jj|jdf|||||||| d| } |jj| dddd} | | d <| S) NFr)add_special_tokensrr)rnrrrrrrrT)rSclean_up_tokenization_spacesrre)rr%rKrrr)r;rGrnrrrrrrrrOoutput generate_idsoutr0r0r@rQs6    zLlama2TaskPipeline.forwardcKrrBr0rr0r0r@rf3rzLlama2TaskPipeline.postprocessNNr#T)rFrrrrr]r)rgrhrirr rjrr2rrr}rErboolfloatrQrfrkr0r0r>r@r!s\ &  !r!cseZdZdZ    d)deeefdededed ed e ee ff fd d Z d e ee ffd dZ ddZ dddddddddgf dedededededed ed!ed"ed#ed$ed e ee ffd%d&Zd e ee ffd'd(ZZS)*Llama2chatTaskPipelineaUse `model` and `preprocessor` to create a generation pipeline for prediction. Args: model (str or Model): Supply either a local model dir which supported the text generation task, or a model id from the model hub, or a torch model instance. preprocessor (Preprocessor): An optional preprocessor instance, please make sure the preprocessor fits for the model if supplied. kwargs (dict, `optional`): Extra kwargs passed into the preprocessor's constructor. Examples: >>> from modelscope.utils.constant import Tasks >>> import torch >>> from modelscope.pipelines import pipeline >>> from modelscope import Model >>> pipe = pipeline(task=Tasks.chat, model="modelscope/Llama-2-7b-chat-ms", device_map='auto', >>> torch_dtype=torch.float16, ignore_file_pattern = [r'.+\.bin$'], model_revision='v1.0.5') >>> inputs = 'Where is the capital of Zhejiang?' >>> result = pipe(inputs,max_length=512, do_sample=False, top_p=0.9, >>> temperature=0.6, repetition_penalty=1., eos_token_id=2, bos_token_id=1, pad_token_id=0) >>> print(result['response']) >>> inputs = 'What are the interesting places there?' >>> result = pipe(inputs,max_length=512, do_sample=False, top_p=0.9, >>> temperature=0.6, repetition_penalty=1., eos_token_id=2, bos_token_id=1, >>> pad_token_id=0, history=result['history']) >>> print(result['response']) >>> inputs = 'What are the company there?' >>> history_demo = [('Where is the capital of Zhejiang?', >>> 'Thank you for asking! The capital of Zhejiang Province is Hangzhou.')] >>> result = pipe(inputs,max_length=512, do_sample=False, top_p=0.9, >>> temperature=0.6, repetition_penalty=1., eos_token_id=2, bos_token_id=1, >>> pad_token_id=0, history=history_demo) >>> print(result['response']) To view other examples plese check tests/pipelines/test_llama2_text_generation_pipeline.py. Nr#Tr%r&r'r(r+rHc  s^|dd}|dd}tj|||d|_ddlm} | ||_tjdd|ji|dS)Nrrrrrr%r0) r9r r6r%rrrr1r2) r;r%r&r'r(r+r<rrrr>r0r@r2^s    zLlama2chatTaskPipeline.__init__cKrrBr0rr0r0r@r}mrz!Llama2chatTaskPipeline.preprocesscKrArBr0rCr0r0r@rEprFz+Llama2chatTaskPipeline._sanitize_parametersrFrrrrr]rzyou are a helpful assistant!rGrnrrrrrrrrrc Ksp| } || d<|| d<|| d<|| d<|| d<|| d<|| d<|| d<| | d <| | d <| | d <|j| |j}|S) Nrernrrrrrrrrr)r%rr)r;rGrnrrrrrrrrrrO inputs_dictrr0r0r@rQss zLlama2chatTaskPipeline.forwardcKrrBr0rr0r0r@rfrz"Llama2chatTaskPipeline.postprocessr)rgrhri__doc__rr rjrrrrr2r}rErrrrQrfrkr0r0r>r@r7sr&      r)=rrtypingrrrrrrI transformersrrrmodelscope.metainfor modelscope.models.baser modelscope.outputsr r r modelscope.pipelines.baserrmodelscope.pipelines.builderrmodelscope.preprocessorsrmodelscope.utils.chinese_utilsrmodelscope.utils.constantrrmodelscope.utils.hubrrmodelscope.utils.loggerr!modelscope.utils.streaming_outputrmodelscope.utils.torch_utilsrrM__all__register_moduletext_generationrtext2text_generationtranslation_en_to_detranslation_en_to_rotranslation_en_to_frrrrrrrr llama2_text_generation_pipeliner!$llama2_text_generation_chat_pipelinerr0r0r0r@s~          j6);><-S