o 㥵i3P3@sddlZddlZddlZddlZddlZddlmZddlmZddl m Z ddl m Z m Z mZmZddlZddlZddlZddlZddlmZddlmZddlmZdd lmZmZmZdd lmZd ej d <d ej!j"_#d ej!j"j$_%e&ej!j"drd ej!j"_'ddl(m)Z)m*Z*ddl+m,Z,m-Z-m.Z.ddZ/ dhdej0dej0dej0de ej0dej0f ddZ1 dhdej0dej0dej0de ej0deej0ej0ff ddZ2 dhde-dej0dej0dej0dej0dej0dej0d ej0dej0dej0fd!d"Z3e3fde.d#ej0dej0d$e4dej0dej0dej0dej0d ej0fd%d&Z5e6e7e3d'd(de,d)ej0d*e4dej0d ej0d+e4f d,d-Z8did/d0Z9eGd1d2d2Z:d'dd3d4d3d.d d5ddd6 d7e;ejde>d:e?d;e?de ej0e@ej0Bfd?d@ZAeGdAdBdBZBeGdCdDdDZC .did:e?fdEdFZDeEejFdGe;dHdIejFdJe;dd dKejFdLej e d dMdd dKejFdNe4d'dIejFdOe4ddIejFdPe>d3dIejFdQe>d4dIejFdRe>d3dIejFdSej e d dMdTdIejFdUe;dVdIejFdWd.dXejFdYe4dZdIejFd[d.dXejFd\d dXejFd]e4d^dIejFd_e d`dId9e;d=e e@e;d>e e@e d+e4d*e4de4de>de>dae d7e;d:e?dbe4dce?d;e?d)min)longrgatherwherescatter_sortcumsumnn functionalsoftmaxscatter masked_fillfloatclip) logitsr(r)r*r+score sorted_logitssorted_indices cum_probssorted_indices_to_removeindices_to_removeprobsr%r%r&logits_to_probs3s$rHcCs&t|d||||d}t|}||fS)N)rr)r@r(r)r*r+)rHr')r@r(r)r*r+rGidx_nextr%r%r&sampleRsrJmodelx input_pos audio_masks audio_partsc  Csn|j||||d}|j} |j} t| ||||dur |dddfndddg} |jD]} | jjjd| jjj dq*t j dg| j t j d}|| || d|jj} d| | dk<|| } | | td|jjD]?}t j |g| j t j d}|| |} | ddddddf}t|||||dur||dnddd} || } | | qmt j| dd} | jS)N)rNrOr)r(r)r*r+devicerrir0)forward_generater@ hidden_statesrJ fast_layers attentionkv_cachek_cachefill_v_cachertensorrQr3forward_generate_fast tokenizersemantic_begin_idfast_embeddingsappendrangeconfig num_codebooksstackT)rKrLrMr(r)r*rNrOr+r@rS codebookslayera codebook_idx short_logitsr%r%r&decode_one_token_ards`           rj cur_tokennum_new_tokensc  Cs.tj|jjd|jjftj|jd} tt|D]r} d} | | kr+| ddd| f} n | dd| | | f} t t j | |||| |||||d }Wdn1sVwY|d7}| d|jjdd}| |jjdd| dd| | df<|d|jtkrnq| ddd| dfS)NrrrQ) rKrLrMr+r(r)r*rNrOr)rrr)rzerosrarb max_seq_lenr"rQr r`rrMATHcloneviewr\ get_token_idr)rKrkrMrlr(r)r*rNrOdecode_one_tokenr+iwin_sizewindow next_tokenr%r%r&decode_n_tokenssH   rzr)ru num_samplespromptmax_new_tokensr{c Ks|d}|d|dd}||jjkr td|d|jj|r5|||jjkr0|jj|}||} n|jj} | |}|j|j} } t| |j||jjt | jdWdn1scwYd|jj } tj d|| d} tj | |jjf| | d}||ddd|f<|}tj|d | tjd }tj|d | tjd }tj|d | tjd }t}|||d| d | |||||}||dd||df<tj|g| tjd } t||d| d | |d||||||d }|ddd|d|df}||dd|ddf<|S)zp Takes a conditioning sequence (prompt) as input and continues to generate as many tokens as requested. rNzInput sequence length z exceeds max_seq_len max_batch_sizerprrrQrmr(rPr)r*r)r(r)r*rNrOru)sizerepeatrarp ValueErrorrQrr setup_cachesnext parametersrbarangeemptyrZbfloat16rjrsr"rz)rKr|r}rNrOrur{sampling_kwargsrdT_newrQr codebook_dimrMrseqr(r)r*prefill_decode first_tokenrLr%r%r&generates|            " rFcCstj|dd}|j||d}tdt|tr"t}t}tdntdt ||j d|j j t |jdWdn1sEwY|rhtd tj|tjr[d nd tjrcd nddd }||fS)NT) load_weightsrPzRestored model from checkpointzUsing DualARTransformerzUnsupported model typerr~zCompiling function...inductor aot_eagerzreduce-overhead)backendmode fullgraph)rfrom_pretrainedr!r info isinstancerjrrrQrrarprrrcompilecuda is_availableeval)checkpoint_pathrQ precisionrrKruprefill_n_tokensr%r%r& init_model=s0       rc@s<eZdZUeded<dZeejed<dZ ee ed<dS)GenerateResponse)rJractionNcodestext) __name__ __module__ __qualname__r__annotations__rrrTensorrstrr%r%r%r&r_s rg?g?i) r{r}r)r*r(riterative_prompt chunk_length prompt_text prompt_tokensrQrurrrrrrc" csd|krdksJdJdd|kr dks%JdJdd|kr2dks7JdJd| duo>| du}|rLt| trL| g} | g} |dus\t| t| ks\Jdd d | D} td d |D}|j}td d}|jj}|rt | | D]\}}|j t |dt |dgdddq|j t |dgddd|j ||jjd\}}}|d|dkrtd|dd|d|j|d}td|tj||tjd}tj||tjd}tj||tjd}t|D]}tjrtjg}d}|d}t}t|||||||||d }|dkr4|dkr4| r4tdt|ddtjr?tjt|}|d|}||}td|d|d d!|d d"td#||d$d d%tjrtd&tjd$d d'|dd|d(f} | dk sJd)|dd|df}!| |!!| dk sJd*| t"d+| |d,V|d7}t"d-d.VqdS)/Nrrztop_p must be in (0, 1]z$repetition_penalty must be in (0, 2)ztemperature must be in (0, 2)Fz0Prompt text and tokens must have the same lengthcSsg|]}|qSr%)cpu).0rvr%r%r& sz!generate_long..css|] }|jr|VqdSN) requires_gradnumelrpr%r%r& sz generate_long.. interleave)modality)r)rT)add_endspeaker)rbizPrompt is too long: z > rzEncoded text: rP) rKr|r}rNrOrur(r)r*zCompilation time: z.2f secondsz Generated z tokens in .02fz seconds, z tokens/seczBandwidth achieved: geAz GB/szGPU Memory used: z GBrzNegative code foundzNegative code found: rJ)rrrr)r)#rrlensumrr\r rarpzipr_r rencode_for_inferencerbrrr!r rrrZr>r`rr synchronizetime perf_counterrmax_memory_reservedrrallrr)"rKrQrurr{r}r)r*r(rrrrr use_prompt model_sizer\base_content_sequence max_lengthtcencodedrNrO sample_idxglobal_encodedseg_idx prompt_lengtht0ytokens_generated tokens_secrdecodedr%r%r& generate_longfs$$$            rc@s.eZdZUeded<dZeeeBed<dS)WrappedGenerateResponse)successerrorstatusNresponse) rrrrrrrr Exceptionr%r%r%r&rs rc@s eZdZUeed<ejed<dS)GenerateRequestrequestresponse_queueN)rrrdictrqueueQueuer%r%r%r&rs rcsDttfdd}tj|ddS)Nc std\}}t|jd|jjt|jdWdn1s)wY   }|dur=dS|j }|j }zt d ||d|D] }|td|dqNWn"ty}}ztt|td|dWYd}~nd}~wwq3) Nrrr~T)rKrur)rrrr%)rrrQrrarprrrsetgetrrrputrrr r traceback format_exc)rKruitemkwargsrchunkerrrQ init_event input_queuerr%r&workersB    z(launch_thread_safe_queue..workerT)targetdaemon)rr threadingEventThreadstartwait)rrQrrrr%rr&launch_thread_safe_queues rz--textuQ你说的对, 但是原神是一款由米哈游自主研发的开放世界手游.)typedefaultz --prompt-text)rrmultiplez--prompt-tokens) path_typeexistsz --num-samplesz--max-new-tokensz--top-pz--repetition-penaltyz --temperaturez--checkpoint-pathzcheckpoints/openaudio-s1-miniz--devicerz--compile/--no-compile)rz--seed*z--half/--no-halfz(--iterative-prompt/--no-iterative-promptz--chunk-lengthi,z --output-dirtemprseedhalf output_dircCstj|dd| r tjntj}|dur*t|t|kr*tdt|dt|dtdt }t || || d\}}t | |j d|j jt|jd Wdn1s\wYtjrktjtd t |d d |durd d|D}t| tjrtj| t|| |||||||| | |||d}d}g}|D]T}|jdkr||jtd|jq|jdkr|rtj|d|d}t|tj|dd !td|tdg}|d7}qt"d|qdS)NT)exist_okzNumber of prompt text (z) and prompt tokens (z) should be the samezLoading model ...rrr~zTime to load model: rrcSsg|] }tt|qSr%)r from_numpynploadrr%r%r&rkszmain..)rKrQrurr{r}r)r*r(rrrrrrrJzSampled text: rcodes_z.npyr0zSaved codes to z Next samplezError: )#osmakedirsrrrrrr rrrrQrrarprrrrrr manual_seedrrr_rrpathjoinrsavecatrnumpyr)rrrr{r}r)r*r(rrQrrrrrrrrrKru generatoridxrrcodes_npy_pathr%r%r&main"st0             r__main__r)F)Ir rrrr contextlibr dataclassesrpathlibrtypingrrrrclickrrrtorch._inductor.configlogurur r transformersr fish_speech.content_sequencer r rfish_speech.tokenizerrenviron _inductorracoordinate_descent_tuningtritonunique_kernel_nameshasattrrtorch.nn.attentionrr&fish_speech.models.text2semantic.llamarrrr'rrHrJrjr"rzno_gradinference_moderrrrrQcallabler>boollistrrrrcommandoptionrrr%r%r%r&s             $   S 5 Y"         .            , V