o ߥi@sNddlZddlZddlZddlZd ddZd ddZddZGdd d ZdS) NcCsPi}t|d<tj|d<t|d<|dur&|jdkr&tj||d<|SNrng_state_torch rng_state_np rng_state_rndcudarng_state_torch_cuda)torch get_rng_statenprandom get_stategetstatetyper)device random_statesrX/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/models/nlp/peer/sas_utils.pyget_random_statess  rcCsZt|dtj|dt|d|dur)|jdkr+tj|ddSdSdSr)r set_rng_stater r set_statesetstaterr)rrrrrset_random_statess rcCs|durdSddg}t|r&t|rd|d<t|r%d|d<nTt|turztt|D]+}t||r]t||rL|dd7<t||r]|dd7<q2|ddkrl|dd7<|ddkrz|dd7<t |dkr|SdS)Nr ) r is_tensorisnananyisinfrtuplerangelensum)dataresultirrr check_nan_inf%s,    r%c@s&eZdZdddZddZddZdS) SequenceSideInfoNcs|dur|_n ddlm}|d_tjj_fddt jj Dt fddt t D_ ddD}tj|tjd_dS) Nr)ElectraTokenizerzgoogle/electra-small-generatorcsg|] }j|gqSr) tokenizerdecode.0r$)selfrr Ls z-SequenceSideInfo.__init__..cs$g|]}|dddkr|qS)r##rr*)tokensrrr-Qs$cSs\g|]*}|ddkr|ddkrdn|dddkr%dtdt|dntdt|qS)r[]r.r/r)minr )r+trrrr-Rs  dtype)r( transformersr'from_pretrainednltktokenizepunktPunktSentenceTokenizer sen_tokenizerr vocab_sizesetr ind_subtokensrtensorint8 len_tokens)r,r(r'tmpr)r,r0r__init__Bs"   zSequenceSideInfo.__init__cs|j|}tdd|j|dD|ddd<tfddtt D}tfddtt D}t||fS)NcSsg|]}t|dqS)r.)r )r+xrrrr-]s z1SequenceSideInfo.getSenTokIdx.. input_idsrrcs$g|]}|tj|tjdqSr7)r onesrDr* sen_lengthsrrr-dscs g|] }tj|tjdqSrJ)r arangerDr*rLrrr-hs) r?r<r arrayr(batch_encode_plusr! concatenaterr )r,sentence_position_embedding inputs_str seq_len_total sentencesidx_senidx_tokrrLr getSenTokIdxZs    zSequenceSideInfo.getSenTokIdxcsd}tdttjfrd}ttjdr4j}tjt fdd|Dj d}ntjt fdd Dj d}t }j d }|ddd|f|d <|ddd |d |f|d <d krtjdd \}} jt|} t| dkrtfdd| Djdd} n tjj tjd} d| dddf<| ddd df} td dD]"} t| | k| ddddf| k}t|dkrn| d | |<q| |d<j}||d<|r|D] }|| ||<q|S)NFrT batch_decodecs g|] }|jdqSr)rXshape)r+ input_str inputs_idr,rRrrr-xs z;SequenceSideInfo.generate_seq_side_info..)rcs(g|]}j|jdqSrZ)rXr(r)r[)r+ input_orir]rrr-s r ss_sentence_position_in_sequencer.ss_token_position_in_sentence)return_inversecsg|]}|kqSrr)r+st)r^rrr-s)axisr7 r2ss_token_position_in_whole_wordss_token_string_length) isinstancelistr ndarrayrrChasattrr(rYrOrnumpydictr[uniquerB intersectionrAr stackrcharzerosrDr logical_andrElongkeys)r,rRr^ is_np_arrayrS sen_tok_idxside_info_dict seq_lengthrn_rB idx_tok_ww idx_tok_ww_1r$posinputs_str_lenkeyrr]rgenerate_seq_side_infoost        z'SequenceSideInfo.generate_seq_side_infoN)__name__ __module__ __qualname__rGrXrrrrrr&@s  r&r) r r;rlr rrrr%r&rrrrs