o qoi5@s"dZddlZddlZdZddZdEddZd d Zd d Zd dZ ddZ ddZ ddZ ddZ ddZddZddZddZdd Zd!d"Zd#d$ZdFd%d&Zd'd(Zd)d*Zd+d,Zd-d.Zd/d0Zd1d2Zd3d4Zd5d6ZdFd7d8Zd9d:Zd;d<Z d=d>Z!d?d@Z"dAdBZ#dCdDZ$dS)GzO Kaldi IO function implement(for binary format), test pass in Python 3.6.0 NFcCstrt|dSdSN)debugprint)inforN/home/ubuntu/.local/lib/python3.10/site-packages/kaldi_python_io/_io_kernel.py print_infos rcCs|st|dSr) RuntimeError)okrrrrthrow_on_errorsr cC(t|d}t|dkd|dS)z\ Generally, there is a space following the string token, we need to consume it  zExpect space, but gets Nbytesdecodereadr )fdspacerrr expect_spacercCr )zd Read the binary flags in kaldi, the scripts only support reading egs in binary format BzExpect binary flag, but gets Nr)rflagsrrr expect_binary rrcCsFd} t|d}|dks|dkrn||7}q|dkrdS|S)zU Read {token + ' '} from the file(this function also consume the space) r TrrN)rrrstrip)rkeycrrr read_token(srcCs|t|ddS)z< Write a string token, following a space symbol rNwritestrencode)rtokenrrr write_token5sr%cCs&t|}t||kd|d|dS)zC Check weather the token read equals to the reference zExpect token 'z ', but gets N)rr )rrefr$rrr expect_token<sr'cCst|}|r t||S)zI Read the binary flags following the key(key might be None) )rr)rrrrrread_keyDsr(cCs|tddS)z$ Write a binary symbol rNr rrrrwrite_binary_symbolNsr*cCsBt|d}t|dkd||d}td|}|dS)z: Read a value in type 'int32' in kaldi setup rExpect '\04', but gets irrrrr structunpack)rint_sizeint_strint_valrrr read_int32Us   r5cCs*|tdtd|}||dS)z" Write a int32 number r+r.N)r!r"r#r0pack)rint32int_packrrr write_int32`s r9cCs>t|d}t|dkd||d}td|}|S)z> Read a value in type 'BaseFloat' in kaldi setup rr+r,r-fr/)r float_size float_str float_valrrr read_float32is  r>cCstd||dvrtd||dkrdnd}|dkr!tjntj}t|}t|}td|d|||||}tj||d }|||S) z Read common matrix(for class Matrix in kaldi setup) see matrix/kaldi-matrix.cc:: void Matrix::Read(std::istream & is, bool binary, bool add) Return a numpy ndarray object z Type of the common matrix: )FMDMUnknown matrix type in kaldi: r?r-z Size of the common matrix: z x dtype) rr npfloat32float64r5r frombufferreshape)rmat_typer; float_typenum_rowsnum_colsmat_datamatrrrread_common_matus rPcCsp|jtjtjfvrtd|j|jtjkrdnd}t|||j\}}t||t|||| dS)z# Write a common matrix Unsupported numpy dtype: r?r@N) rDrErFrGr r%shaper9r!tobytes)rrOrJrLrMrrrwrite_common_mats    rTcCsztd||dvrtd||dkrdnd}|dkr!tjntj}t|}td||||}tj||dS) zc Read float vector(for class Vector in kaldi setup) see matrix/kaldi-vector.cc z Type of the common vector: )FVDVrArUr-rBz Dim of the common vector: rC)rr rErFrGr5rrH)rvec_typer;rKdimvec_datarrrread_float_vecsrZcCst|jtjtjfvrtd|j|jtjkrdnd}t|||jdkr)td|j}t||| | dS)z" Write a float vector rQrUrVrz%write_float_vec accept 1D-vector onlyN) rDrErFrGr r%ndimsizer9r!rS)rvecrWrXrrrwrite_float_vecs   r^cs:|rtt}tjfddt|Dtjd}|S)z, Read int32 vector (alignments) csg|]}tqSrr5).0_r)rr sz"read_int32_vec..rC)rr5rEarrayranger7)r direct_accessvec_sizer]rr)rread_int32_vecs "rgcCsdt|dt|}t|}td|d|dg}t|D]}t|}t|}|||fq|S)z Reference to function Read in SparseVector Return a list of key-value pair: [(I1, V1), ..., (In, Vn)] SVz Read sparse vector(dim = z, row = ))r'r5rrdr>append)rrX num_elems sparse_vecraindexvaluerrrread_sparse_vecs  rocCs:td|t|}g}t|D] }|t|q|S)zs Reference to function Read in SparseMatrix A sparse matrix contains couples of sparse vector  Following matrix type: )rr5rdrjro)rrJrL sparse_matrarrrread_sparse_mats  rrc Cs|\}}}}td|d||dkrt||d|ks J|dd||d|d}}tj|tjdtj} t| |d} | |d|} tj|tj dtj} t| ||} | d k} | d k} t | | | d | d d | d t | | d| d| dd| d| d | d| d d| d S|dkrt |d} tj|tjdtj}nt |d} tj|tj dtj}||||| }|S)a In format CM(kOneByteWithColHeaders): PerColHeader, ...(x C), ... uint8 sequence ... first: get each PerColHeader pch for a single column then : using pch to uncompress each float in the column We load it seperately at a time In format CM2(kTwoByte): ...uint16 sequence... In format CM3(kOneByte): ...uint8 sequence... z Uncompress to matrix z X CMrBNrCr-g@@rrgP@rgO@g`@CM2go@) rlenrErHuint16astyperF transposerIuint8wherefloat)cdatacps_typeheadmin_valprangerLrMcheadcmainpchr} le64_index gt92_indexincuint_seqrOrrr uncompresss6 """  rcCs"t|}t|}t|}|||fS)zh Read the member in struct Index in nnet3/nnet-common.h Return a tuple (n, t, x) r_)rntxrrrread_index_tuples rcCstd|dd}|dkr+t|dkrd|dfS|dkr'tdd|dt|S||d}t|dkrD|d|d||d fS|dkrQtdd|dt|S) z Wapper to handle struct Index reading task(see: nnet3/nnet-common.cc) static void ReadIndexVectorElementBinary(std::istream &is, int32 i, std::vector *vec) Return a tuple(n, t, x) brr}FzUnexpected character z( encountered while reading Index vector.r)r0r1rabsr r)rrmcur_setr prev_indexrrr read_indexs&      rcCsLt|dt|}td|g}t|D] }t|||}||q|S)zp Read several Index and return as a list of index: [(n_1, t_1, x_1), ..., (n_m, t_m, x_m)] zz Size of index vector: )r'r5rrdrrj)rr\rmr. cur_indexrrrread_index_vec<s    rcCstd|td|d}td||d|d}}|dkr+|d|}n|d kr6d||}n|d kr?||}ntd d |||}t|||}|S) za Reference to function Read in CompressMatrix Return a numpy ndarray object rpffiiz Compress matrix header: rrwrsrBrxCM3Fz!Unknown matrix compressing type: )rr0r1rr r)rrJrrLrM remain_size compress_datarOrrrread_compress_matKs   rcCs6|ddkr t||S|ddkrt||St||S)zr Reference to function Read in class GeneralMatrix Return compress_mat/sparse_mat/common_mat rCS)rrrrP)rrJrrrread_float_matds     rcCs4|rt|t|}|ddkrt||St||S)z% Read float matrix or vector V)rrrZr)rrerWrrrread_float_mat_vecps    rcCsDt|tjr|jdkrt||dSt||dStdt|)z& Write float matrix or vector rzUnsupport type: N) isinstancerEndarrayr[rTr^ TypeErrortype)r mat_or_vecrrrwrite_float_mat_vec}s  rcCstt|di}t|}||d<td|t|}||d<t|t|}t||}||d<t|t|d|S)z Reference to function Read in class NnetIo each NnetIo contains three member: string, Index, GeneralMatrix I store them in the dict:{'name': ..., 'index': ..., 'matrix': ...} znamez Name of NnetIo: rmmatrixz )r'rrrr)rnnet_iorrmrJrOrrr read_nnet_ios   rcCsJt|dt|dt|}g}t|D] }|t|qt|d|S)z Reference to function Read in class NnetExample Return a list of dict, each dict represent a NnetIo object a NnetExample contains several NnetIo z zz )r'r5rdrjr)rnum_ioegsrarrrread_nnet3_egss    rcc( t|}|s dSt|}||fVq)zh Usage: for key, eg in read_nnet3_egs(ark): print(key) ... TN)r(r)rrrrrrread_nnet3_egs_arks rccr)z Read float matrix/vector Usage: for key, mat in read_ark(ark): print(key) ... TN)r(r)rrobjrrrread_float_arks rccr)z( Read int23 vector (alignments) TN)r(rg)rralirrrread_int32_alis r)r )F)%__doc__r0numpyrErrr rrrr%r'r(r*r5r9r>rPrTrZr^rgrorrrrrrrrrrrrrrrrrrrsH        0