o [۷i$@sUddlmZddlmZmZddlmZmZmZddl m Z m Z ddl m Z ddlmZmZmZddlmZdd lZd[ddZd[ddZd\ddZd]ddZGd d!d!eZiZd"ed#<d^d&d'Zd^d(d)ZGd*d+d+eZ Gd,d-d-e ej!d.Z!Gd/d0d0e ej"d.Z"Gd1d2d2e ej#d.Z#Gd3d4d4e ej$d.Z$Gd5d6d6e ej%d.Z%Gd7d8d8e ej&d.Z&Gd9d:d:e ej'd.Z'Gd;d<dd>e ej)d.Z)Gd?d@d@e ej*d.Z*GdAdBdBe ej+d.Z+GdCdDdDe ej,d.Z,GdEdFdFe ej-d.Z-GdGdHdHe ej.d.Z.GdIdJdJe ej/d.Z/GdKdLdLe ej0d.Z0GdMdNdNe ej1d.Z1GdOdPdPe ej2d.Z2GdQdRdRe ej3d.Z3GdSdTdTe ej4d.Z4GdUdVdVe ej5d.Z5GdWdXdXe ej6d.Z6GdYdZdZe ej7d.Z7d S)_) annotations)ABCabstractmethod)AnyCallableSequence)log2ceil) DTypeLike)GGML_QUANT_SIZESGGMLQuantizationTypeQK_K)LazyNumpyTensorNshape Sequence[int] quant_typer returntuple[int, ...]cCs`t|\}}|d|dkrtd|dd|jd|dg|dd|d||RS)NrzQuantized tensor row size () is not a multiple of z block size ()r ValueErrornamerr block_size type_sizerA/home/ubuntu/vllm_env/lib/python3.10/site-packages/gguf/quants.pyquant_shape_to_byte_shape ""r cCs`t|\}}|d|dkrtd|dd|jd|dg|dd|d||RS)Nrrz Quantized tensor bytes per row (rz type size (rrrrrrquant_shape_from_byte_shaper!r"func"Callable[[np.ndarray], np.ndarray]arr np.ndarrayotyper oshapec sz|d|jdf}d}|D]}||9}qtj||d}|jddp$d}tjfddt||Dd|d||S) Nrr )rdtypercsg|]}|qSr)ravel).0groupr#rr %sz,_apply_over_grouped_rows..)axisout)reshapernpempty concatenate array_split) r#r%r'r(rowsosizedimr1n_groupsrr.r_apply_over_grouped_rowss & r;ncCs6t|}t|}|td||}t||S)N)absr3floorsign)r<aflooredbrrr np_roundf+s rDc@s eZdZdS) QuantErrorN)__name__ __module__ __qualname__rrrrrE2s rEz)dict[GGMLQuantizationType, type[__Quant]] _type_traitsdataqtypecCsb|tjkr |jtjddS|tjkr|jtjddSt|}dur(| |St d|j d)NFcopyzQuantization for  is not yet implemented) r F32astyper3float32F16float16rIgetquantizeNotImplementedErrorrrJrKqrrrrU8s   rUcCsb|tjkr |tjS|tjkr|tjtjSt |}dur(| |St d|j d)NzDequantization for rN) r rOviewr3rQrRrSrPrIrT dequantizerVrrWrrrrZCs    rZc@s2eZdZUded<ded<ded<dZded<d Zd ed <d Zd ed<dZded<ddZd?ddZ e ddZ e e d@ddZ e e d@ddZe dAd d!Ze dAd"d#Ze dBd&d'Ze dBd(d)Ze dCd+d,Ze dCd-d.Ze dDd2d3Ze dDd4d5Ze dEd9d:Ze dFd;d<Ze dFd=d>ZdS)G__Quantr rKintrrNz,np.ndarray[Any, np.dtype[np.float32]] | Nonegrid)rrztuple[int, int] grid_shaperztuple[int | float, ...]grid_mapz bytes | Nonegrid_hexcCstdS)Nz-Quant conversion classes can't have instances) TypeError)selfrrr__init__Xsz__Quant.__init__rNonecCsb||_t|\|_|_tj|jtj|j fd|_ tj|j tj |j fd|_|tvs+J|t|<dS)N) meta_noop)rKr rrr_wrap_fn_Quant__quantize_arrayr3uint8_Quant__shape_to_bytes_Quant__quantize_lazy_Quant__dequantize_arrayrQ_Quant__shape_from_bytes_Quant__dequantize_lazyrI)clsrKrrr__init_subclass__[s    z__Quant.__init_subclass__cCs:|jdus |jdur dSttt|j}|dksJ|jjd|}tj |jtj d}| d}t |dk|d|d@tj ddgtj d d >}|d |d B}| d tj d dtddd|Dtj d d|f?}|d|>d@ d }tj |jtjd d}tj||dd}| ddg|jR|_dS)Nrr))rr=@ )r r=.r.r )rr cSg|]}|qSrrr,irrrr/xz%__Quant.init_grid..r )r rrr0)r]r`r rlenr_rKrr3 frombufferrhr2wherearrayrangerQtake_along_axisr^)rn bits_per_elemelems_per_byter]r_rrr init_gridis 4:z__Quant.init_gridblocksr&cCtNrVrnrrrrquantize_blocks~z__Quant.quantize_blockscCrrrrrrrdequantize_blocksrz__Quant.dequantize_blocksr7cCsp|jtjdd}|j}|j|j}|||jf}||}|jtj ks&J|jd|j ks0J|| |S)NFrLr) rPr3rQrsizerr2rr)rhrrirnr7rn_blocksrrrr quantize_rowss  z__Quant.quantize_rowscCsl|tj}|j}|j|j}|||jf}||}|jtj ks$J|jd|j ks.J|| |S)Nr) rYr3rhrrrr2rr)rQrrlrrrrdequantize_rowss   z__Quant.dequantize_rowsrrcC t||jSr)r rKrnrrrr__shape_to_bytes z__Quant.__shape_to_bytescCrr)r"rKrrrr__shape_from_bytesrz__Quant.__shape_from_bytesrcCst|j|tj||jdSN)r%r'r()r;rr3rhrirrnrrrr__quantize_arraysz__Quant.__quantize_arraycCs$|t|j|tj||jdSr)rr;rr3rQrlrrrrr__dequantize_arraysz__Quant.__dequantize_array lazy_tensorrrcCdSrrrnrrrr__quantize_lazyz__Quant.__quantize_lazycCrrrrrrr__dequantize_lazyrz__Quant.__dequantize_lazytensornp.ndarray | LazyNumpyTensorboolcCs|jd|jdkS)Nrr)rrrnrrrr can_quantizesz__Quant.can_quantizecCsB||std|jd|jjt|tr||S||S)Nz!Can't quantize tensor with shape z to ) rrErrKr isinstancerrjrgrrrrrUs    z__Quant.quantizecCst|tr ||S||Sr)rrrmrkrrrrrZs   z__Quant.dequantize)rKr rrdrr&rr&)r7r&rr&)rr)rr&rr&)rrrr)rrrr)rrrr&)rFrGrH__annotations__r]r^r_r`rcro classmethodrrrrrrrirlrgrkrjrmrrUrZrrrrr[NsP                 r[c@(eZdZed ddZed ddZdS) BF16rr&rcCsh|tj}t|d@dk|td@tdB|}t|d|d?d@d?}|tjtjS)Niili@ir*r )rYr3uint32ruint64rPuint16rh)rnrr<rrrrs *zBF16.quantize_blockscCs |tjtjd>tjS)Nr*)rYr3int16rPint32rQrrrrrs zBF16.dequantize_blocksNrrFrGrHrrrrrrrrs  r)rKc@r) Q4_0rr&rcCs|jd}t|jddd}tj||dd}|d}tjddt|dkdd |}Wdn1s6wYtj||td tjd  tj  dd }| |d |j d f}|ddddf|dd ddft d>B}| tjtj }tj||gddS)NrrTr0keepdimsr|ignoredivider g!@rqrtr=.ru)rr>argmaxr3rerrstatertruncrQrPrhclipr2rrSrYr5)rnrrimaxmaxdidqsrrrrs ..zQ4_0.quantize_blockscCs|jd}t|dg\}}|tjtj}||dd|jdftj ddgtj dd?}|t d@|dftj t d }||tjS) Nrr=rr rurqr r r=r rtrp) rr3hsplitrYrSrPrQr2rrrhint8)rnrrrrrrrrs 2*zQ4_0.dequantize_blocksNrrrrrrrs  rc@r) Q4_1rr&rc Cs&|jd}|jddd}|jddd}||d}tjddt|dkdd|}Wdn1s5wYtj|||td tjd tj  dd}| |d |j d f}|d dddf|d dddft d >B}|tj tj }|tj tj }tj|||gddS)NrrTrrtrrr ?rqr=.rur|)rrminr3rrrrQrPrhrr2rrSrYr5) rnrrrrrrrmrrrrs  2.zQ4_1.quantize_blockscCs|jd}t|dg\}}t|dg\}}|tjtj}|tjtj}||dd|jdftj ddgtj dd?}|t d@|dftj}|||S) Nrr=rr rurqrrt) rr3rrYrSrPrQr2rrrh)rnrrrrestrrrrrrs 2  zQ4_1.dequantize_blocksNrrrrrrrs  rc@r) Q5_0rr&rc CsH|jd}t|jddd}tj||dd}|d}tjddt|dkdd |}Wdn1s6wYtj||td tjd  tj  dd }| |d |j d f}|ddddft d@|dd ddft d>B}tj| |d dft d?ddd |d} | tjtj }tj|| |gddS)NrrTrr|irrr g0@rqr=.rtru littler0bitorder)rr>rr3rrrrrQrPrhrr2rpackbitsrSrYr5) rnrrrrrrrXrqhrrrr$s .8.zQ5_0.quantize_blockscCs|jd}t|dg\}}t|dg\}}|tjtj}|tj}||dftj ddt dDtjdd ?}||d d|j dftj ddgtj dd ?}|td@tj }|t d @|d f}||t d>Btj t d }||tjS)Nrr=rur cSrxrrryrrrr/Cr{z*Q5_0.dequantize_blocks..rrqr rrrrtr*)rr3rrYrSrPrQrr2rrrrhr)rnrrrrrrqlrrrr9s  22$zQ5_0.dequantize_blocksNrrrrrrr#s  rc@r) Q5_1rr&rc Cs`|jd}|jddd}|jddd}||d}tjddt|dkdd|}Wdn1s5wYtj|||td tjd tj  dd}| |d |j d f}|d dddft d @|d dddft d>B}tj | |ddft d?ddd |d} |tjtj }|tjtj } tj|| | |gddS)NrrTrrrrr rrqr=.rtrurrrr|)rrrr3rrrrQrPrhrr2rrrSrYr5) rnrrrrrrrXrrrrrrrNs  28.zQ5_1.quantize_blocksc Cs,|jd}t|dg\}}t|dg\}}t|dg\}}|tjtj}|tjtj}|tj}||dftj ddt dDtjdd ?}||d d|j dftj ddgtj dd ?}|td@tj }|t d @|d f}||t d>Btj}|||S) Nrr=rur cSrxrrryrrrr/pr{z*Q5_1.dequantize_blocks..rrqrrrrt) rr3rrYrSrPrQrr2rrrrh) rnrrrrrrrrrrrrds  22 zQ5_1.dequantize_blocksNrrrrrrrMs  rc@r) Q8_0rr&rcCst|jdddd}tjddt|dkdd|}Wdn1s'wYt||}|tjtj }|tj tj }tj ||gddS) Nr Trrrrr|) r>rr3rrrDrPrSrYrhrr5)rnrrrrrrrr{s zQ8_0.quantize_blockscCsFtj|dgdd\}}|tjtj}|tjtj}||S)Nr=r r|)r3splitrYrSrPrQr)rnrrxrrrrszQ8_0.dequantize_blocksNrrrrrrrzs  rc@eZdZedddZdS)Q2_Krr&rc Cs(|jd}t|tdg\}}t|tdg\}}t|dg\}}|tjtj}|tjtj}||d@tj|tddf}||d?tj|tddf} tj gdtj dd } ||d dd f| ?t d @}||tddftj}||| }||d fS) Nrr*rur=rtr rr=rurqr r rur rr) rr3rrrYrSrPrQr2rrh) rnrrscalesrrrdmindlmlshiftrrrrs $$  zQ2_K.dequantize_blocksNrrFrGrHrrrrrrrrc@r)Q3_Krr&rcCs|jd}t|tdg\}}t|tdg\}}t|dg\}}|tjtj}t|dg\}} ||ddftj ddgtj dd?}||df}| |ddftj gd tj dd ?} | |df} |t d @| t d @t d>B}|tj t d tj}|||ddf} ||ddd ftj gd tj dd?} ||ddd tj ddt dDtj dd?} | |dtdft d @} | |dtdft d@} | t dA} | tj | t d>tj tj} | | |tfS)Nrrpru r rqr r=r r*r)r rur rtrrrrcSrxrrryrrrr/r{z*Q3_K.dequantize_blocks..r r rpr r=) rr3rrrYrSrPrQr2rrhrr)rnrrhmaskrrrrlscaleshscalesrrrrXrrrrs( **&,4*zQ3_K.dequantize_blocksNrrrrrrrrrc@s,eZdZdZed ddZed d d Zd S)Q4_Krrr&rtuple[np.ndarray, np.ndarray]cCs|jd}|tj}||ddf}tj|ddd\}}}tj|d@|d@|d?d @Bgd d}tj|d@|d?|d?d @Bgd d}||d f||d ffS) Nrrrur|?rtr=0rrp)rrYr3rhr2rr5)rrrrm_dscrrrr get_scale_mins  &&zQ4_K.get_scale_minrc Cs|jd}t|dg\}}t|dg\}}t||jg\}}|tjtj}|tjtj}t |\}} ||tj |ddf}|| tj |ddf} | |dddftj ddgtj d d?}|t d @ |ddftj}|||  |t fS) Nrr=rr rrurqrrt)rr3r K_SCALE_SIZErYrSrPrQrrr2rrhr) rnrrrrrrrrrdmrrrrs ,"zQ4_K.dequantize_blocksN)rr&rrr)rFrGrHr staticmethodrrrrrrrrs  rc@r)Q5_Krr&rcCs|jd}t|dg\}}t|dg\}}t|tjg\}}t|tdg\}}|tjtj }|tjtj }t |\} } || tj  |ddf}|| tj  |ddf} | |dddftj ddgtj d d ?} | |dddftj d d tdDtj d d ?}| t d @ |ddf} |t d@ |ddf}| |t d>Btj } || |  |tfS)Nrr=rprr rrurqrcSrxrrryrrrr/ r{z*Q5_K.dequantize_blocks..rrt)rr3rrrrrYrSrPrQrr2rrhr)rnrrrrrrrrrrrrrXrrrrs ,6zQ5_K.dequantize_blocksNrrrrrrr rrc@r)Q6_Krr&rc Csj|jd}t|tdg\}}t|tdg\}}t|tdg\}}|tjtj}|tjtj}|| |tddf}| |dddftj ddgtj d d ?}|t d @ |dd f}| |ddd ftj gd tj d d ?}|t d@ |dd f}||t d>Btjtd }| |tddftj}|| |tfS)Nrr=rur*r rrrrqrrtrrrr) rr3rrrYrrPrQrSr2rrh) rnrrrrrrrrXrrrr)s ,,$zQ6_K.dequantize_blocksNrrrrrrr(rrc@r) TQ1_0rr&rc Cs|jd}t|jddd}tjddt|dkdd|}Wdn1s*wYt||}|tjtdtj }|ddd f|dd d f|dd df}}}| |dd d ftj gd tj d d}tj |dd |df}| |dd dftj gd tj d d}tj |dd |df}| |dddftj gdtj d d}tj |dd |df}tj |||gdd}|tjddd}|tj }|tjtj }tj ||gddS)NrrTrrrr .r)Qrsrr rqr r rr rr|r*ru)rrrsrr)rr>rr3rrrDrPrrhr2rsumr5rrSrY) rnrrrrrqs0qs1rrrrr@s&  4,,, zTQ1_0.quantize_blocksc Csp|jd}t|tdtddg\}}t|tdg\}}|tjtj}|dddf|dddf}}||dddftj gd tj d d }||df}||ddd ftj gd tj d d }||df}||dddftj gd tj d d}||df}tj |||gdd}|tj dd?tj t d}||tjS)Nrrurrr.rrr )r rrsrrrqrr*)r rrsrrr|rrp)rr3rrrYrSrPrQr2rrhr5rr) rnrrrrrrrrrrrrYs "",,,&zTQ1_0.dequantize_blocksNrrrrrrr?s  rc@r) TQ2_0rr&rcCs0|jd}t|jddd}tjddt|dkdd|}Wdn1s*wYt||}|tjtdtj }| |ddd ftj gd tj d  d >}|d dddf|d dddfB|d dddfB|d dddfB}| |df}|tj  tj }tj||gddS)NrrTrrrr rurrrqr.r=rr|)rr>rr3rrrDrPrrhr2rrSrYr5)rnrrrrrrrrrps  ,HzTQ2_0.quantize_blockscCs|jd}t|tdg\}}|tjtj}||dddftj gdtj dd?}|d @|dftj t d}||tjS) Nrrurr rrrqrr) rr3rrrYrSrPrQr2rrhr)rnrrrrrrrrs ,$zTQ2_0.dequantize_blocksNrrrrrrros  rc@s:eZdZdZed ddZeddd Zedd d Zd S)MXFP4)rr r=rrurrprrrririrr&rcCsBt|dktdt|>t|dtd>}|tjS)Nr=i r )r3rrrYrQ)rbitsrrre8m0_to_fp32_halfs6 zMXFP4.e8m0_to_fp32_halfrc CsH|jd}t|jddd}tjddt|dktt|dddtj }Wdn1s6wY| |}tj |j tj d d }t||d d f|tj|||jd f}tj|ddd}||d|jdtj }|dddf|ddd ft d >B}|||jdf}tj||gdd S)NrrTrrrr=rrq)r r r*r rur|)rr>rr3rrr?rrPrhrrkvaluesrr2rQrargminr5) rnrrrererrsbestrrrrrs 0 4*zMXFP4.quantize_blockscCs|jd}t|dg\}}||}||d|jdftjddgtjdd?}|td@tj }tj|j tj dddd}tj ||d d ||jf}|| tj S) Nrr r=rurqrrtr*rr|)rr3rrr2rrrhrYrrrrPrQ)rnrrrrrrrrrrs  0zMXFP4.dequantize_blocksN)rr&rr&r) rFrGrHrrrrrrrrrrr s  r c@s4eZdZUdZded<dZdZdZed d d Z d S)IQ2_XXSs !"$'(+-.03569:.rpr r r rprvr)rr3rrYrSrPrQrr2rr~rrhrrrr]rM) rnrrrrdbsignsrr]rrrrs$ ,.60zIQ2_XXS.dequantize_blocksNr) rFrGrHrrr^r_r`rrrrrrrs   rc@&eZdZdZdZdZed ddZd S) IQ2_XS)rprs00000200050008000a001100140016001900200022002500280041004400460049005000520055005800610064008000820085008800910094009900a000010104010601090110011201150118011a0121012401400142014501480151015401600168018101840190010002020205020802110214022002410244025002550280028a02010404040604090410041204150418042104240440044204450448045104540456046004810484049004000502050505080511051405200541054405500561058005010604061006260640064206840600080208050808080a08110814082008250841084408500858088008a008aa08010904091009400981098909000a200a280a960aa00a011004100610091010101210151018102110241040104210451048105110541060106a108110841090100011021105110811111114112011411144115011801194119611011204120612101240126012001402140514081411141414201441144414491450146414801401150415101540150016141649160118041810181218401854188618001905196619511aa91a00200220052008200a201120142020204120442050208020a020012104211021402148216521002222228022a82201240424102429244024002541255225992501261a26a626002808280a28202855288828a22868299029082a202a822a882a8a2a014004400640094010401240154018402140244040404240454048404a405140544060406540814084409040004102410541084111411441204141414441504180418541a2410142044210421242294240420044024405440844114414441944204441444444504480449444014504451045244540459a4500460a4644465046014804481048404845485448624800491149444950496949044a00500250055008501150145020502850415044505050805001510451105115514051425100524452aa520154045410542154405460548154a154005508558055885521566856a156005814584158505899581a5940594259855a0160046010604060546062608660a960006124624a62926200641664106540654565a46501686a682569066a546a626a00800280058008801180148020802a8041804480508080808280a880aa8001810481068110814081518159810082208280828282a082a8820184048410841284158440846084898400854485a58518866a860088088825885a8880888288a8880689228a808a888a968aa88a019004901090409056908490009122916491569289920094059444945094589429959095929541965198a6984999159a609a00a002a008a00aa020a02aa0a0a051a159a1a6a100a202a208a22aa280a2a0a240a495a465a698a60aa820a822a828a8a0a8a8a804a984a986a928aa2aaa91aaaaaarr&rc Cs|jd}t|dg\}}t|dtdg\}}|tjtj}|tj}| |ddftj ddgtj d d?}|d @ |df}|td |td }| |dddf}tj t jtj d ddd }tj||d ? |ddfdd}| |ddftj ddtdDtj d d?}|t d@}t|dktdtd}| |dddf}|jdusJtj|j|td@ |dddfdd} | |dddf} || | |dfS)Nrr=rprr rurqr r r=rtrrr"rsr|cSrxrrryrrrr/>r{z,IQ2_XS.dequantize_blocks..r r rpir)rr3rrrYrSrPrQrr2rrhr~rrrrrr]) rnrrrrrrr$r%r]rrrr,s&  * 4*zIQ2_XS.dequantize_blocksNrrFrGrHr^r_r`rrrrrrr's#r'c@r&) IQ2_S)irprs00000200050008000a0011001400160019002000220025002800410044004600490050005200550058006100640066006900800082008500880091009400a000a500aa0001010401060109011001120115011801210124014001420145014801510154015601590160016501680181018401900192019501a101a40100020202050208021102140220022a02410244024602490250025502800285028a029402a20201040404060409041004120415041804210424042604290440044204450448044a0451045404560459046004620465048104840486048904900495049804a104a40400050205050508050a0511051405160519052005250528054105440546054905500552055505580561056405800582058505880591059405a00501060406060609061006150640064506480651065406600681068406900600080208050808081108140816081908200825082a084108440846084908500852085508580861086408800885089408aa08010904091009120915091809210940094509480951095409600981099009000a110a140a220a280a2a0a500a990a01100410061009101010121015101810211024102610401042104510481051105410561059106010621065106810811084108610901095109810a110a41000110211051108110a1111111411161119112011221125112811411144114611491150115211551158116111641180118211851188119111941101120412091210121512211224124012451251125412811284129012001402140514081411141414161419142014251428144114441446144914501452145514581461146414801482148514881491149414a014011504150615091510151215151518152115241540154215451548155115541560158115841590150016051608161116141620164116441650168016aa160118041806180918101815181818211840184218451848185118541860188118841800190219051908191119141920194119441950196919a219041a101a401a561a00200220052008201120142016201920202025202a2041204420502052205520642080208a209420aa2001210421102112211521212140214221452151215421602181218421902100220a22222228222a224422502288228a22a8220124042406240924102415241824212424244024422445244824512454246024812484249024002505250825112514252025412544255025662580250126042610264026592600280528112814284128442850288a28aa2801290429102995290a2a222a642a882a8a2a014004400640094010401240154018401a4021402440264040404240454048404a40514054405640594060406240654081408440904095409840a140a440004102410541084111411441164119412041224125414141444146414941504152415541584161416441804182418541884191419441a04101420442104212421542184224424042454248425142544260428142844200440244054408440a44114414441644194420442244254428444144444446444944504452445544584461446444804482448544884491449444a044014504450645094510451245154518452145244540454245454548455145544560456a4581458445904500460246054608461146144620464146444650468046a546014804480948104812481548184821482448404842484548484851485448604884489048004902490549084911491449204941494449504980499649014a044a104a404a0050025005500850115014501650195020502250255028504150445046504950505052505550585061506450805082508550885091509450015104510651095110511251155118512151245140514251455148515151545160518151845190510052055208521152145220524152445250526952805201540454065409541054125415541854215424544054425445544854515454546054815484549054005502550555085511551455205541554455505580550156045610562656405600580258055808581158145820584158445850585a5880580159045910594059005a195a855aa85a016004600660106012601560186021602460406045604860516054606060846090600061026105610861116114612061416144615061806199610462106240625662a162006405640864116414642064416444645064806401650465106540654a6568659265006694660168046810686568986800692a69426aa16a00800280058008801180148019802080258041804480508052805580588061808080858091809480018104810981108112811581188121812481408142814581488151815481818184819081a981008205820a82118214824182448250820184048406840984108412841584188421844084428445844884518454846084818484849084008502850585088511851485208541854485508580858a85018604861086298640860088058811881488418844885088a2880189048940896589228a588a5a8a828aa28a01900490099010901290159018902490409042904590489051905490609081908490909000910591119114914191449150915a910192049210924092a69200940294059408941194149420944194449450948094969401950495109540959895a19500964696649601980498109826984098a998009949995299909a00a005a00aa014a022a02aa041a044a050a0a2a0aaa040a165a102a20aa222a228a22aa282a288a28aa2a8a201a404a410a440a489a4a4a400a519a551a60aa828a8a2a854a986a908aa0aaa20aa22aa28aa88aaaaaarr&rc Cs|jd}t|dg\}}t|tdg\}}t|tdg\}}t|tdg\}}|tjtj}||ddftj ddgtj dd ?}|d @|df}|td |td } | |dddf} ||ddftj d dt dDtj dd?}|t d@}t |dktdtd}||dddf}||ddftj gdtj dd?}|tj |d@tj d>|dfB}|jdusJtj|j||dddfdd} | |dddf} | | ||dfS)Nrr=rprrr rurqr)rtrrcSrxrrryrrrr/r{z+IQ2_S.dequantize_blocks..r*rr rrr|rr3rrrYrSrPrQr2rrhrrrr]r) rnrrrrrr%rrr$r]rrrrs( *4** zIQ2_S.dequantize_blocksNrr+rrrrr,JsCr,c@r&) IQ3_XXS)rru)rurr$,4>s0000020004001100130017002000220031004200730075000101030110011201210125013001320141015401700100020202040211022002220231023302370251025702750201030703100312032503700313043704440457047304750401050705320552053506640610071407160743076107011003101010121021102310301032103410471050100011021111112011221101120312101212122112301272120013021320133113461366130114051450142015241546157115051622174017002002201120132020202220262031204220012103210521102112212121302163216721702100220222112217222022222237224022552201231023142370237423352453240325272541257425012703271627452701301030123021302330503065307230003102312031313144314631013203321032253252327232113333333034473472340035063522355535143636366336333760370440174035403740534057407441204237424042604266420743454304445144644425454345704505471047124730471250415070500051065126515551145232527252025353531054235427547254025531555056245742572460446046606460216161611762646230633663446405655265336603672167037005700770107032705270267140711272457252720073157333736073217441740075027524753076rr&rc Cs|jd}t|dg\}}t|tdg\}}|tjtj}|tj}|td|d?tjtd}| |dddf}| |ddftj gdtjd  d ?}tj t j tjd  d } |td @ |dddf}tj| |dd }| |dddftj ddtdDtjd  d?}|td@}t|dktdtd}| |dddf}|jdusJtj|j| |dddfdd } | |dddf} || | |dfS)Nrr=rurrrr rrqr r!rr|cSrxrrryrrrr/r{z-IQ3_XXS.dequantize_blocks..rpr#r)rr3rrrYrSrPrQrr2rr~rrrhrrrr]) rnrrrrrrr$r%rr]rrrrs&  (*6 zIQ3_XXS.dequantize_blocksNrr+rrrrr.sr.c@r&) IQ3_S)r(ru)r rrrrs rts00000100020005000700100011001200140016002000210025003300400042004500470051005300600062007100740077000001010102010401100111011501200123012701310135014401610165017201000201020502070210021302160221022502300234024202450247025102530270027302030311031503200322033103330336034403500352036703710375030004130417042104240432044004430451047004020504052005220526053305410545054705660573050606110613063106520671060007020704072007220726073307500754070010011002100410101011101310151017102010221031103410361054105610611072100011011103110611101114112111301133114111501152117011761100121212151217122012241232124012431255126012721201130413071310131313211327133013341341136213701303140514121414143114331442144614501454140115101513152115301532155115201624162716441646160117031710171217211735174117621770170020012003200520072010201220142016202120232027203020322041204320452050205220672070207320752000210221102113211721222125213121342142215121012204220722212223223022372241225322572271227422002302230523112322232423312333234223502366230124072420242324322435244124722475240425112522253725402553257025002602260726212655266126052711272627302743275027023011301330153017302230313033303530423044304730513063307130013103310531143121312331403160317231763100321232203232323432503201331033143321332333273330334133433347335533733303341134163422343134523460346434013510351235253532354435563573351636413601370337203722373537004004401240204024402740324041405040704002410741114113412241304135414341514155410142034210421542214233424042574262427042044311431343204322433143354300440244244437444044714405450745214562451346344660461047154730474347514702501050145022504050445047505250665074500151035105511251215132517251005211522352305236525352025307531053275344535153655373530154045420543254465412552655515553554256025704572257116013601560316033606060006120612761646112623462426255626262706200631463216340632564436462640065036534656065056640661167136700700470077020702270367040705470627002711171247143714571017204721072167221723072517202733273357353730174057413742074507422754275027631760077rr&rc Cs|jd}t|dg\}}t|tdg\}}t|tdg\}}t|tdg\}}|tjtj}||ddftj ddgtj dd ?}|d @|df}|dd|} | |dddf} ||ddftj d d t dDtj dd ?}|t d@}t |dktdtd}||dddf}||ddftj dd t dDtj d?}|d@tj |df}|tj |d>B}|jdusJtj|j||dddfdd} | |dddf} | | ||dfS)Nrr=rurrprr rqr)rtcSrxrrryrrrr/r{z+IQ3_S.dequantize_blocks..r*cSrxrrryrrrr/"r{rr|r-) rnrrrrrrr%rr$r]rrrr s* *4. zIQ3_S.dequantize_blocksNrr+rrrrr4s#r4c@s0eZdZdZdZdZedZe d dd Z d S) IQ1_S)irp)rrr s 00000200050008000a00110015002000220028002a00450051005400560065008000820088008a009500a000a200a800aa000401050111011401160119011a012501410146014901520155015a0161016401660168018501910194019601a5010002020208020a0215022002220228022a02450251025902640269028002820288028a02910295029902a002a202a802aa0211041404160425044104490455045a046404650491049904a5040105040505050605150518051a052905400545054a0550055105540555055605590560056205650568056a0581059105950598059a05a105a405a505a605a90514061906410644065006520655065806600661066606690685069106940699060008020808080a0815082008220828082a0845085108560865088008820888088a089508a008a208a808aa0805091109140919092409250941095009510955096109640969099109940996099909a509000a020a080a0a0a150a200a220a280a2a0a450a510a590a610a650a800a820a850a880a8a0a950aa00aa20aa80aaa0a101011101410191024102510411044105010551058106110641065106910911094109610a110a5100111041106110911101112111511181121112411291145114a11501151115211541155115611591160116511841192119511a111a41111121412161225124012461249125212551258125a12641266128512911294129612a51201140614091414141514181419142114261441144514461448144a145114541455145614591462146514681484148914901494149514981499149a14a114a414a514a914021505150a151115141515151615191520152215251528152a1541154415451546155115521554155515561559155a1561156415651566156915801582158415851588158a159015911594159515961599159a15a015a215a51501160416051606161516161618161a1621162616401642164416451648164a1651165516561658165916611664166516681669166a1686168a1692169516a416a91611181618251841184418461849185018551858185a1860186118641866186918851891189418a5181019121915191a19211925194219441945194819511954195519561959195a19601965196a1989199119921995199819a119a619a919091a161a241a261a441a461a491a501a521a551a581a611a661a691a851a911a961a9a1a0020022008200a20152020202220252028202a20452051205920612065208020822088208a209520a020a220a520a820aa2005211121142119212521422144214921552158215a2161216421652166218521902196219921a521012208220a22112215222022222228222a2245225122562259226522812288228a2291229522a022a222a822aa220524142416241924252444244524462449245224552458245a2466248524912494249924a124a52409251525212529254025452548255125542555255925622565256825892590259425952598259a25a125a425a625a925052610261226192625264126492655266026612669268426862690269a260028022808280a2815282028222828282a2845285128542865288028822888288a28a028a228a828aa2809291129142919292529462949295229552961296429662969298529902996299929a429a529002a022a082a0a2a202a222a282a2a2a452a512a562a592a652a802a822a882a8a2a952aa02aa22aa82aaa2a054011401640254049405240554058405a4061406440664094409940a140a6400041014104410641094112411541164118411a41214126412941454148414a41514154415541564159415a41654168416a41814184418641904192419541a041a141a241054211421442164225424142524255425a426442694289429442a5420144154419442944454448444a44514454445544564461446244654468446a44814486448944904492449544a044a144a9440145024505450a4511451445154516451945204525452a45414544454545464549455045514554455545564558455945614564456545664569458245844585458845914594459545964599459a45a545a845aa450146054609461446154618461a462146244629464046424645464846504651465246554656465946624665466846814685468a4694469546a146a446a6460548114815481a48254842484948504855485848614864486648694885489148944896489948a5480149054906490a491049144915491849214924492649404945494a4951495249544955495649594960496249654966496a49864989499249954996499849a149a449a649a949164a444a464a494a554a584a5a4a644a694a944aa54a01500450055006500950125015501a5021502450295040504550485051505450555056505950655068508650895095509850a050a150a650a9500551085109510a5111511451155116511851195120512551265128512a5141514451455146514951505151515251545155515651585159515a51615164516551665169518251855191519451955196519951a051a551aa5101520652125215521a5221522452425245524a525152545255525652595262526552855290529252955299529a52a452045405541154145415541654185419542154255428542a54415444544554465449544a5450545154545455545654585459545a54615462546454655466546954805488548a5491549454955496549954a154a454a554aa5401550255045505550655095510551155125514551555165519551a552155245525552655295540554155425544554555465548554955505551555255545555555655585559555a5560556155645565556655685569556a5581558455855589558a559055915594559555965598559955a155a455a555a655a95500560156025604560656085609561156145615561856195620562156225624562556265628562956415645564656485649564a56505651565256545655565656585659565a566156645665566956825685568656885689568a56915695569a56a256a556a656a856a956045805580658095810581558185821582a58455848584a585158545855585658585859586058625864586558825889589058925895589858a158a9580159025905590a5911591459155916591959255941594459455946594959505951595259545955595659585959595a5961596459655966596959815985598959915994599559965998599959a559045a085a155a1a5a205a255a265a295a455a485a495a515a555a565a585a595a625a655a685a6a5a815a8a5a925a955a965a985a9a5aa15a05601460166019602560446050605560566058605a60616064606660696081609660a5600161046106610961126115612161226126612961456149615161556156615961656166616a6184618a6192619561a161a661a9611162166219624062416246625562566258626062856291629662a56211641264156416641a6421642664296440644264456448644a64516454645564566459645a646064626465648464856489649064926494649564966498649a64a164a464a964056508650a651165156516651965446545654665496550655165546555655665596561656465656566656965866589658a6591659565966599659a65a265a565a665a86502660966156620662666286629664066456648664a66516654665566566658665a666066656668668066826685668a669466966698669966a066a466a666aa661668196825684168526855685a6861686968856891689868a668016904691069156921692469266929694069416945694669486951695469556956695969606965696a69826984698a699569a169a469a569a969116a166a186a416a446a496a506a556a586a5a6a646a656a696a866a946a986a9a6aa66a0080028008800a802080228028802a8045805080518054805680598065808080828088808a809580a080a280a880aa8005811181148116811981258141814481498150815281558156815881598164816681698185818981948196819981a5810082028208820a8215822082228228822a8251825482598265828082828288828a829582a082a282a882aa821484198441844484518455845a846184648469849484998401850985128515851a85268529854085418545854885518554855585568559855a856585668568856a8581858485868589859085928595859885a68511861686198625864186448649864a865086558659865a86618666866a86858691869a86a4860088028808880a8815882088228828882a8841884588518854885988658869888088828888888a889588a088a288a888aa8805890689118914891689258941894489468949895089528955895a8961896489858996899989a589008a028a088a0a8a158a208a228a288a2a8a458a518a548a568a808a828a888a8a8a958aa08aa28aa88aaa8a059011901690189019902590419046904990559058905a9069906a9085909190949096909990a59001910491069109911091159118911a912191249126912991409145915091519154915591569159916291659184918691929195919891a191a491a691a99105921192149219922592449246924992509252925592589266926992859294929692a992019404940694109415941894269440944a945194549455945694589459946094619462946594849486949294949495949894a194a9940095059508950a951095119514951595169519952195259529952a9541954495459546954995509551955295549555955695589559955a95619564956595669569958195859588959195929594959595969599959a95a095a295a595a895aa950196049610961596199620962696299645964896499651965296559656965996659668968296849689968a96929694969596a496a696a99605981698199825984198469850985298559856985a98649865988598919896989998a59804990699099910991299159918991a99209921992499269940994299459948994a99519954995599569959996299659966996a99819984999099929995999a99a199a699059a159a259a449a469a499a509a559a589a619a859a919a949a959a969a00a002a008a00aa015a020a022a028a02aa045a051a054a056a059a080a082a088a08aa095a0a0a0a2a0a8a0aaa005a109a111a114a116a119a11aa146a149a151a155a158a15aa161a164a185a190a192a196a199a102a208a20aa210a219a222a228a22aa245a251a256a259a265a280a282a288a28aa295a2a0a2a2a2a8a2aaa219a425a441a444a450a454a455a458a45aa461a465a466a468a469a485a406a509a510a512a515a518a526a529a542a545a551a554a555a556a559a565a56aa581a584a585a586a589a592a595a598a505a611a616a61aa621a625a644a646a64aa652a655a656a658a660a662a686a690a695a696a699a6a1a6a4a6a6a600a802a808a80aa820a822a828a82aa851a854a856a859a880a882a888a88aa895a8a0a8a2a8a8a8aaa805a914a919a921a925a941a950a955a95aa961a966a969a990a996a900aa02aa08aa0aaa20aa22aa28aa2aaa51aa54aa56aa80aa82aa88aa8aaa95aaa0aaa2aaa8aaaaaag?rr&rc CsR|jd}t|dg\}}t|tdg\}}|tjtj}|tj}|d|d?d@d}| |dddf}t |td@dk|j |j }| |dddf}| |ddftj gd tjd  d ?}|tj|d@d> |dfB}|j dusJtj|j | |dddfd d } | |dddf} || | |dfS)Nrr=rprrr rirrrrsrqr rr|ru)rr3rrrYrSrPrQrr2rdeltarr]r) rnrrrrrrrr9r]rrrrs  "*" zIQ1_S.dequantize_blocksNr) rFrGrHr^r_r`r3rQr9rrrrrrr7-s r7c@s2eZdZejZejZejZejZedddZ dS)IQ1_Mrr&rc Cs|jd}t|tdg\}}t|tdg\}}|tj}||dftd@tjgdtjdd?}|d |d B|d B|d B}|tj tj |d f}||dd tjgdtjdd?}|d@|df}|d|d }||ddd d f}||dd ftjddgtj dd?}| tj|d@ tjd>|dfB}t |d@dk|j |j } | |dddd f} |jdusJtj|j||dd d fdd} | |ddddf} || | |dfS)Nrrpr*rui)rrprurrq)r rurvrw).r=).rr rr8r rr=r)rr|)rr3rrrYrr2rrSrPrQrhrr9r]r) rnrrrrrrrrr9r]rrrrs&  2 (** zIQ1_M.dequantize_blocksNr) rFrGrHr7r^r_r`r9rrrrrrr:sr:c@seZdZdZedddZdS) IQ4_NL)iiiiiiiir r6r&5EYqrr&rcCs|jd}t|dg\}}|tjtj}||dd|jdftj ddgtj dd?}|t d@|ddf}tj |j tj dddd }tj ||dd tj|df}||S) Nrr=rr rurqrrtr*r|)rr3rrYrSrPrQr2rrrhrrr)rnrrrrrrrrrs 2"zIQ4_NL.dequantize_blocksNr)rFrGrHrrrrrrrr;sr;c@r)IQ4_XSrr&rc Cs|jd}t|dg\}}t|dg\}}t|tdg\}}|tjtj}|tj}| |ddftj ddgtj d d?}| |ddftj d d t td Dtjd d ?}| |dft d @}| |dftj t d@}||t d>Btj t d }||tj |ddf} | |dddftj ddgtj d d?}| |dd dft d @}tj tjtj d d} tj| |ddtj |dd f}| | |dfS)Nrr=rrrr rurqr)cSsg|]}d|qS)r=rryrrrr/sz,IQ4_XS.dequantize_blocks..r)r rr rtrr*r)r r r rr|)rr3rrrYrSrPrQrr2rrhrrr;rr) rnrrrrscales_hscales_lrrrrrrrr s"  *8 $,$zIQ4_XS.dequantize_blocksNrrrrrrrA rrA)rrrr rr) r#r$r%r&r'r r(rrr&)r<r&rr&)rJr&rKr rr&)8 __future__rabcrrtypingrrrmathrr numpy.typingr constantsr r rlazyrnumpyr3r r"r;rD ExceptionrErIrrUrZr[rrrrrrrrrrrrrr rr'r,r.r4r7r:r;rArrrrsR          ~"%*-,20!3AFh4G$'