o 2wi,@sddlmZddlmZddlmZddlZdZeej dr)dZej j Z ej j Z nej j Z ej j Z ej dd e d  dQd ejd ejd ejd ejdejdeejdeejdeejeejffddZej dde d  dQd ejdejdejdejdeejdeejdejfddZej dde dd ejdejfddZej dde dd ejdejd ejfd!dZej d"d#e d"dRd ejfd%dZej d&d'e d&d ejd(ejdejfd)dZe d&d*d ejd(ejfd+dZej d,d-e d,  dQd ejdejdejdeejdeejdejf d.dZej d/d0e d/ $dRd ejdeejejejejeejffd1dZej d2d3e d2d ejd4ejd5ed6ed7eedejdejfd8dZej d9d:e d9d ejd4ejd5ed6ed7eedejd ejddfd;dZej dejdeejejff d?dZej d@dAe d@d ejd4ejdBejd5edejdejf dCdZej dDdEe dDd ejd4ejdBejd5edejd ejf dFdZej dGdHe dGd ejdBejd5edeejejffdIdZej dJdKe dJd ejdejdLeed4ejdBejd5edejfdMdZej dNdOe dNd ejdejdLeed4ejdBejd5ed ejddfdPdZdS)S)Sequence)prod)OptionalNF register_fakeTz"bitsandbytes::int8_mixed_scaled_mmz{(Tensor A, Tensor CA, Tensor CB, Tensor SCA, Tensor SCB, Tensor? outlier_cols=None, Tensor? bias=None) -> (Tensor, Tensor?)ACACBSCASCB outlier_colsbiasreturnc CsXg|jdd|jdR}tj||j|jd}tj}|j|tj d} || fS)Nrdevicedtyper) shapetorchemptyrrlibraryget_ctxnew_dynamic_size new_emptyint64) rrrr r r r shapeCoutsubArN/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/bitsandbytes/_ops.py_s  r zbitsandbytes::int8_scaled_mmzm(Tensor A, Tensor B, Tensor row_stats, Tensor col_stats, Tensor? bias=None, ScalarType? dtype=None) -> TensorB row_stats col_statsrcCs6g|jdd|jdR}tj||j|ptjdS)Nrrr)rrrrfloat16)rr!r"r#r rrrrrr 4s z bitsandbytes::int8_linear_matmulz(Tensor A, Tensor B) -> TensorcCsbt|jtjkddt|jtjkddg|jdd|jdR}tj||jtjdS)NcSdSNzA must be int8rrrrrI_..cSr%NzB must be int8rrrrrr'Jr(rrr)r_checkrint8rrrint32)rr!rrrrr Gsz$bitsandbytes::int8_linear_matmul.outz'(Tensor A, Tensor B, Tensor! out) -> ()rcsgjdd|jdRtjtjkddt|jtjkddtjkfddtjjkfddtjtjkfdddS) NrrcSr%r&rrrrrr'[r(r)cSr%r*rrrrrr'\r(cddjSNExpected out.shape == , got rr)rrrrr']cdjdjSNzExpected out.device == r1rrrrrrr'^c djS)Nz!Expected out.dtype == int32, got rr)rrrr'_ )rrr+rr,rr-)rr!rr)rrrrr Ws  z#bitsandbytes::int8_vectorwise_quantz<(Tensor A, float threshold=0.0) -> (Tensor, Tensor, Tensor?)cCsntj|j|jtjd}tjt|jdd|jtjd}|dkr%||dfStj }|||j |tj dfS)Nrrr;r) rrrrr,rfloat32rrrrr)r thresholdout_rowr"r rrrr hs " z%bitsandbytes::int8_vectorwise_dequantz"(Tensor A, Tensor stats) -> TensorstatscCs(t|jtjkddtj|tjdS)NcSr%r&rrrrrr'zr(r)r)rr+rr, empty_liker<rr?rrrr xsdefaultcCs||dddS)Nrg@ ?)viewrArrrr szbitsandbytes::int8_mm_dequantzc(Tensor A, Tensor row_stats, Tensor col_stats, ScalarType? dtype=None, Tensor? bias=None) -> TensorcCs,t|jtjkddtj||ptjdS)NcSr%)NzA must be int32rrrrrr'r(r)r)rr+rr-r@r$)rr"r#rr rrrr szbitsandbytes::int8_double_quantzL(Tensor A, float threshold=0.0) -> (Tensor, Tensor, Tensor, Tensor, Tensor?)cCstj|tjd}tj|tjd}tjt|jdd|jtjd}tj|jd|jtjd}tj }|j |tj d}|||||fS)Nrrr) rr@r,rrrrr<rrrrr)rr=r>out_colr"r# outlier_nr rrrr s"zbitsandbytes::dequantize_4bitza(Tensor A, Tensor absmax, int blocksize, str quant_type, int[] shape, ScalarType dtype) -> Tensorabsmax blocksize quant_typercCst|tj|||jdS)N)rr)r_check_is_sizerr)rrGrHrIrrrrrr s z!bitsandbytes::dequantize_4bit.outzj(Tensor A, Tensor absmax, int blocksize, str quant_type, int[] shape, ScalarType dtype, Tensor! out) -> ()csdt|tjkfddtjjkfddtjkfdddS)Ncr.r/r2r)rrrrr'r3r)cr4r5r6rr7rrr'r8cddjSNzExpected out.dtype == r1rrrrrrr'r3)rrJr+rrr)rrGrHrIrrrr)rrrrrr s  zbitsandbytes::quantize_4bitzW(Tensor A, int blocksize, str quant_type, ScalarType quant_storage) -> (Tensor, Tensor) quant_storagecCs`t||}||  }tj|f|jtjd}tj|d|jddf|j|d}||fS)NrrC)rrJnumelrrr<itemsize)rrHrIrNnblocksrGrrrrr s  $z"bitsandbytes::dequantize_blockwisezQ(Tensor A, Tensor absmax, Tensor code, int blocksize, ScalarType dtype) -> Tensorcodecs4t|tjtjkfddtj|dS)Ncr9NzA must be uint8, got rrrrrr'r:r)r)rrJr+ruint8r@)rrGrTrHrrrVrr s z&bitsandbytes::dequantize_blockwise.outzZ(Tensor A, Tensor absmax, Tensor code, int blocksize, ScalarType dtype, Tensor! out) -> ()cst|tjtjkfddtjjkfddtjjkfddtjkfdddS)Ncr9rUrrrVrrr'r:r)cr4r/r2rr7rrr'r8cr4r5r6rr7rrr'r8crKrLrrrMrrr'r3)rrJr+rrWrr)rrGrTrHrrr)rrrrr s  z bitsandbytes::quantize_blockwisez:(Tensor A, Tensor code, int blocksize) -> (Tensor, Tensor)cCsLt||}||  }tj|f|jtjd}tj|tjd}||fS)Nrr)rrJrPrrr<r@rW)rrTrHrRrSrGrrrrr s  zbitsandbytes::gemv_4bitzW(Tensor A, Tensor B, int[] shapeB, Tensor absmax, Tensor code, int blocksize) -> TensorshapeBcst|tdkfddtjtjtjtjfvfddtjtj tjtjtjfvfddgj dd|dR}tj |j jdS)Nrcr9Nz5A must be a vector with leading dimensions of 1, got r2rrVrrr'r:r)cr9Nz-A must be float16, bfloat16, or float32, got rrrVrrr' r:cr9NzNB must be backed by storage of type uint8, bfloat16, float16, or float32, got rrr!rrr'$r:rr) rrJr+rPsizerr$bfloat16r<rWrrr)rr!rXrGrTrHrr)rr!rr s "  zbitsandbytes::gemv_4bit.outz`(Tensor A, Tensor B, int[] shapeB, Tensor absmax, Tensor code, int blocksize, Tensor! out) -> ()cst|tdkfddtjtjtjtjfvfddtjtj tjtjtjfvfddtj gj dddRkfddtj j kfddtjjkfd ddS) Nrcr9rYr2rrVrrr';r:r)cr9rZrrrVrrr'>r:cr9r[rrr\rrr'Br:rcs*dgjdddRdjS)Nr0rrr1r2r)rrrXrrr'Fs*cr4r5r6rr7rrr'Hr8cr4rLrrr7rrr'Ir8) rrJr+rPr]rr$r^r<rWrr)rr!rXrGrTrHrr)rr!rrXrr 0s "   ")NN)r;)collections.abcrmathrtypingrr_IS_TORCH_GTE_24hasattrrrregister_kernel impl_abstractimpldefineTensortupler rintstrrrrrs                   .  ,