o oiŤ,@s ddlZddlZddlmZddlmZmZmZm Z m Z m Z ddl Z ddlZddlmZddlmZddlmZmZddlmZiZerejr ejejejfejejfejejfej ej!ej"fej#ej$fejejejfej%ej&ej'fd Z(ej)ej*fej+ej,fej-ej.fej/ej0fej)ej*fej+ej,fd Z1ej2ej3ej4fej5ej6ej7fej8ej9ej:fej;ejej?ej@fejAejBejCfd ZDGd d d ZEGdddZFGdddZGiZHdeHejI<deHejJ<deHejK<deHejL<deHejM<ejNdddZOejPQdkr dejfddZRn ddlSZSdejfddZRejIeOdddZTdddZUdd!d"ZVdd#d$ZWed%eXd&dd'd(ZYed%eXd&dd)d*ZZdd,d-Z[dd/d0Z\dd2d3Z]dd5d6Z^dd7d8Z_ed9eXd&d:d;Z`dZaed?eXd&d@edAejPjbfdBdCZcd@edAejdfdDdEZedFe edAe ejdfdGdHZfed?eXd&dIdJZged?eXd&dKdLZhedMeXd&ddNdOZiedMeXd&ddQdRZjedMeXd& P    ddSdTZk  U VddFedWe ejdXeldAefdYdZZmGd[d\d\Zn    ] ddFejd^e ejd_e ejdWe ejdAe ejenff d`daZo     ] ddFejdbe end_e ejd^e ejdWe ejdcepdAejfdddeZqddgdhZrdddfdejLfdFejd_e ejdWe ejfdidjZsdddfdejLfdFejd_e ejdWe ejfdkdlZtdddfddmejLfdFejd_e ejdWe ejdAe ejenffdndoZu    fddFejdbe end_e ejdWe ejdcepdAejf dpdqZv    fddFejdbe end_e ejdWe ejdcepdAejf drdsZw    f md dFejdbe end_e ejdWe ejdcepdAejf dtduZxed?eXd&  d dFed^e ejdWe ejdAe ee eefffdvdwZyed?eXd&    d dFedxe e eefd_e ejd^e ejdWe ejdAef dydzZzed?eXd&ddFed^edWe ejdAefd{d|Z{ed?eXd&ddFed^edWe ejdAefd}d~Z|        d de}dedededeldeldepdelde ejdeldeldeldeldelde ejdeldAdf"ddZ~edeXd&   d de}dededede ejdeldeldeldepdeldede ejdede ejdede ejdeldelde ejdeldAdf*ddZ  dde}dededede ejdeldeldeldeldeldepdeldede ejdede ejdeldeldAdf&ddZed?eXd&ddededepdepfddZdedededefddZejMfddZ    ddFededWe ejfddZ   ddFededWe ejfddZ   ddFededWe ejfddZedeXd&ddejfdFejdejde eje}fde eje}fdWe ejde e eje}ff ddZdejfdFejdejdWe ejfddZ  d dFejdejdejdWe ejde ejf ddZedeXd&    d dFejdbe e eje}fdejdejdWe ejde ejf ddÄZ    ddFejde ejde ejde ejdAe ejeje ejff ddƄZddFejfddȄZGddʄdʃZGdd̄d̃ZGdd΄d΃ZddЄZdd҄ZejfddԄZedeXd&     ddFejde ejde ejde ejde ejdAe ejejejeje eff ddلZ     ddFejde ejde ejde ejde ejf ddۄZdFejdejfddބZddFejfddZedMeXd&dddZ dde eejfdejdWe ejfddZd ddZdZedeXd&dddZedeXd&dddZedeXd&ejdfddZed?eXd&ejfddZed?eXd&ddZed?eXd&ddZdS(N)prod)AnyDictIterableOptionalTupleUnion)Tensor) deprecated)pack_dict_to_tensorunpack_tensor_to_dict)lib)adammomentumrmsproplionadagradlambademamix)rrrrrlars)rrrrrrc@s6eZdZdZddZddZeddZd d d ZdS) GlobalPageManagerNcCtdNzCall get_instance() instead RuntimeErrorselfrK/home/ubuntu/.local/lib/python3.10/site-packages/bitsandbytes/functional.py__init__zzGlobalPageManager.__init__cCs g|_dSN) paged_tensorsrrrr initialize} zGlobalPageManager.initializecC&|jdur|||_|j|jSr" _instance__new__r$clsrrr get_instance   zGlobalPageManager.get_instanceFcCs$|jdddD]}t||qdS)N)r#prefetch_tensor)rto_cputrrr prefetch_alls zGlobalPageManager.prefetch_allF) __name__ __module__ __qualname__r(r r$ classmethodr,r2rrrrrws rc@s4eZdZdZddZddZeddZdd ZdS) CUBLAS_ContextNcCrrrrrrrr r!zCUBLAS_Context.__init__cCs i|_dSr")contextrrrrr$r%zCUBLAS_Context.initializecCr&r"r'r*rrrr,r-zCUBLAS_Context.get_instancecCsP|j|jvr"tj}tj|tt |j|j<tj||j|jSr") indexr9torchcudacurrent_device set_devicectc_void_pr get_context)rdevice prev_devicerrrrAs     zCUBLAS_Context.get_context) r4r5r6r(r r$r7r,rArrrrr8s  r8c@s,eZdZdZddZddZeddZdS)Cusparse_ContextNcCrrrrrrrr r!zCusparse_Context.__init__cCstt|_dSr")r?r@r get_cusparser9rrrrr$szCusparse_Context.initializecCr&r"r'r*rrrr,r-zCusparse_Context.get_instance)r4r5r6r(r r$r7r,rrrrrDs rDr<)r:acCs tj|Sr")r;r< device_ofrHrrr_cuda_device_ofs rKcCstSr") contextlib nullcontextrJrrrrKr!dtyperBcGspt|t|}tt|}t|ttj}t j j ||d}t j ||t|d|}d|_|j|_|S)Nshape)rOcountT) dtype2bytesrrcget_managed_ptrr?c_size_tcastPOINTERc_intnp ctypeslibas_arrayr; frombufferviewis_pagedr: page_deviceid)rOrBrQ num_bytescuda_ptrc_ptr new_arrayoutrrr get_pagedsreFcCsR|jsJd|r d}n|j}t|j|}tt|t |t |dS)Nz%Only paged tensors can be prefetched!r.) r^r_rSrOnumelr cprefetchget_ptrr?rUc_int32)Ar0deviceidr`rrrr/s "r/TcCsd}|jtjkrttd|dd}t|}n|jtjkr-ttd|dd}t|}|dur8t d|t|dd}|rN|rNt ||durNt ||t |t ||t | |jsd|jrktjdSdS)Nc_fp32_uint8zFunction not implemented: is_managedF)rOr;float32getattrrr?c_floatuint8c_uint8NotImplementedErrorr/rhc_int64rfr^r< synchronize) func_namerjBvalueprefetchfunccvaluerorrrelementwise_funcs$       r~cCstd|d|dS)Nfillr~)rjrzrBr{rrrrsrz-Function will be removed in a future release.)categorycCstd|dddS)Narangerr)rjrBrrrrrcCstd||ddS)N_mulrr)rjryrBrrrr rrcCs|rdnd}d|}|s|dkr|sd|nd|d}t|d|}d|}|dkr/|S|d}t|d|dg|||dS) NrGrr ?r)r;linspacerfr tolist)signed total_bitsadd_zerosign total_valuesvaluesgaplrrrcreate_linear_maps    0r+ew?c Cszddlm}Wnty}ztd|d}~ww|rC|t|dddd}dgd}|t|dddd }n&|t|dddd}dgd }|t|dddd }|||}t|}|j }|| }| d ksJ|S) Nr)normzZScipy is required for `create_normal_map`. Install `bitsandbytes` with the `[test]` extra.g? r.rr) scipy.statsr ImportErrorppfr;rrr sortrmaxrf) offsetuse_extra_valueriev1v2v3vrrrrcreate_normal_map#s.  $  "    rcCs|}|}|rdnd}||||ksJg}g}ttd|| d||dD] \} } |d| q)g} ttjddg|d} d|d} td|D]I}| D]D}|dkrZdnd}tt|D]\} }||d| d 7}qb|dkr~|d| }n |d|| d }| ||r| | qRqNt| d|ksJ| |dkrdt| }t|D]} | dq| t | }|| }|S)Nr rrG)repeatrr) enumeraterangeappendlist itertoolsproductlenrr;r r)r exponent_bitsprecision_bitsrephas_signevaluespvaluesivalrlstbiasevalue bit_patternrzpvalrcoderrrcreate_fp8_map@sD *        rc Csg}||rdnd}d||d}t|D]U}t|r%d|||dn d|||dd}tjdd|tjd}|dd|ddd} |d|d || 7}|rk|d|d | | 7}q|d krtjdd|dtjd}|dd|ddd} |d|d || 7}|r|d|d | | 7}|d |d t|d|ksJd t|} t| D]}|d q|tj |tjdS) a+ Creates the dynamic quantiztion map. The dynamic data type is made up of a dynamic exponent and fraction. As the exponent increase from 0 to -7 the number of bits available for the fraction shrinks. This is a generalization of the dynamic type where a certain number of the bits and be reserved for the linear quantization region (the fraction). n determines the maximum number of exponent bits. For more details see (8-Bit Approximations for Parallelism in Deep Learning)[https://arxiv.org/abs/1511.04561] r rGg?rONr.@ rrr) rintr;rrprrrrtensor) rmax_exponent_bitsrdata non_sign_bitsadditional_itemsrfraction_items boundariesmeansrrrrcreate_dynamic_mapks:        rcCsnt|d|dd}|}|ddt|}t|D]}|dq|t|}||}|S)NrGr ) num_quantilesrr) estimate_quantilesrrrrrr absr)rjrqrrrrrcreate_quantile_maps    rzDThis function is deprecated and will be removed in a future version.cCsdS)Nrowrrrrrget_special_format_strsrtensorscCsd}t}|D]}|durt|dds||jM}||jjq|s.tddd|Dt|dkr@td d d|D|S) apVerifies that the input tensors are all on the same device. An input tensor may also be marked as `paged`, in which case the device placement is ignored. Args: tensors (`Iterable[Optional[torch.Tensor]]`): A list of tensors to verify. Raises: `RuntimeError`: Raised when the verification fails. Returns: `Literal[True]` TNr^FzZAll input tensors need to be on the same GPU, but found some tensors to not be on a GPU: cSg|]}|j|jfqSrrQrB.0r1rrr zis_on_gpu..r zcInput tensors need to be on the same GPU, but found the following tensor and device combinations: cSrrrrrrrrr)setrqis_cudaaddrBr:rr)ron_gpugpu_idsr1rrr is_on_gpus   rzDThis function is deprecated and will be removed in a future release.rreturncCstj|jSr")r;r<current_streamrBrrrrget_tensor_streamsrcCsttj|jjSr")r?r@r;_C_cuda_getCurrentRawStreamrBr:rrrr_get_tensor_streamsrrjcCs|durdSt|S)zGets the memory address of the first element of a tenso Args: A (`Optional[Tensor]`): A PyTorch tensor. Returns: `Optional[ct.c_void_p]`: A pointer to the underlying tensor data. N)r?r@data_ptr)rjrrrrhs rhcCstj}tj||Sr")r;r<r=r>)rBrCrrrpre_calls  rcCstj|dSr")r;r<r>)rCrrr post_callsrzkThe layout transformation operations will be removed in a future release. Please use row-major layout only.c Csnd|tjkrdndd|d|d|rdnd}tt|s2t|td|d |d |d |tt|S) N ctransform_r __to_r1nz"Transform function not supported:  to z for data type z and transpose=)r;int8hasattrrprint ValueErrorrq)rOorderAorderOut transposenamerrrget_transform_funcs2  rrc CsDtj}t|}|dkr|d}n |dkr|d|d}|d} ||f} |r5|} | }| } |ddd|f} |dks=|dkrF||||d| fS|d kr]d | d d } ||| f||d| fS|d kr|d | d d } d |dd }||| f||d| fS|dkrd | d d } d |d d }||| f||d| fStd|)NrGrr r.rcolrNcol32r col_turingrr col_amperezTo_order not supported: )r;zerosrru) rQrOrBto_order from_orderr init_funcdimsrowscolsstatetmprrrget_transform_buffers6 r c s(|dur |j|f}n|d}|dur#t|d|j|j||d\}}n|d|f}t|j|||}|dtdkrJtd} td} n2|durjt} tfdd|D} t| | } t| } ntdd} td} t |j} || t |t || | ||fS)Nr rrGcsg|]}|qSrr)rrrPrrrWsz$nvidia_transform..) rQr rOrBrrr?rirr8r,rArh) rjrrrdrrld new_stater|dim1dim2rptrrrPrnvidia_transform:s* "   r`?rrdrc CsB|dkrtd|d|dkrtd||dkr)|dkr)dd|}|dur7tjd tj|jd }t||gt|j}|jtjkr]t t |t |t |t |n#|jtjkrxt t |t |t |t |ntd |jt||dkrtd|}td d ||j}||}|S)a Estimates 256 equidistant quantiles on the input tensor eCDF. Uses SRAM-Quantiles algorithm to quickly estimate 256 equidistant quantiles via the eCDF of the input tensor `A`. This is a fast but approximate algorithm and the extreme quantiles close to 0 and 1 have high variance / large estimation errors. These large errors can be avoided by using the offset variable which trims the distribution. The default offset value of 1/512 ensures minimum entropy encoding -- it trims 1/512 = 0.2% from each side of the distrivution. An offset value of 0.01 to 0.02 usually has a much lower error but is not a minimum entropy encoding. Given an offset of 0.02 equidistance points in the range [0.02, 0.98] are used for the quantiles. Parameters ---------- A : torch.Tensor The input tensor. Any shape. out : torch.Tensor Tensor with the 256 estimated quantiles. offset : float The offset for the first and last quantile from 0 and 1. Default: 1/(2*num_quantiles) num_quantiles : int The number of equally spaced quantiles. Returns ------- torch.Tensor: The 256 quantiles in float32 datatype. rzQQuantile estimation needs at least 256 values in the Tensor, but Tensor had only z values.zgCurrently only a maximum of 256 equally spaced quantiles are supported, but the argument num_quantiles=rr rGN)rrNzNot supported data type r)rfrur;rrprBrrrOrcestimate_quantiles_fp32rhr?rrrXfloat16cestimate_quantiles_fp16rroundrlongto)rjrdrrrBstepidxrrrrds2 "    * * rc@seZdZdZdZddeDZgdZ       dddZd d Ze d e e e fd e jd dfddZdddZddZddZdS) QuantStatezWcontainer for quantization state components to work with Params4bit and similar classesfp4nf4cCsg|]}d|qS)bitsandbytes__r)rxrrrrszQuantState.) absmax quant_map nested_absmaxnested_quant_map quant_state quant_type blocksizerOrQnested_blocksize nested_dtype nested_offsetNc Cs>||_||_||_||_||_||_||_||_|du|_dSr") r!rQrrOr'r&rstate2nested) rr!rQrr'r&rOrr+rrrr s zQuantState.__init__cCsR|jr|j|j|j|j|j|jg|jg}||S|j|j|j|jd|jg}||S)a$ ensures compatibility with older quant state scheme with nested lists. assumes the following layout: state = [qabsmax, input_shape, A.dtype, blocksize, [offset, state2], quant_type] state2 = [absmax, input_shape, A.dtype, blocksize, None, quant_type] N)r,r!rQrOr'rr+r&)rr list_reprrrr __get_item__s  zQuantState.__get_item__qs_dictrBrc Cspdd|D}t|sd|vrtdt|dks'|ddd|jvr3td |jd |dt|dkrG|d}|t||d d |D}t|  |j s\Jd |vrt t|d|}||d ||d|d|tt |dd}nd\}}||d|d||d|d|tt |d|ddurt |dnd||d}|S)aO unpacks components of state_dict into QuantState where necessary, convert into strings, torch.dtype, ints, etc. qs_dict: based on state_dict, with only relevant keys, striped of prefixes. item with key `quant_state.bitsandbytes__[nf4/fp4]` may contain minor and non-tensor quant state items. cSs(g|]\}}d|vrt|tjr|qS)r% isinstancer;r rkrrrrrs(z(QuantState.from_dict..r&zs z(QuantState.from_dict..r#r*r(r$r))r!r'rrONNr!r'r"rOrQN)r&r!r'rrOrQrr+)itemsrrr5valid_qs_type_keysupdater poprkeysissubset valid_qs_keysr;rfloatrrqSize)r+r/rBqs_key first_qs_keyrr+r%rrr from_dicts@ $        zQuantState.from_dictFcCs|j|j|j|jt|jdt|jd}|j r6| |j j|j j|j j t|j jd|j d|s:|Sdd|D}dd|D}t||d|j<|S)z returns dict of tensors and strings to use in serialization via _save_to_state_dict() param: packed -- returns dict[str, torch.Tensor] for state_dict fit for safetensors saving ztorch.)r&r!r'r"rOrQ)r#r(r$r)r*cSs"i|] \}}t|tjr||qSrr0r2rrrr61"z&QuantState.as_dict..cSs"i|] \}}t|tjs||qSrr0r2rrrr62rDzquant_state.bitsandbytes__)r&r!r'rstrrOstriptuplerQr,r:r+cloneritemr8r )rpackedr/qs_packed_dictnon_tensor_dictrrras_dicts,  zQuantState.as_dictcCs\|j||_|j||_|jr,|j||_|jj||j_|jj||j_dSdSr")rrr!r,rr+)rrBrrrr6sz QuantState.tocCst|tsdStj|j|jddo^|j|jko^tj|j|jddo^|j|jko^|j|jko^|j |j ko^|j durC|j durC|j |j kn|j |j uo^|j durY|j durY|j |j kS|j |j uS)NFgư>)atol) r1rr;allcloser!rQrrOr'r&rr+)rotherrrr__eq__?s,        zQuantState.__eq__)NNNNNNNr3)r4r5r6__doc__valid_quant_typesr9r>r r.r7rrErr;rBrCrMrrQrrrrrs& " 2 rrr!c Cs|durdtvrt|jtd<td}|dur.|}||  }tj|f|jtjd}|dur:tj|tj d}|jj dkr|dvsFJ||j}t |||gt |Lt |t |t |t |t|t|f}|jtjkr|tj|n |jtjkrtj|n|jtjkrtj|ntd|jWdn1swYn|}tt |t |t |t |t|t||r|} || 8}t||dd \} } t| |||j| | d } || fSt||||jd } || fS) aWQuantize a tensor in blocks of values. The input tensor is quantized by dividing it into blocks of `blocksize` values. The the absolute maximum value within these blocks is calculated for scaling the non-linear quantization. Args: A (`torch.Tensor`): The input tensor. Supports `float16`, `bfloat16`, or `float32` datatypes. code (`torch.Tensor`, *optional*): A mapping describing the low-bit data type. Defaults to a signed 8-bit dynamic type. For more details, see (8-Bit Approximations for Parallelism in Deep Learning)[https://arxiv.org/abs/1511.04561]. absmax (`torch.Tensor`, *optional*): A tensor to use to store the absmax values. out (`torch.Tensor`, *optional*): A tensor to use to store the result. blocksize (`int`, *optional*): The size of the blocks. Defaults to 4096. Valid values are 64, 128, 256, 512, 1024, 2048, and 4096. nested (`bool`, *optional*): Whether to additionally quantize the absmax values. Defaults to False. Raises: ValueError: Raised when the input data type is not supported. Returns: `Tuple[torch.Tensor, QuantState]`: A tuple containing the quantization results. - `torch.Tensor`: The quantized tensor. - [`QuantState`]: The state object used to undo the quantization. NdynamicrBrOrcpurTr@?Blockwise quantization only supports 16/32-bit floats, but got F)r'r,)r!rr'rOrr+r!rr'rO) name2qmaprrrBrfr;rrp zeros_likerstyperrKrhr?rirXrOrrcquantize_blockwise_fp16bfloat16cquantize_blockwise_bf16cquantize_blockwise_fp32rrWcquantize_blockwise_cpu_fp32 c_longlongmeanquantize_blockwiser) rjrr!rdr'r,rblocksargsrqabsmaxr+r%rrrrjWsp#              rjr%r'c Cs|dus |dus J|dur#|dur#dtvrt|jtd<td}|dur0t|||tjd}|j}|jrLt |j|j }||j 7}|j tjkrL| }|dur[tj|j|j |jd}|jjdkr|j|j}|jdvrvtd|jdt|||gt|bt|jt|t|t|t|jt|t|f}|j tjkrtj|n |j tjkrtj|n|j tjkrtj |ntd |j Wd|SWd|SWd|S1swY|S|j!}t"t|t|t|t|t#|jt#||S) aDequantize a tensor in blocks of values. The input tensor is dequantized by dividing it into blocks of `blocksize` values. The the absolute maximum value within these blocks is used for scaling the non-linear dequantization. Args: A (`torch.Tensor`): The quantized input tensor. quant_state ([`QuantState`], *optional*): The quantization state as returned by [`quantize_blockwise`]. Required if `absmax` is not provided. absmax (`torch.Tensor`, *optional*): A tensor containing the scaling values. Required if `quant_state` is not provided and ignored otherwise. code (`torch.Tensor`, *optional*): A mapping describing the low-bit data type. Defaults to a signed 8-bit dynamic type. For more details, see (8-Bit Approximations for Parallelism in Deep Learning)[https://arxiv.org/abs/1511.04561]. Ignored when `quant_state` is provided. out (`torch.Tensor`, *optional*): A tensor to use to store the result. blocksize (`int`, *optional*): The size of the blocks. Defaults to 4096. Valid values are 64, 128, 256, 512, 1024, 2048, and 4096. Ignored when `quant_state` is provided. Raises: ValueError: Raised when the input data type is not supported. Returns: `torch.Tensor`: The dequantized tensor. The datatype is indicated by `quant_state.dtype` and defaults to `torch.float32`. NrUr_rNrWrXzThe blocksize of zJ is not supported. Supported values: [4096, 2048, 1024, 512, 256, 128, 64]r^)$r`rrrBrr;rpr!r,dequantize_blockwiser+rrOr?emptyrQrbrr'rrrKrhr?rXrfrrrcdequantize_blockwise_fp16rdcdequantize_blockwise_bf16cdequantize_blockwise_fp32rWcdequantize_blockwise_cpu_fp32rh)rjr%r!rrdr'r,rlrrrrnsv)                    rnr]cCs|durd}d}|dkr gd}n(|dkrgd}n|dkr$gd}n|dkr:|d kr6gd ddd }ntd |durFtd |dtj||d}|||dks^J|S)Nr<r)rg 6GgfgTFٿgI4ҿg০ǿg Org__?g`\?g?g@g?g4?g`?g`v"?rr)rg?g @g(@g@g@rg@rgg g(ggggint4)rrrFrrGr rrr.iiaf4r])rg|8geg:Kڞ׿gH2퓊cпg}Yu-ÿgQ #(DrgF?g`_?g 0E?gL_߹E?gƶ=?ga@?gкv -?rr.z94-bit AbnormalFloats currently only support blocksize 64.z Typename z not supportedrB)rur;rdiv_rrrf)typenamerBr'rrrr get_4bit_type%s,   rcCt|||||d|SNr quantize_4bitrjr!rdr'compress_statistics quant_storagerrr quantize_fp4wrcCrNrrrrrr quantize_nf4rrrc Cs|jjdkrtd|jj|dvrtd|d|}|j}|dur7||  } tj| f|jtjd}|durPt|d} tj|d | d f||jd }|d vsVJt |||gt |fdt |t |t |t |t |f} |jtjkr|d krtj| n:tj| n4|jtjkr|d krtj| n$tj| n|jtjkr|d krtj| ntj| ntd |jWdn1swYt||jd} |r|} || 8}t|dd\}}~t|||j|| || |d}||fSt|||j|| |d}||fS)aQuantize tensor A in blocks of 4-bit values. Quantizes tensor A by dividing it into blocks which are independently quantized. Args: A (`torch.Tensor`): The input tensor. Supports `float16`, `bfloat16`, or `float32` datatypes. absmax (`torch.Tensor`, *optional*): A tensor to use to store the absmax values. out (`torch.Tensor`, *optional*): A tensor to use to store the result. blocksize (`int`, *optional*): The size of the blocks. Defaults to 64. Valid values are 64, 128, 256, 512, 1024, 2048, and 4096. compress_statistics (`bool`, *optional*): Whether to additionally quantize the absmax values. Defaults to False. quant_type (`str`, *optional*): The data type to use: `nf4` or `fp4`. Defaults to `fp4`. quant_storage (`torch.dtype`, *optional*): The dtype of the tensor used to store the result. Defaults to `torch.uint8`. Raises: ValueError: Raised when the input data type is not supported. Returns: Tuple[`torch.Tensor`, `QuantState`]: A tuple containing the quantization results. - `torch.Tensor`: The quantized tensor with packed 4-bit values. - [`QuantState`]: The state object used to undo the quantization. r<z0Device type not supported for FP4 quantization: r4-bit quantization data type  is not implemented.NrVrGr rNrXrr^r{r)r')r!rQrOr'rr&rr+)r!rQrOr'rr&)rBrbrurfrQr;rrprSrrKrhr?rirXrOrdrcquantize_blockwise_bf16_fp4cquantize_blockwise_bf16_nf4rcquantize_blockwise_fp16_fp4cquantize_blockwise_fp16_nf4cquantize_blockwise_fp32_fp4cquantize_blockwise_fp32_nf4rrrirjr)rjr!rdr'rr&rr input_shaperkmodrlrrrmr+rrrrrs~ !              rcCt|||||dSrdequantize_4bitrjr%r!rdr'rrrdequantize_fp4rcCrrrrrrrdequantize_nf4rrc Cs|dvr td|d|dvrtd|d|dur2|dur$|dus&Jt||j|j||d}n|j}|jrNt|j|j}||j 7}|jt j krN| }|dur]t j |j|j|jd }|}t|||gt|}t|kdt|t|t|t|jt||f}|jt jkr|jd krtj|n    "rzyThis function is deprecated and will be removed in a future release. Please use optimizer_update_8bit_blockwise instead. qmap1qmap2max1max2new_max1new_max2cCsd}|dkrt|j}t|j}t|||||| | | | ||g |jtjkr|jtj krt |dt |t |t |t |t |t |t |t |t |t |t |t | t | t | t | t | t |t |t |t |t |nk|jtjkr|jtj krt |dt |t |t |t |t |t |t |t |t |t |t |t | t | t | t | t | t |t |t |t |t |n td|jd|jt|dS)a Performs an inplace Adam update. Universal Adam update for 32/8-bit state and 32/16-bit gradients/weights. Uses AdamW formulation if weight decay > 0.0. Parameters ---------- optimizer_name : str The name of the optimizer. Choices {adam, momentum} g : torch.Tensor Gradient tensor. p : torch.Tensor Parameter tensor. state1 : torch.Tensor Adam state 1. state2 : torch.Tensor Adam state 2. beta1 : float Adam beta1. beta2 : float Adam beta2. eps : float Adam epsilon. weight_decay : float Weight decay. step : int Current optimizer step. lr : float The learning rate. qmap1 : torch.Tensor Quantization map for first Adam state. qmap2 : torch.Tensor Quantization map for second Adam state. max1 : torch.Tensor Max value for first Adam state update. max2 : torch.Tensor Max value for second Adam state update. new_max1 : torch.Tensor Max value for the next Adam update of the first state. new_max2 : torch.Tensor Max value for the next Adam update of the second state. gnorm_scale : float The factor to rescale the gradient to the max clip value. unorm_vec : torch.Tensor The tensor for the update norm. max_unorm : float The maximum update norm relative to the weight norm. rrr rrN)r;rrr?rrBrrOrprsstr2optimizer8bitrhr?rrrirfrrr)rrrrr+rrrrrrrrrrrrrrrrrCrrroptimizer_update_8bitGsrM      rabsmax1absmax2cCsvd}|jtjkr|jtjkrt|d}n:|jtjkr(|jtjkr(t|d}n'|jtjkrC|jtjkrCtt|dkrCt|d}n td|jd|jt ||||| | ||gt |R|t |t |t |t |t |t |t |t |t | t | t | t | t | t |t |t |t |t |t |WddS1swYdS)Nrr rrGrr)rOr;rprsstr2optimizer8bit_blockwiserrdrrrrKrhr?rrrirrf)rrrrr+rrrrrrrrrrrrrrrrrroptimizer_update_8bit_blockwisesH    "rgrad gnorm_vec percentilec Cst|j}t||g|jtjkr&tt|t|t |t | n$|jtj krAt t|t|t |t | n td|jdt|t||d}t|\}}t||}d} ||kro||} ||| fS)aApplies percentile clipping grad: torch.Tensor The gradient tensor. gnorm_vec: torch.Tensor Vector of gradient norms. 100 elements expected. step: int The current optimization steps (number of past gradient norms). zGradient type z not supported!dr)rrBrrOr;rprcpercentile_clipping_g32rhr?rirfrcpercentile_clipping_g16rrsqrtr) rrrrrC current_gnormvalsr clip_valuerrrrpercentile_clippings2      r histogramindex1index2sourcecCst|jdks J|jtjksJ|jtjksJ|jtjks!J|jtjks)J|jjdks1J|jjdks9J|jjdksAJ|jjdksIJt |jd}t | }t ||||gt t|t|t|t|||dS)NrGr<r)rrQrOr;rpint32rBrbr?rirfrrchistogram_scatter_add_2drh)rrrrmaxdim1rrrrhistogram_scatter_add_2d<s(rc Cstjs tj|j|ks|j|kr td|jd|j|j}|j}|}|} d} t|dkr|t|dkr||sI| sI|jd|jdkrId} n|rZ| sZ|jd|jdkrZd} n|rk| rk|jd|jdkrkd} n|s{| r{|jd|jdkr{d} nt|dkrt|dkr|s| s|jd|jdkrd} n|r| s|jd|jdkrd} ny|r| r|jd|jdkrd} nh|s| r|jd|jdkrd} nWt|dkr#t|dkr#|s| s|jd|jdkrd} n8|r| s|jd|jdkrd} n'|r| r|jd|jdkrd} n|s#| r#|jd|jdkr#d} |durc|j} | sbt|dkrbt|dkrb| d|dkrb| d|dkrb|d|dkrb|d|dkrbd} nt|dkrt|dkr|s| s|d|df} n|r| r|d|df} n|r| s|d|df} n|s| r|d|df} nt|dkrt|dkr|s| s|d|d|df} n|r| r|d|d|df} ny|r| s|d|d|df} ng|s| r|d|d|df} nUt|dkrXt|dkrX|s#| s#|d|d|df} n5|r5| r5|d|d|df} n#|rG| sG|d|d|df} n|sX| rX|d|d|df} | sltd |d |d |d | d | S) Nz3Expected torch.int8 input tensors A and B, but got  and TrGr rFrz?Tensor dimensions incorrect for matrix mulitiplication: A x B:  x z with transpose for A x B: r4) r;r<is_initializedinitrO TypeErrorrQrr) rjryrd transposed_A transposed_B expected_typesAsBtAtBcorrectsoutrrr check_matmulNs  "" "H            rrycCs|durtd||jdkrtd|j}|d}|j}|jr.t|j|j}||j7}|dur\t|jdkrMt j |jd|jd|f|j |j d}nt j |jd|f|j |j d}d} |d} |d} |d} |d} |jddd}t |||||jgt| } t| } t| } t| } t|}t| } t|}t||j t jt jt jt jfvr5|j t jkrt| | | t|t|t|t|jt|| || t|j| n`|j t jkrt| | | t|t|t|t|jt|| || t|j| n@|j t jkr-t| | | t|t|t|t|jt|| || t|j| n td |j td |j Wd|SWd|SWd|S1sYwY|S) NzIstate cannot be None. gemv_4bit() requires the state from quantize_4bit()r.zcDimensions of A are invalid. Must be a vector with the leading dimensions of "1", e.g. [1, 1, 2048]rrr sizerOrBrGz%Matmul not implemented for data type )rrfrQr!r,rnr+rrr;rorOrBrrr?rirrKrsrdrrprcgemm_4bit_inference_naive_fp16rhr'cgemm_4bit_inference_naive_bf16cgemm_4bit_inference_naive_fp32ru)rjryrdrrrBshapeboutr!rmr3ldaldcldbrrrr gemv_4bits  (            8 8 88rcCsvt|||||}|durtj|tj|jd}t|jdkr>t|jdkr>|jd|jdkr>|jd|jdkr>t|||S|j}|j}|rUt|dkrU|d|df}n|rht|dkrh|d|d|df}|ryt|dkry|d|df}n|rt|dkr|d|d|df}t|dkr4|d|jdkrd}n |d|jdkrd}t|jdkr|d|jdkrd}n)|d|jdkrd}n|d|jdkrd}n |d|jdkrd}t|dkr|d}||rdnd} nt|dkrt|dkr|d|d}|d} |d} |d} ||r,dnd} |d} nHt|dkr|t|dksDJ|d|dkrV|d|dks`t d|d |d}d}|d} |d}|d|d} | } |d} | } t |j}t |||gt|t|t|t| t|t| t|t|t|t| t| t|  |S) NrrrrGr FTzMOnly bsi,bso->io supported for tensor contractions, but dims for A x B were: r)rr;rrrBrrQ batched_igemmstriderr8r,rArrcigemmr?rrirh)rjryrdrrrrrrrrr3rrrrrrigemm s(   $rcCst|jdkrt|jdkstd|jd|jt|||||}|dur0tj|tj|jd}|r=| d}d}nV| }|d|jdkrU| }| d}n>|d|jdkrgd }| d}n,|ddkrx| }| d}n|ddkr| }| d}n | }| d}|r| d}d}n8| }|d|jdkr| }| d}d}n|d|jdkr| d}d }n | }| d}d}|jd} |jd} |jd} |jd} | } |jd|jd}|jd|jd}|jd|jd}t |j}t|||gt|t|t|t| t| t| t|t|t|t|t|t| t|t|t|t| |S) Nrz@Expected 3-dimensional tensors for bmm, but got shapes A and B: rrr FrrGT)rrQrrr;rrrB is_contiguousr contiguousr8r,rArrcbatched_igemmr?rrirhc_longc_uint32)rjryrdrrrrsr num_batchrrr3rstrideAstrideBstrideCrrrrros           rzeigemmlt is deprecated and will be removed in a future release. Please use int8_linear_matmul instead.SASBSoutcCsn|dur|ddkrtd|dd|dur(|ddkr(td|ddt||||d}||jdffS)Nr rz 0: Nr.zQint8_linear_matmul only supports B^T @ A. Inner dimensions do not match: B @ A = z @ rFrVrz#int8_linear_matmul not implemented!z$cublasLt ran into an error! shapeA=z , shapeB=z , shapeC=z (lda, ldb, ldc)=z (m, n, k)=)rQrOr;rndimrmatmulr?r1rrcopy_rorBrrKr8r,rArhr?rirr cigemmlt_32 cigemmlt_8rur)rjryrdrOshapeAshapeBshapeCr3rrrrrr ctxptrAptrBptrC ptrRowScaler has_errorrrrr sz             "  r  row_stats col_statsrc Cs|jtjksJ|dur|jtjksJ|dur tj|tjd}t|}t|}t|}t|}t|} tt|j dd} t|j d} t |||||gt |t ||||| | | t|Wd|S1sowY|S)aPerforms dequantization on the result of a quantized int8 matrix multiplication. Args: A (`torch.Tensor` with dtype `torch.int32`): The result of a quantized int8 matrix multiplication. row_stats (`torch.Tensor`): The row-wise quantization statistics for the lhs operand of the matrix multiplication. col_stats (`torch.Tensor`): The column-wise quantization statistics for the rhs operand of the matrix multiplication. out (`torch.Tensor`, *optional*): A pre-allocated tensor to store the output of the operation. bias (`torch.Tensor`, *optional*): An optional bias vector to add to the result. Returns: `torch.Tensor`: The dequantized result with an optional bias, with dtype `torch.float16`. Nrr.)rOr;rr empty_likerhr?rirrQrrKrcdequant_mm_int32_fp16r) rjrrrdrrptrOut ptrRowStats ptrColStatsptrBiasnumRowsnumColsrrrint8_mm_dequantK s*  r&z?mm_dequant is deprecated. Please use int8_mm_dequant() instead.cCst|||||Sr")r&)rjr%rrrd new_row_stats new_col_statsrrrr mm_dequanty s r) nnz_block_ptrcCs|sJd}|dus|dur?|d|jd}|dkr)||k}||d|dur2t||}|dur?|jddd}|||fS)a "Determine the quantization statistics for input matrix `A` in accordance to the `LLM.int8()` algorithm. The row-wise and column-wise absmax values are determined. For more information, see the [LLM.int8() paper](https://arxiv.org/abs/2208.07339). This function is useful for training, but for inference it is advised to use [`get_row_absmax`] instead. The column-wise quantization scales are not typically needed in inference scenarios. Args: A (`torch.Tensor` with dtype `torch.float16`): Input tensor. row_stats (`torch.Tensor`, *optional*): If provided, calculation of row statistics is skipped. col_stats (`torch.Tensor`, *optional*): If provided, calculation of column statistics is skipped. nnz_block_ptr (`torch.Tensor`, *optional*): Not used. threshold (`float`, *optional*): An optional threshold for sparse decomposition of outlier features. No outliers are held back when 0.0. Defaults to 0.0. Returns: `Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]`: A tuple containing quantization statistics. - `torch.Tensor` with dtype `torch.float32`: The row-wise quantization statistics. - `torch.Tensor` with dtype `torch.float32`: The column-wise quantization statistics. - `torch.Tensor` with dtype `torch.bool`, *optional*: A mask indicating the locations of outliers in the input tensor. Nr.rrFdimkeepdim)is_floating_pointrr]rQ masked_fill_get_row_absmaxamaxr?)rjrrr* threshold outlier_maskabsArrrget_colrow_absmax s !   r5c Cs|jtjksJt|jdd}|jd}tj|ftj|jd}t|gt |"t t |t |t |t |t |t|Wd|S1sOwY|S)aTDetermine the quantization statistics for input matrix `A` in accordance to the `LLM.int8()` algorithm. For more information, see the [LLM.int8() paper](https://arxiv.org/abs/2208.07339). Args: A (`torch.Tensor` with dtype `torch.float16`): The input matrix. threshold (`float`, *optional*): An optional threshold for sparse decomposition of outlier features. No outliers are held back when 0.0. Defaults to 0.0. Returns: `torch.Tensor` with dtype `torch.float32`: The absolute maximum value for each row, with outliers ignored. Nr.rN)rOr;rrrQrorprBrrKrcget_row_statsrhr?rrrir)rjr2rrrrrrr0 s$      r0c @s4eZdZdedededejdejdejf ddZd S) COOSparseTensorrrnnzrowidxcolidxrcCs|jtjksJ|jtjksJ|jtjksJ||ks J||ks(J||ks0J||_||_||_||_||_ ||_ dSr") rOr;rrrfrrr8r9r:r)rrrr8r9r:rrrrr s zCOOSparseTensor.__init__N)r4r5r6rr;r r rrrrr7 sr7c@eZdZddZdS)CSRSparseTensorcCs|jtjksJ|jtjksJ|jtjksJ||ks J||ks(J||dks2J||_||_||_||_||_ ||_ dSNr ) rOr;rrrfrrr8rowptrr:r)rrrr8r>r:rrrrr  zCSRSparseTensor.__init__Nr4r5r6r rrrrr<  r<c@r;)CSCSparseTensorcCs|jtjksJ|jtjksJ|jtjksJ||ks J||ks(J||dks2J||_||_||_||_||_ ||_ dSr=) rOr;rrrfrrr8colptrr9r)rrrr8rCr9rrrrr r?zCSCSparseTensor.__init__Nr@rrrrrB rArBcCsztj|jdd\}}|dtj|jdftj|jjd}|j| | dd| dt |j|j |j||j|jSNT return_countsr rNr)r:srcr,)r;uniquer9add_rrrrBscatter_rrcumsum_r<rr8r:r)cooArcountsr>rrrcoo2csr s   rNcCst|j\}}|j|}|j|}tj|dd\}}|dtj|jdftj |jj d}|j | | dd|dt|j|j|j|||SrD)r;rr:r9rrHrIrrrrBrJrrrKrBrr8)rLr col2rowidxr9r colvaluesrMrCrrrcoo2csc# s    rQcCsLtj|ftj|d}tj|ftj|d}tj|f||d}t||||||S)NrN)r;rrr7)rrr8rBrOr9r:rrrr coo_zeros/ srRzDThis function is deprecated. Please use `int8_double_quant` instead.out_colout_rowc Csd}t||||||d\}}}}} |dkrN| durNtjd|jd|jtjd} |dd| f} t|jd|jd| | | d| | d | }|||| |fS)aTDetermine the quantization statistics for input matrix `A` in accordance to the `LLM.int8()` algorithm. The statistics are determined both row-wise and column-wise (transposed). For more information, see the [LLM.int8() paper](https://arxiv.org/abs/2208.07339). This function exists for backwards compatibility only. It is advised to use [`int8_double_quant`] instead. The difference is that this function will return a [`COOSparseTensor`] for outliers instead of a column index. Args: A (`torch.Tensor` with dtype `torch.float16`): The input matrix. col_stats (`torch.Tensor`, *optional*): A pre-allocated tensor to hold the column-wise quantization scales. row_stats (`torch.Tensor`, *optional*): A pre-allocated tensor to hold the row-wise quantization scales. out_col (`torch.Tensor`, *optional*): A pre-allocated tensor to hold the column-wise quantized data. out_row (`torch.Tensor`, *optional*): A pre-allocated tensor to hold the row-wise quantized data. threshold (`float`, *optional*): An optional threshold for sparse decomposition of outlier features. No outliers are held back when 0.0. Defaults to 0.0. Returns: `Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, Optional[torch.Tensor]]`: A tuple containing the quantized tensor and relevant statistics. - `torch.Tensor` with dtype `torch.int8`: The row-wise quantized data. - `torch.Tensor` with dtype `torch.int8`: The column-wise quantized data. - `torch.Tensor` with dtype `torch.float32`: The row-wise quantization scales. - `torch.Tensor` with dtype `torch.float32`: The column-wise quantization scales. - `COOSparseTensor`, *optional*: A structure representing the outlier values from the input tensor. Nr2rrrVr )int8_double_quantr;rrQrBrr7rfrepeat_interleaverrrflattenr?) rjrrrSrTr2 coo_tensor quant_row quant_col outlier_cols outlier_rowsoutliersrrr double_quant6 s*(  r_c Cst||d\}}}t||d\}}} |dkr | dur || d}t|t|dtj } |dur:| |}|durC| | } || || |fS)aLDetermine the quantization statistics for input matrix `A` in accordance to the `LLM.int8()` algorithm. The statistics are determined both row-wise and column-wise (transposed). For more information, see the [LLM.int8() paper](https://arxiv.org/abs/2208.07339). This function is useful for training, but for inference it is advised to use [`int8_vectorwise_quant`] instead. This implementation performs additional column-wise transposed calculations which are not optimized. Args: A (`torch.Tensor` with dtype `torch.float16`): The input matrix. col_stats (`torch.Tensor`, *optional*): A pre-allocated tensor to hold the column-wise quantization scales. row_stats (`torch.Tensor`, *optional*): A pre-allocated tensor to hold the row-wise quantization scales. out_col (`torch.Tensor`, *optional*): A pre-allocated tensor to hold the column-wise quantized data. out_row (`torch.Tensor`, *optional*): A pre-allocated tensor to hold the row-wise quantized data. threshold (`float`, *optional*): An optional threshold for sparse decomposition of outlier features. No outliers are held back when 0.0. Defaults to 0.0. Returns: `Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, Optional[torch.Tensor]]`: A tuple containing the quantized tensor and relevant statistics. - `torch.Tensor` with dtype `torch.int8`: The row-wise quantized data. - `torch.Tensor` with dtype `torch.int8`: The column-wise quantized data. - `torch.Tensor` with dtype `torch.float32`: The row-wise quantization scales. - `torch.Tensor` with dtype `torch.float32`: The column-wise quantization scales. - `torch.Tensor` with dtype `torch.int32`, *optional*: A list of column indices which contain outlier features. rUrNr) int8_vectorwise_quantr5 masked_fillr;rmulC unsqueezerrrrXr?) rjrrrSrTr2rZr\rr3r[rrrrVx s* "  rVstatscCs||dddS)aYDequantizes a tensor with dtype `torch.int8` to `torch.float32`. Args: A (`torch.Tensor` with dtype `torch.int8`): The quantized int8 tensor. stats (`torch.Tensor` with dtype `torch.float32`): The row-wise quantization statistics. Returns: `torch.Tensor` with dtype `torch.float32`: The dequantized tensor. r.r g@ ?)r])rjrerrrint8_vectorwise_dequant s rfc Cs"|jtjksJt|gt|jdd}|jd}tj||jtjd}tj|j|jtj d}d}|dkrL| |k}| rLt |j dd d}t|$tt|t|t|t|t|t|t|Wdn1swwY|dkr|durd|dd|f<|||fS)awQuantizes a tensor with dtype `torch.float16` to `torch.int8` in accordance to the `LLM.int8()` algorithm. For more information, see the [LLM.int8() paper](https://arxiv.org/abs/2208.07339). Args: A (`torch.Tensor` with dtype `torch.float16`): The input tensor. threshold (`float`, *optional*): An optional threshold for sparse decomposition of outlier features. No outliers are held back when 0.0. Defaults to 0.0. Returns: `Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]`: A tuple containing the quantized tensor and relevant statistics. - `torch.Tensor` with dtype `torch.int8`: The quantized data. - `torch.Tensor` with dtype `torch.float32`: The quantization scales. - `torch.Tensor` with dtype `torch.int32`, *optional*: A list of column indices which contain outlier features. Nr.rVrr)r,r )rOr;halfrrrQrorBrprranyargwherer]rKrcint8_vector_quantrhr?rrrir)rjr2rrrrTr\r^rrrr` s2      r`c Cst|j}|dur|j|f}n|d}|dur)t|d|j|j||d|\}}n|d|f}|d} t| dkrHt| d} t| d} nt| d| d} t| d} t||g|dkr|rst t |t || | n}t t |t || | np|dkr|rt t |t || | n]t t |t || | nP|dkr|rt t |t || | n=t t |t || | n0|dkr|dkrt t |t || | n|dkrt t |t || | n td|d |t|||fS) Nr rrGrrrrz)Transform function not implemented: From r)rrBrQr rOrr?rirrctransform_row2col32Trhctransform_row2col32ctransform_row2turingTctransform_row2turingctransform_row2ampereTctransform_row2amperectransform_turing2rowctransform_ampere2rowrur) rjrrrdrrr rCr rQr rrrr transform sF  $   rsrLcCst|ts2|jr|jtjksJdt|jd|jd||d |d | d}|durFtj |j |jdf|j |jd}|j}|j|ksRJ|j|ks[J|j |ksdJ|j|jdksnJ|rtdnd}||r}dnd}|jd}tj}t|j}t|j} t|j } t|} t|} t|j} t|j }t|j}t|jd}t|}t|}t|j|j|j ||gt||| | | ||||| || t| |S)Nz8Tensor must be `COOSparseTensor or a PyTorch COO tensor.rr )rrr8r9r:rrVFT) r1r7 is_sparselayoutr; sparse_coorQ_nnzindicesrrrorrBrOr8r9rfr:rrrrDr,r9rhr?rirr cspmm_coor)rLryrdr8rrrr ptrRowidx ptrColidx ptrValuesrrcnnzcrowsAccolsAccolsBcldbcldcrrrspmm_coo, sj            rcCs~|durtj|j|jdf|j|jjd}|j}t|j}|j |ks&J|j |ks/J|j |ks8J|j |jdksKJ|j d|j| rQdnd}||rZdnd}|jd}tj|j dd\} } | d} tj| dd\} } | } | } | dd ksJd | dd |jtjtjfvsJt| }t| }t| }t|j }t|j }t|j}t|}t|}t|}t| }t|j}t|j}t|j }t|jd}t|jd}t|}t|}t|j |j |j|||g|jtjkr t||||||||||||||n|jtjkr9t||||||||||||||t||S) Nr rVrz vs FTrE) descendingrz)Current max count per row is 8 but found r4)r;rrrQrBrrOr8rr9rfr:rrrrHcumsumrrrrrhr?rirr cspmm_coo_very_sparse_naive_fp16 cspmm_coo_very_sparse_naive_int8r)rLry dequant_statsrdr8rCrrrrrMr max_countmax_idx ptrOffset ptrMaxCount ptrMaxIdxrzr{r|rrptrDequantStats cnnz_rowsr}r~rcrowsBrrrrrrspmm_coo_very_sparseo s" &          rg_@ztThis function is deprecated and will be removed in a future release. Consider using `int8_vectorwise_quant` instead.vectorc Cs$|dkrt|}t||dtj}||fS|dvr>tjt||dd}t|t|tj}||fS|dkru|j }|}|| }|dkrWd}d |}| }t||} t||| | }||fS|d vr|j }|}tj||ddtj ||dd}d||dk<d |}tj ||dd}t||} t||| | }||fS|d krt Bt|} tj| |dd}|d }| | | k} t|| } | | | | || <t||ttj}Wd||fS1swY||fSdS) Nlinear)rrTr+ zeropointrr go@)vector-zeropoint row-zeropointtruncated-vectorgffffff?)r;rrr?rrrr1rcrOminaminno_grad expand_asr) r r,r&rxqrOdynaqxminxzpxabsxrrrrrvectorwise_quant sV      rzvThis function is deprecated and will be removed in a future release. Consider using `int8_vectorwise_dequant` instead.cCs$|dkr|t|tj}|SdS)Nr)rcrr;rp)rrr&r rrrvectorwise_dequant srznThis function is deprecated and will be removed in a future release. Consider using `int8_mm_dequant` instead.cCs|dkr||tt}|||S|dkr(d||}|||S|dkrqd||}|}t|jdkrIt|jdkrI|d}t|jdkr\t|jdkr\|d}t|jdkrh||9}n||9}||S|dkr|}t|jdkrt|jdkr|d}t|jdkrt|jdkr|d}t|jdkr|d|9}n|d|9}|d|9}||S|d kr|}t|jdkrt|jdkr|d}t|jdkrt|jdkr|d}t|jdkr|||tt9}n |||tt9}||S|d vrd|}t|jdkr/t|jdkr/|d}t|jdkrDt|jdkrD|d}t|jdkrS||t9}n||t9}||t9}||SdS) NrrrrrrGrrr)rr)rcr?rrrQsqueezer1)rS1S2rOr&rr rrrvectorwise_mm_dequant sd                      rcCs|d|d|d}|}t|jdkr(t|jdkr(|d}t|jdkr8||d9}n||d9}||dd9}||7}||S)Nrr rGrr)r?r1sumrrQrr)rrjryrrrOrr rrrdequant_min_max@ s"   rc Cs|d}|d}|dvsJ|jjdksJtj|d|ftj|jd}t|}t|d}t|d}t|} t|} t|} t |j} |dkr[t | | | |||n|dkrit | | | |||t | |S)Nrr )rrr<rNrr)rBrbr;rrfrr?rirhrrcextractOutliers_turingcextractOutliers_amperer) rjrrrformatArdidx_sizerrrptrIdxr rCrrrextract_outliersO s$   rcCs6t|}tt|t|t|t||Sr")r;rarcpipeline_testrhr?rUrf)rj batch_sizerdrrr pipeline_testi s (rr3)T)NTr")TrT)rT)TrrGr)Trr)r)rF)rNFNN)Nrr)NNNrTF)NNNNrTF)Nr])NNNr])NNNr]rr7)NNNN) NrrrrrNrF)rrNr)rrF)r)NFFN)NFF)NNNr)r)NNNNr)r r)r)ctypesr?rmathrtypingrrrrrrnumpyrYr;r typing_extensionsr bitsandbytes.utilsr r cextensionrr`compiled_with_cudacadam32bit_grad_fp32cadam32bit_grad_fp16cadam32bit_grad_bf16cmomentum32bit_grad_32cmomentum32bit_grad_16crmsprop32bit_grad_32crmsprop32bit_grad_16clion32bit_grad_fp32clion32bit_grad_fp16clion32bit_grad_bf16cadagrad32bit_grad_32cadagrad32bit_grad_16cademamix32bit_grad_fp32cademamix32bit_grad_fp16cademamix32bit_grad_bf16rcadam_static_8bit_grad_32cadam_static_8bit_grad_16cmomentum_static_8bit_grad_32cmomentum_static_8bit_grad_16crmsprop_static_8bit_grad_32crmsprop_static_8bit_grad_16clion_static_8bit_grad_32clion_static_8bit_grad_16rcadam_8bit_blockwise_grad_fp32cadam_8bit_blockwise_grad_fp16cadam_8bit_blockwise_grad_bf16"cmomentum_8bit_blockwise_grad_fp32"cmomentum_8bit_blockwise_grad_fp16"cmomentum_8bit_blockwise_grad_bf16!crmsprop_8bit_blockwise_grad_fp32!crmsprop_8bit_blockwise_grad_fp16!crmsprop_8bit_blockwise_grad_bf16clion_8bit_blockwise_grad_fp32clion_8bit_blockwise_grad_fp16clion_8bit_blockwise_grad_bf16!cadagrad_8bit_blockwise_grad_fp32!cadagrad_8bit_blockwise_grad_fp16!cadagrad_8bit_blockwise_grad_bf16"cademamix_8bit_blockwise_grad_fp32"cademamix_8bit_blockwise_grad_fp16"cademamix_8bit_blockwise_grad_bf16rrr8rDrSrprrdrsrrBFIRST_CUDA_DEVICEr< device_countrKrLrer/r~r FutureWarningrrrrrrrrrStreamrr@rrhrrrr rr?rrrjrrnrrrrrrrrrrrrErrrrrrrrrrr@r r r&r)r5r0r7r<rBrNrQrgrRr_rVrfr`rsrrrcrrrrrrrrrrs      %"              + 7   $      '( B4  j fT      q     h  $ $(      g            @ *W k h `    (k .  9%   C :83  CR, ;