o 3wiY@sddlZddlmZmZmZmZmZddlZddlZddlm Z m Z m Z m Z ddl m mZddlZddlmZddlmZddlmZmZeddd ZGd d d ej jZGd d d ej jZGdddej jZdedfddZGddde jZ Gddde Z!Gddde Z"Gdddej jZ#ddZ$Gddde jZ%Gd d!d!e jZ&Gd"d#d#e&Z'Gd$d%d%e&Z(Gd&d'd'e jZ)Gd(d)d)e jZ*Gd*d+d+e jZ+dS),N)AnyOptionalTypeVarUnionoverload)Tensordevicedtypenn) QuantState)GlobalOptimManager)*INVERSE_LINEAR_8BIT_WEIGHTS_FORMAT_MAPPING OutlierTracerTztorch.nn.Module)boundcseZdZdZ        ddededeedeed ed ed ed eed dffdd Z dddZ dddZ ded efddZ Z S)StableEmbeddinga Custom embedding layer designed to improve stability during training for NLP tasks by using 32-bit optimizer states. It is designed to reduce gradient variations that can result from quantization. This embedding layer is initialized with Xavier uniform initialization followed by layer normalization. Example: ``` # Initialize StableEmbedding layer with vocabulary size 1000, embedding dimension 300 embedding_layer = StableEmbedding(num_embeddings=1000, embedding_dim=300) # Reset embedding parameters embedding_layer.reset_parameters() # Perform a forward pass with input tensor input_tensor = torch.tensor([1, 2, 3]) output_embedding = embedding_layer(input_tensor) ``` Attributes: norm (`torch.nn.LayerNorm`): Layer normalization applied after the embedding. Methods: reset_parameters(): Reset embedding parameters using Xavier uniform initialization. forward(input: Tensor) -> Tensor: Forward pass through the stable embedding layer. N@Fnum_embeddings embedding_dim padding_idxmax_norm norm_typescale_grad_by_freqsparse_weightreturnc sJt||||||||| | tjj|| d|_t|dddidSa Args: num_embeddings (`int`): The number of unique embeddings (vocabulary size). embedding_dim (`int`): The dimensionality of the embedding. padding_idx (`Optional[int]`): Pads the output with zeros at the given index. max_norm (`Optional[float]`): Renormalizes embeddings to have a maximum L2 norm. norm_type (`float`, defaults to `2.0`): The p-norm to compute for the `max_norm` option. scale_grad_by_freq (`bool`, defaults to `False`): Scale gradient by frequency during backpropagation. sparse (`bool`, defaults to `False`): Computes dense gradients. Set to `True` to compute sparse gradients instead. _weight (`Optional[Tensor]`): Pretrained embeddings. rweight optim_bits N) super__init__torchr LayerNormnormr get_instanceregister_module_override) selfrrrrrrrrrr  __class__T/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/bitsandbytes/nn/modules.pyr"2s  zStableEmbedding.__init__cCtjj|j|dSNr#r initxavier_uniform_r_fill_padding_idx_with_zeror(r+r+r,reset_parametersa z StableEmbedding.reset_parameterscCN|jdur%t|j|jdWddS1swYdSdSNrrr#no_gradrfill_r3r+r+r,r2l  "z+StableEmbedding._fill_padding_idx_with_zeroinputc CsDt||j|j|j|j|j|j}|t }| ||jj Sr.) F embeddingrrrrrrtor#get_default_dtyper%r r(r<embr+r+r,forwardqs zStableEmbedding.forward)NNrFFNNNrN)__name__ __module__ __qualname____doc__intrfloatboolrr"r4r2rC __classcell__r+r+r)r,rsB   / rcseZdZdZ       ddededeedeed ed ed ed eed ee ddffdd Z dddZ dddZ dedefddZ ZS) EmbeddingzS Embedding class to store and retrieve word embeddings from their indices. NrFrrrrrrrrrrc s8tj||||||||| d t|dddidSr)r!r"r r&r') r(rrrrrrrrrr)r+r,r"s zEmbedding.__init__cCr-r.r/r3r+r+r,r4r5zEmbedding.reset_parameterscCr6r7r8r3r+r+r,r2r;z%Embedding._fill_padding_idx_with_zeror<c Cs&t||j|j|j|j|j|j}|Sr.)r=r>rrrrrrrAr+r+r,rCs zEmbedding.forward)NNrFFNNrD)rErFrGrHrIrrJrKrrr"r4r2rCrLr+r+r)r,rMsD    , rMcseZdZddddddejddf deejdeeded e d e d ej d ed de ddfddZ ddZ ddZddZddZe   d5dejdee efde d ed ddf ddZed6d d!Zd"d#Zd$d%Zd7d&eeeee fd'e fd(d)Zd7d&eeeee fd'e fd*d+Ze , , ,d8d-ed&eeeefd.eee e fd'e def d/d0Zed9d-ed.ee e fd'e defd1d0Zed9d-ed2ed'e defd3d0Zfd4d0ZZS): Params4bitNF@Tfp4data quant_state blocksizecompress_statistics quant_type quant_storagemodule Linear4bit bnb_quantizedrc CsV|dur td}tj|||} || _|| _|| _|| _|| _| | _ || _ || _ | Sr7) r#emptyr_make_subclassrSrTrUrRrVrYrQrW) clsrQ requires_gradrRrSrTrUrVrWrYr(r+r+r,__new__s zParams4bit.__new__cCs"|j}|j|d<|j|d<|S)NrQr])__dict__copyrQr]r(stater+r+r, __getstate__s   zParams4bit.__getstate__cCs^|d|_|d|_|d|_|d|_|d|_|d|_|d|_|d|_|d |_dS) Nr]rSrTrUrRrQrVrYrW) r]rSrTrUrRrQrVrYrWrar+r+r, __setstate__s        zParams4bit.__setstate__cCsHt|t|}|}||t|d|_t|d|_|S)NrRrQ)typer^rcrdr`deepcopyrRrQ)r(memo new_instancerbr+r+r, __deepcopy__s  zParams4bit.__deepcopy__cCs(t|t|}|}|||Sr.)rer^rcrd)r(rhrbr+r+r,__copy__s zParams4bit.__copy__cudaquantized_statsr]cKsttj|||}||_tj||d|_|jj|_|jj |_ |jj |_ d|_ |j |_||_|jdur8|j|j_|S)N)qs_dictrT)r#rr[r?r]r from_dictrRrSnestedrTrUrYr rVrW)r\rQrlr]rrWkwargsr(r+r+r,from_prequantized s      zParams4bit.from_prequantizedr+cCsH|duri}tj||i|WdS1swYdSr.)r#_CDisableTorchFunctionSubclass)r\functypesargsrpr+r+r,__torch_function__%s   $zParams4bit.__torch_function__cCsZ|j|}tjj||j|j|j|j d\}}||_||_ |j dur(||j _ d|_ |S)N)rSrTrUrVT) rQ contiguousr?bnb functional quantize_4bitrSrTrUrVrRrWrY)r(rww_4bitrRr+r+r, _quantize,s  zParams4bit._quantizecC |jddSNcpurr?r3r+r+r,r< zParams4bit.cpur non_blockingcC |j|dur d|dS||dSNrkrrrr(rrr+r+r,rk? zParams4bit.cudacCrNxpurrrr+r+r,rBrzParams4bit.xpu.r(r cCdSr.r+r(rr rr+r+r,r?Ez Params4bit.tocCrr.r+r(r rr+r+r,r?MtensorcCrr.r+r(rrr+r+r,r?Prc stjjj|i|\}}}}|dur|jdkr|js||S|jdur*|j|t t j|||d|j |j|j |j |j|jd}|S)Nmetarr r)r]rRrSrTrUrV)r#rr_nn _parse_torerYr~rRr?rNr!r]rSrTrUrVr(rvrprr rconvert_to_format new_paramr)r+r,r?Ss    )FrkN)r+NNF....) rErFrGr#uint8rrr rIrKstrr r^rcrdrirj classmethoddictrrqrwr~rrrrkrrrr?rLr+r+r)r,rNs       "" &rNrW) Embedding4bitrXcCsrt|jdddur dSt|dddurtd|jjddks"Jt|jts2t|j|jdd|_|j|j_dS)NrRzhFP4 quantization state not initialized. Please call .cuda() or .to(device) on the LinearFP4 layer first.T)rVrY) getattrrwarningswarnshape isinstancerNrVrR)rWr+r+r,'fix_4bit_weight_quant_state_from_moduleis rcsTeZdZdZddddejdffdd ZddZfd d Zd ej fd d Z Z S)rXa This class is the base module for the 4-bit quantization algorithm presented in [QLoRA](https://arxiv.org/abs/2305.14314). QLoRA 4-bit linear layers uses blockwise k-bit quantization under the hood, with the possibility of selecting various compute datatypes such as FP4 and NF4. In order to quantize a linear layer one should first load the original fp16 / bf16 weights into the Linear4bit module, then call `quantized_module.to("cuda")` to quantize the fp16 / bf16 weights. Example: ```python import torch import torch.nn as nn import bitsandbytes as bnb from bnb.nn import Linear4bit fp16_model = nn.Sequential( nn.Linear(64, 64), nn.Linear(64, 64) ) quantized_model = nn.Sequential( Linear4bit(64, 64), Linear4bit(64, 64) ) quantized_model.load_state_dict(fp16_model.state_dict()) quantized_model = quantized_model.to(0) # Quantization happens here ``` TNrPc sHt||||t|jjd||||d|_||_d|_d|_||_dS)aw Initialize Linear4bit class. Args: input_features (`str`): Number of input features of the linear layer. output_features (`str`): Number of output features of the linear layer. bias (`bool`, defaults to `True`): Whether the linear class uses the bias term as well. Fr]rTrUrVrWN) r!r"rNrrQ compute_dtypecompute_type_is_setrRrV) r(input_featuresoutput_featuresbiasrrTrUrVrr)r+r,r"s  zLinear4bit.__init__cCs|jtjtjfvr|j|_dS|jtjkrM|jtjkr0||jdkr0t dtj ddd|jtjkrO||jdkrQt dtj ddddSdSdSdS)NzInput type into Linear4bit is torch.float16, but bnb_4bit_compute_dtype=torch.float32 (default). This will lead to slow inference.ignorez .*inference.)messagezInput type into Linear4bit is torch.float16, but bnb_4bit_compute_dtype=torch.float32 (default). This will lead to slow inference or training speed.z.*inference or training) r r#float32bfloat16rfloat16numelrrrfilterwarnings)r(xr+r+r,set_compute_types   zLinear4bit.set_compute_typecsdt|||t|jdddur.|jjjddD]\}}|r#|n|||d|<qdSdS)zc save weight and bias, then fill state_dict with components of quant_state rRNT)packedzweight.)r!_save_to_state_dictrrrRas_dictitemsdetach)r( destinationprefix keep_varskvr)r+r,rs zLinear4bit._save_to_state_dictrcCst||jdur|jj|jkr|jj|j|j_|js%||d|_|j}|jdur3||j}|jdur:dn|j|j}tj ||j ||j j d|S)NT)rrR) rrr rQr?rrrry matmul_4bitrtrR)r(r inp_dtyperr+r+r,rCs   "zLinear4bit.forward) rErFrGrHr#rr"rrrrCrLr+r+r)r,rXzs$%  rXc.eZdZdZdddejdffdd ZZS) LinearFP4z' Implements the FP4 data type. TNc t|||||d||dS)Q Args: input_features (`str`): Number of input features of the linear layer. output_features (`str`): Number of output features of the linear layer. bias (`bool`, defaults to `True`): Whether the linear class uses the bias term as well. rPNr!r"r(rrrrrTrVrr)r+r,r"zLinearFP4.__init__rErFrGrHr#rr"rLr+r+r)r,rsrcr) LinearNF4a"Implements the NF4 data type. Constructs a quantization data type where each bin has equal area under a standard normal distribution N(0, 1) that is normalized into the range [-1, 1]. For more information read the paper: QLoRA: Efficient Finetuning of Quantized LLMs (https://arxiv.org/abs/2305.14314) Implementation of the NF4 data type in bitsandbytes can be found in the `create_normal_map` function in the `functional.py` file: https://github.com/TimDettmers/bitsandbytes/blob/main/bitsandbytes/functional.py#L236. TNc r)rnf4Nrrr)r+r,r"#rzLinearNF4.__init__rr+r+r)r,rsrc s:eZdZ     ddeejdeejdeejfddZfd d Zd d Zd d ee e e e fde fddZd d ee e e e fde fddZddZe   d!ded ee e e fdee ee fde def ddZed"dede ee fde defddZed"dedede defddZfddZZS)# Int8ParamsNTFrQCBSCBcCs8|dur td}tj|||}||_||_||_|Sr7)r#rZrr[rrhas_fp16_weights)r\rQr]rrrobjr+r+r,r^Cs zInt8Params.__new__csP|jr t|S|jj|tjd}tj |\}}}||_||_ ||_ |S)Nrr ) rr!r?rQrxr#rryrzint8_vectorwise_quantrr)r(rBrr_r)r+r,r~Ss zInt8Params._quantizecCrrrr3r+r+r,r`rzInt8Params.cpurrcCrrrrr+r+r,rkcrzInt8Params.cudacCrrrrr+r+r,rfrzInt8Params.xpuc CsDt|jt|t|j||j|jt|j|t|j|d}|S)N)rQr]rrr) rer^r`rfrQr]rrr)r(rgrhr+r+r,riis   zInt8Params.__deepcopy__.r(r rcCrr.r+rr+r+r,r?urz Int8Params.tocCrr.r+rr+r+r,r?}rrcCrr.r+rr+r+r,r?rcsztjjj|i|\}}}}|dur#|jdkr#|jjjdkr#||Stt j |||d|j |j d}|j |_ |j|_|S)Nrrr)r]r)r#rrrrrerQrr~rr!r?r]rrrrr)r+r,r?s  )NTFNNrrr)rErFrGrr#rr^r~rrrIrrrKrkrrirrr r?rLr+r+r)r,rBsL  ""  &rc Cs||d}|durdS||dd}t|tjr!|}t|tr1|tvr1td|t|tr>|tvr>t|}|dkrItd|dS)Nr weight_formatrowz'Expected supported weight format - got z+Only 'row' weight format is supported, got ) getpoprr#ritemrIr ValueError) state_dictrlocal_metadatastrict missing_keysunexpected_keys error_msgsrrr+r+r,maybe_rearrange_weights rcs<eZdZdZd fdd ZddZdedefd d ZZS) Embedding8bita This class implements [LLM.int8()](https://arxiv.org/abs/2208.07339) algorithm for embedding layer Quantization API is similar to Linear8bitLt: ```python import torch import torch.nn as nn from bitsandbytes.nn import Embedding8bit fp16_module = nn.Embedding(128, 64) int8_module = Embedding8bit(128, 64) int8_module.load_state_dict(fp16_module.state_dict()) int8_module = int8_module.to(0) # Quantization happens here ``` Ncs8tj||||d|jjj|_t|jjddd|_dS)NrFrr])r!r"rrQr r)r(rrrr r)r+r,r"s zEmbedding8bit.__init__cCtd)Nz.Saving Embedding8bit module is not implementedNotImplementedErrorr(rrrr+r+r,rz!Embedding8bit._save_to_state_dictr<rcCst|jds td|jj}|jj}|j|j|jfksJ|j|jfks&Jt ||}t || |jd}||d}| |j S)NrzKEmbedding layer is not quantized. Please call .cuda() or .to(device) first.rg_@) hasattrr RuntimeErrorrQrrrrr=r>viewr?r )r(r<rows row_statscompressed_outputcompressed_output_statsoutputr+r+r,rCs    zEmbedding8bit.forward)NN) rErFrGrHr"rrrCrLr+r+r)r,rs rcsTeZdZdZddejdffdd ZdefddZd d Z ded efd d Z Z S)ra3 This is the base class similar to Linear4bit. It implements the 4-bit quantization algorithm presented in [QLoRA](https://arxiv.org/abs/2305.14314) for embeddings. Quantization API is similar to Linear4bit: ```python import torch import torch.nn as nn from bitsandbytes.nn import Embedding4bit fp16_module = nn.Embedding(128, 64) quantized_module = Embedding4bit(128, 64) quantized_module.load_state_dict(fp16_module.state_dict()) quantized_module = quantized_module.to(0) # Quantization happens here ``` NrPcsntj||||d|jjj|_t|jjdd|||d|_|jj}||dkr5td|d|ddSdS)NrFrrzEmbedding size z is not divisible by block size z#. This will lead to slow inference.) r!r"rrQr rNrSrr)r(rrr rUrVrrSr)r+r,r"s   zEmbedding4bit.__init__r<c CsR|j|jjjdks J|jjtj|j|jdd}tj j j ||j|jd|ddd}|j | |jddfksCJ|j|jj}|jjj}|j |j|fksZJtj j j ||j||dd}|j | |fkswJt|jj}||_tg|j |jR|_ tj ||}|j g|j |jRksJ||jS)Nrrrr<r)rrrRrSrQrr#rrr rzr>rrabsmaxr`rfSizerydequantize_4bitr?r ) r(r< w_4bit_uint8 output_4bitblocks_per_embr output_absmaxoutput_quant_staterr+r+r, _forward_with_partial_dequantize s6$    z.Embedding4bit._forward_with_partial_dequantizecCr)Nz.Saving Embedding4bit module is not implementedrrr+r+r,r+rz!Embedding4bit._save_to_state_dictrcCsVt||j|jjjdkr||Stj|jj |jj}t j jj ||d |jS)Nrr)rrrrRrSrryrzrrQr#r r>r?r )r(r<dequantized_weightr+r+r,rC.s zEmbedding4bit.forward) rErFrGrHr#rr"rrrrCrLr+r+r)r,rs!rc&eZdZdejdffdd ZZS) EmbeddingFP4Nctj|||d||ddS)NrPr rUrVrrr(rrr rVrr)r+r,r"= zEmbeddingFP4.__init__rErFrGr#rr"rLr+r+r)r,r< rcr) EmbeddingNF4Ncr)Nrrrrr)r+r,r"PrzEmbeddingNF4.__init__rr+r+r)r,r Orr csfeZdZdZ     ddedeffdd Zfd d Zfd d Zd dZde j fddZ Z S) Linear8bitLtaZ This class is the base module for the [LLM.int8()](https://arxiv.org/abs/2208.07339) algorithm. To read more about it, have a look at the paper. In order to quantize a linear layer one should first load the original fp16 / bf16 weights into the Linear8bitLt module, then call `int8_module.to("cuda")` to quantize the fp16 weights. Example: ```python import torch import torch.nn as nn import bitsandbytes as bnb from bnb.nn import Linear8bitLt fp16_model = nn.Sequential( nn.Linear(64, 64), nn.Linear(64, 64) ) int8_model = nn.Sequential( Linear8bitLt(64, 64, has_fp16_weights=False), Linear8bitLt(64, 64, has_fp16_weights=False) ) int8_model.load_state_dict(fp16_model.state_dict()) int8_model = int8_model.to(0) # Quantization happens here ``` TNrrcsht||||t|_||_||j_||j_|dkr#|s#d|j_t |j j ||d|_ | t dS)ay Initialize Linear8bitLt class. Args: input_features (`int`): Number of input features of the linear layer. output_features (`int`): Number of output features of the linear layer. bias (`bool`, defaults to `True`): Whether the linear class uses the bias term as well. r TrN)r!r"ry MatmulLtStaterbindex thresholdruse_poolrrrQ"_register_load_state_dict_pre_hookr)r(rrrrrr rr)r+r,r"s  zLinear8bitLt.__init__c st|||d}t|j|}t|j|}||}|d}|jjsW|dur=|r+|n|||<tjdtj d||<dS|durY|rE|n|||<tjdtj d||<dSdSdS)Nrrr)r ) r!rrrrbrrr#rr) r(rrrscb_nameparam_from_weightparam_from_statekey_name format_namer)r+r,rs   z Linear8bitLt._save_to_state_dictc st|||||||t|}|D]4} | t|d} | dkrF|jjdur*td|| } |jj| |jjdurA|jj|j_| | qdS)NrzLoading a quantized checkpoint into non-quantized Linear8bitLt is not supported. Please call module.cuda() before module.load_state_dict()) r!_load_from_state_dictlistlenrrrcopy_rbremove) r(rrrrrrrunexpected_copykey input_name input_paramr)r+r,rs0      z"Linear8bitLt._load_from_state_dictcC,|jj|j_|jj|j_d|j_d|j_dSr.rrrbrr3r+r+r,init_8bit_state   zLinear8bitLt.init_8bit_statercCs|j|j_|jjdur||jdur%|jj|jkr%|jj |j|j_t j ||j|j|jd}|jj sA|jjdurA|jj|j_|SN)rrb) trainingrb is_trainingrrr!rr rQr?rymatmulrr(routr+r+r,rCs   zLinear8bitLt.forward)TTr NN) rErFrGrHrIr"rrr!r#rrCrLr+r+r)r,r bs # " 'r cs6eZdZd fdd ZddZddZd d ZZS) OutlierAwareLinearTNcs"t||||d|_d|_dSr)r!r" outlier_dim is_quantized)r(rrrrr)r+r,r"s zOutlierAwareLinear.__init__cCr)NzJPlease override the `forward_with_outliers(self, x, outlier_idx)` functionr)r(r outlier_idxr+r+r,forward_with_outliersrz(OutlierAwareLinear.forward_with_outlierscCr)NzEPlease override the `quantize_weights(self, w, outlier_idx)` functionr)r(r|r,r+r+r,quantize_weightrz"OutlierAwareLinear.quantize_weightcCsf|jdurt}|std||j}||_|js1||j|j}|jj |d|_dSdS)NzTPlease use OutlierTracer.initialize(model) before using the OutlierAwareLinear layerT) r*rr&is_initializedprint get_outliersrr+r.rQr)r(rtracerr,r|r+r+r,rCs   zOutlierAwareLinear.forward)TN)rErFrGr"r-r.rCrLr+r+r)r,r)s r)cs:eZdZ      d fdd ZddZd d ZZS) SwitchBackLinearBnbTFr Nc sft||||t|_||_||j_||j_||j_|dkr'|s'd|j_ t |j j ||d|_ dS)Nr Tr) r!r"ryr rbr rrmemory_efficient_backwardrrrrQ) r(rrrrr4rr rr)r+r,r"s  zSwitchBackLinearBnb.__init__cCrr.r r3r+r+r,r!-r"z#SwitchBackLinearBnb.init_8bit_statecCsF|j|j_|jjdur|tj||jd|jd|j }dSr#) r$rbr%rrr!ry matmul_mixedhalfrr'r+r+r,rC3s  (zSwitchBackLinearBnb.forward)TTFr NN)rErFrGr"r!rCrLr+r+r)r,r3sr3),r`typingrrrrrrr#rrr r torch.nn.functionalrzr= bitsandbytesrybitsandbytes.functionalr bitsandbytes.optimr bitsandbytes.utilsr rrrMr ParameterrNrLinearrXrrrrrrrr r r)r3r+r+r+r,s:   jOy$+R/d