o oi>v@sddlmZmZddlmZddlmZddlm Z ddl Z ddl m Z GdddZGdd d ZGd d d e jjZGd d d eZGdddeZdS))abc defaultdict)deepcopy)chain)OptionalNc@seZdZddZdS)MockArgscCs|D] }t||||qdSN)setattr)self initial_datakeyr P/home/ubuntu/.local/lib/python3.10/site-packages/bitsandbytes/optim/optimizer.py__init__szMockArgs.__init__N)__name__ __module__ __qualname__rr r r rrs rc@sJeZdZdZdZddZddZeddZd d Z dd d Z d dZ dS)GlobalOptimManagerzK A global optimizer manager for enabling custom optimizer configs. NcCtd)NzCall get_instance() instead) RuntimeErrorr r r rrzGlobalOptimManager.__init__cCs"i|_i|_d|_d|_g|_dS)NF) pid2config index2config optimizeruses_config_overridemodule_weight_config_triplerr r r initializes  zGlobalOptimManager.initializecCs&|jdur|||_|j|jSr) _instance__new__r)clsr r r get_instance&s   zGlobalOptimManager.get_instancecCsrt|}t|dtsd|ig}t|D]"\}}t|dD]\}}t||jvr5|jt||j||f<qqdS)Nrparams)list isinstancedict enumerateidrr)r r" param_groups group_indexgroupp_indexpr r rregister_parameters-s z&GlobalOptimManager.register_parameterscCsd|_t|tjjr |g}t|tjr|g}|dur(|dur(|dus$J||i}|durJ|D]}t||jvrB|jt||q.||jt|<q.dSdS)a Override initial optimizer config with specific hyperparameters. The key-values of the optimizer config for the input parameters are overridden This can be both, optimizer parameters like `betas` or `lr`, or it can be 8-bit specific parameters like `optim_bits` or `percentile_clipping`. Arguments: parameters (`torch.Tensor` or `list(torch.Tensors)`): The input parameters. key (`str`): The hyperparamter to override. value: The hyperparameter values. key_value_dict (`dict`): A dictionary with multiple key-values to override. Example: ```py import torch import bitsandbytes as bnb mng = bnb.optim.GlobalOptimManager.get_instance() model = MyModel() mng.register_parameters(model.parameters()) # 1. register parameters while still on CPU model = model.cuda() # use 8-bit optimizer states for all parameters adam = bnb.optim.Adam(model.parameters(), lr=0.001, optim_bits=8) # 2. override: the parameter model.fc1.weight now uses 32-bit Adam mng.override_config(model.fc1.weight, 'optim_bits', 32) ``` TN) rr$torchnn ParameterTensorr'rupdate)r parametersr valuekey_value_dictr,r r roverride_config7s%  z"GlobalOptimManager.override_configcCs|j|||fdSr)rappend)r module param_nameconfigr r rregister_module_overridelsz+GlobalOptimManager.register_module_override)NNN) rrr__doc__rrr classmethodr!r-r6r;r r r rrs  5rcseZdZdfdd ZddZfddZdd d Zd d ZddZe dddZ ddZ ddZ ddZe jfddZddZZS) Optimizer8bit FcsXt||d|_i|_||_tj|_t |_ hd|_ |dkr*| dSdS)az Base 8-bit optimizer class. Arguments: params (`torch.tensor`): The input parameters to optimize. optim_bits (`int`, defaults to 32): The number of bits of the optimizer state. is_paged (`bool`, defaults to `False`): Whether the optimizer is a paged optimizer or not. F> max1max2qmap1qmap2state1state2absmax1absmax2new_max1new_max2 gnorm_vec unorm_vecN) superr initialized name2qmapis_pagedFGlobalPageManagerr!page_mngrmngnon_castable_tensor_keys fill_qmap)r r"defaults optim_bitsrP __class__r rrqs     zOptimizer8bit.__init__cCs(tjdd|jd<tjdd|jd<dS)NT)signeddynamicFudynamic)rQcreate_dynamic_maprOrr r rrVszOptimizer8bit.fill_qmapcst|dSr)rM __setstate__)r staterYr rr_szOptimizer8bit.__setstate__Tc s(t|}j}|d}t|t|krtddd|D}dd|D}tddt||Dr5tddd ttd d|Dtd d|DD}fd d tt }|d D]\} } | |vru|| } | | || <qa| || <qaddfddt||D}  || ddS)a1Load an optimizer state. Arguments: state_dict (`dict`): An optimizer state (should be returned from a call to `state_dict`) to load. move_to_device (`bool`, defaults to `True`): Whether to move the optimizer's state to the device. r(zz0Optimizer8bit.load_state_dict..csrarbrcrer r rrhricss|] \}}||kVqdSrr )rfp_lens_lenr r rrhriz]loaded state dict contains a parameter group that doesn't match the size of optimizer's groupcSsi|]\}}||qSr r )rfold_idr,r r r sz1Optimizer8bit.load_state_dict..cs|]}|dVqdSrbr rer r rrhcsrnrbr rer r rrhrocst|tjrr|jtjkr|j}|St|tr?|D]\}}|j vr5r4|j ||<q!|||<q!|St|t j rSt |fdd|DS|S)zBMake a deep copy of value, casting all tensors to device of param.c3s|]}|VqdSrr )rfv)castparamr rrhsz>Optimizer8bit.load_state_dict..cast..)r$r.r1is_floating_pointdtypeuint8tor%itemsrUdevicecontainer_abcsIterabletype)rrr4krp)rqmove_to_devicer )rrrrqs     z+Optimizer8bit.load_state_dict..castr`cSs|d|d<|S)Nr"r )r* new_groupr r r update_groups z3Optimizer8bit.load_state_dict..update_groupcsg|] \}}||qSr r )rfrgng)rr r sz1Optimizer8bit.load_state_dict..)r`r(N) rr(rd ValueErroranyzipr from_iterablerr%rwr_) r state_dictr}groups saved_groups param_lens saved_lensid_mapr`r|rprrr(r )rqr}r rrload_state_dicts6  zOptimizer8bit.load_state_dictc Cst|jD];\}}t|dD]0\}}||jvr?|j|}|D]\}}t|tjr>t|dd}|s>||j |j||<q!qqdS)Nr"rPF) r&r(r`rwr$r.r1getattrrvrx) r gindexr*pindexr,valuesr|rprPr r rto_gpus    zOptimizer8bit.to_gpuc Cs|jjD]]\}}}t||}|dusJt|tjs"t|tjs"Jd}t|jD]7\}}|r1n0t|dD](\}} |r?n!t | t |kr_||jj t | <|jj t | |jj ||f<d}q7q)qdS)NFr"T) rTrrr$r.r1r0r&r(r'rr) r r8attrr:pmodulefoundrr*rr,r r rcheck_overridess"  zOptimizer8bit.check_overridesNc Csd}|durt |}Wdn1swYg}|js-||d|_t|jD]:\}}t|dD]/\}}|jdurFq<|j|}t |dkrY| ||||| || ||||tj qps%  J  & r>csneZdZ            dd eed eeffd d ZeddZeddZ Z S)Optimizer2StateMbP??g+?:0yE>r?NdTFrrc sDd|ks td|d|kstd|t|tr1|ddddd}dd |D}tt|D]}d||krEd ksRntd |d ||q7d|ks^td |t|||||||d}t |||||duri}||d<d|d<| |d<| |d<| |d<| |d<| |d<t ||_ n||_ ||_ dS)ah Base 2-state update optimizer class. Arguments: optimizer_name (`str`): The name of the optimizer. params (`torch.tensor`): The input parameters to optimize. lr (`float`, defaults to 1e-3): The learning rate. betas (`tuple`, defaults to (0.9, 0.999)): The beta values for the optimizer. eps (`float`, defaults to 1e-8): The epsilon value for the optimizer. weight_decay (`float`, defaults to 0.0): The weight decay value for the optimizer. optim_bits (`int`, defaults to 32): The number of bits of the optimizer state. args (`object`, defaults to `None`): An object with additional arguments. min_8bit_size (`int`, defaults to 4096): The minimum number of elements of the parameter tensors for 8-bit optimization. percentile_clipping (`int`, defaults to 100): Adapts clipping threshold automatically by tracking the last 100 gradient norms and clipping the gradient at a certain percentile to improve stability. block_wise (`bool`, defaults to `True`): Whether to independently quantize each block of tensors to reduce outlier effects and improve stability. max_unorm (`float`, defaults to 0.0): The maximum value to normalize each block with. skip_zeros (`bool`, defaults to `False`): Whether to skip zero values for sparse gradients and models to ensure correct updates. is_paged (`bool`, defaults to `False`): Whether the optimizer is a paged optimizer or not. alpha (`float`, defaults to 0.0): The alpha value for the AdEMAMix optimizer. t_alpha (`Optional[int]`, defaults to `None`): Number of iterations for alpha scheduling with AdEMAMix. t_beta3 (`Optional[int]`, defaults to `None`): Number of iterations for beta scheduling with AdEMAMix. rInvalid learning rate: Invalid epsilon value: (),cSsg|]}t|qSr )float)rfbr r rrsz,Optimizer2State.__init__..? Invalid beta parameter at index : Invalid weight_decay value: )rrrrrrrNrXrrrrrr)rr$strreplacestripsplitrangerdr%rMrrroptimizer_name)r rr"rrrrrXrrrrrrrPrrrirWrYr rr\s:<   zOptimizer2State.__init__c CsT||||}|ddkrtj}n|ddkrtj}n td|d||dkr/tj}|j|}d|d<|tjkrR|j|tjd|d <|j|tjd|d <n|tjkr|ddkrd |jvrg| |jd  |j |jd <|jd  |j |jd <|j|tjd|d <|jd |d <|j|tjd|d <|jd |d<|dr|}|d} | |ddkrdnd7} tj | ftj|j d|d<tj | ftj|j d|d<n0tj dtj|j d|d<tj dtj|j d|d<tj dtj|j d|d<tj dtj|j d|d<|ddkrtj d|j d|d<|dd kr(tj d|j d|d!<dSdS)"NrXr?rL(Amount of optimizer bits not supported: rrrrtrDrEr\r]rBrCrrrFrGrr@rHrArIrrrrxrJrrrK rr.rrurrr`rrOrVrvrxzeros r r*r,rrr:rtr`nblocksr r rrsL       zOptimizer2State.init_statec Cs|j|_|j|_|j|}|j}||||}|dd7<|d}|ddkr=t||d||d\} } } nd} |djtj krtj |j |||d|dd |d ||d |d |ddt |dd krr|ddnd|d|d| |ddkr|dnd|d|dddS|djtj kr|dstj|j |||d|d |dd |dd|d ||d |d|d|d|d|d|d|d| |ddkr|dnd|dd|d|d|d<|d<|d|d|d<|d<dS|djtj krR|drTtj|j |||d|d |dd |ddt |dd kr/|ddnd|d|d ||d |d|d|d|d|d| |dd dSdSdS)!NrrrrrJrrDrrrrrErrrrrKrrrrrBrCr@rArHrI) gnorm_scalerKrrFrGrr)data contiguousrr`rrQrrtr.roptimizer_update_32bitrrdruoptimizer_update_8bitoptimizer_update_8bit_blockwise r r*r,rrr`rr:r current_gnorm clip_valuerr r rrs              zOptimizer2State.update_step)rrrrr?NrrTrFFrNN) rrrrintrr.rrrrr r rYrr[s2` 3rcsVeZdZ          dfd d Zed dZeddZZS)Optimizer1Staterrrrrr?NrrTFcsd|ks td|d|kstd|tt|D]}d||kr*dks7ntd|d||qd|ksCtd|t||||d}t|||||d ur|i}||d <d |d <| |d <| |d <| |d<| |d<| |d<t||_n||_||_d S)a Base 1-state update optimizer class. Arguments: optimizer_name (`str`): The name of the optimizer. params (`torch.tensor`): The input parameters to optimize. lr (`float`, defaults to 1e-3): The learning rate. betas (`tuple`, defaults to (0.9, 0.0)): The beta values for the optimizer. eps (`float`, defaults to 1e-8): The epsilon value for the optimizer. weight_decay (`float`, defaults to 0.0): The weight decay value for the optimizer. optim_bits (`int`, defaults to 32): The number of bits of the optimizer state. args (`object`, defaults to `None`): An object with additional arguments. min_8bit_size (`int`, defaults to 4096): The minimum number of elements of the parameter tensors for 8-bit optimization. percentile_clipping (`int`, defaults to 100): Adapts clipping threshold automatically by tracking the last 100 gradient norms and clipping the gradient at a certain percentile to improve stability. block_wise (`bool`, defaults to `True`): Whether to independently quantize each block of tensors to reduce outlier effects and improve stability. max_unorm (`float`, defaults to 0.0): The maximum value to normalize each block with. skip_zeros (`bool`, defaults to `False`): Whether to skip zero values for sparse gradients and models to ensure correct updates. is_paged (`bool`, defaults to `False`): Whether the optimizer is a paged optimizer or not. rrrrrrr)rrrrNrXrrrrrr) rrrdr%rMrrrr)r rr"rrrrrXrrrrrrrPrrWrYr rrQs02  zOptimizer1State.__init__c Cs||||}|ddkrtj}n|ddkrtj}n td|d||dkr/tj}|j|}d|d<|tjkrH|j|tjd|d <no|tjkr|ddkrhd |jvr\| |jd  |j |jd <|j|tjd|d <|jd |d <|d r|}|d } | |d dkrdnd7} tj | ftj|j d|d<ntj dtj|j d|d<tj dtj|j d|d<|ddkrtj d|j d|d<|ddkrtj d|j d|d<dSdS)NrXr?rLrrrrrrDr\rBrrrrrFrr@rHrrrrrJrrrKrrr r rrs>         zOptimizer1State.init_statec Cs0|j|_|j|_|j|}|j}||||}|dd7<|d}|ddkr=t||d||d\} } } nd} |djtj kr~tj |j |||d|dd |d ||d d|ddd d |d | |dd krr|dnd|d|dddS|djtj kr|dstj |j |||dd|dd |dd|d ||d |dd|dd|dd|d | |dd kr|dnd|dd|d|d|d<|d<dS|djtj kr|drtj|j |||dd|dd |ddd d |d ||d |dd|dd|d | |dddSdSdS)NrrrrrJrrDrrrrrrrrKrrrrBr@rH)rrFr)rrrr`rrQrrtr.rrrrurrrr r rrs             zOptimizer1State.update_step) rrrrr?NrrTrFF) rrrrr.rrrrr r rYrrPs$N +r) collectionsrryrcopyr itertoolsrtypingrr.bitsandbytes.functional functionalrQrroptim Optimizerr>rrr r r rs    [lv