o i7 @s@dZddlZddlZddlZddlZddlmZddlmZm Z ddl m Z ddl m Z mZmZmZmZmZddl mZddlZddlmZddlmmmZddlZddlmZddlmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(dd l)m*Z*dd l+m,Z,dd l-m.Z.m/Z/dd l0m1Z1dd l2m3Z3m4Z4m5Z5m6Z6m7Z7ddl8m9Z9m:Z:m;Z;mZ>ddl?m@Z@ddlAmBZBddlCmDZDmEZEmFZFmGZGmHZHmIZImJZJmKZKmLZLmMZMmNZNmOZOmPZPmQZQmRZRmSZSddlTmUZUmVZVddlWmXZXmYZYmZZZm[Z[ddl\m]Z]m^Z^m_Z_ddl`maZambZbmcZcmdZdmeZeddlfmgZgmhZhddlimjZjmkZkddllmmZmddlnmoZompZpmqZqmrZrddlsmtZtmuZuddlmvZvewexZygdZzdd iZ{d!d"Z|e%eqj}ge!eqj}ge"eqj}eqj~giZe%d#e!d#e"d#iZ $  %dd&eee d'fd(dfd)d*Zd+d,Zdd-d.Zdd/d0Zdd1d2ejd3ee@d4ee@d5eeejjegeffd6d7Zd8d9Zd:d;Zd#d#d<d=d>Zedfd2ejjd?ed5eeejjegefd@eejjfdAdBZdCejd(ejfdDdEZdCejd(ejfdFdGZdCejd(ejfdHdIZeGdJdKdKeZdddLdMeejdNeejfdOdPZeVedddLdQejjd?edMeejdNeejd(ejjf dRdSZeGdTdUdUeZdVdWZeVedQejjd?ed(ejjfdXdYZeGdZd[d[eZeVedQejjd?ed(ejjfd\d]ZeGd^d_d_eZd`daZeVedbdcdQejjd?eddefdedfZdCejd(ejfdgdhZdCejd(ejfdidjZdCejdkejd(ejfdldmZdCejdkejd(ejejffdndoZeGdpdqdqeZdrdsZeVedbdcdQejjd?ed(ejjfdtduZeGdvdwdweZeVedbdcdQejjd?efdxdyZdzd{ZeGd|d}d}eZd~dZeVedbdcdQejjd?edded(ejjfddZ  ddCejde4dejdeejdeejf ddZeGdddeZddZeVedbdcdQejjd?eddefddZeGdddeZeVedQejjd?efddZdePdejdejd(dfddZeGdddeZdddLdMeejdNeejfddZeVedddLdQejjd?edMeejdNeejd(ejjf ddZeGdddeZeZeeehZdQejjded?efddZded?efddZdQejded?efddZdQe1d(ejfddZejeeeeee*gdS)ap Quantization APIs Generally these APIs can be applied directly to any model with Linear modules to obtain quantized linear ops. The intended usage involves applying torch.compile to the model afterwards both because primitives were designed based on the fusions that come along with it and because that is how we access the intended quantized and mixed GEMM kernels N) OrderedDict) dataclassfield)partial)AnyCallableListOptionalTupleUnion) AOBaseConfig) AffineQuantizedTensorCutlassSemiSparseLayout Float8Layout Int4CPULayout Int4XPULayout PlainLayoutSemiSparseLayoutTensorCoreTiledLayoutto_affine_quantized_floatx!to_affine_quantized_floatx_staticto_affine_quantized_intx)Target)Layout) e4m3_dtype e5m2_dtype) Float8Linear)Float8MMConfigFP8Granularity_check_hardware_support!_granularity_is_a_1_128_w_128_128_normalize_granularity)(Float8StaticActivationFloat8WeightConfigGemliteUIntXWeightOnlyConfig%Int8DynamicActivationInt4WeightConfigUIntXWeightOnlyConfig)$LinearActivationWeightObservedTensor)AffineQuantizedObserverBase)KernelPreference)Float8PackingFormat Float8TensorInt4ChooseQParamsAlgorithmInt4PackingFormatInt4PlainInt32TensorInt4PreshuffledTensor Int4TensorInt4TilePackedTo4dTensor Int8TensorIntxChooseQParamsAlgorithmIntxOpaqueTensorIntxPackingFormatIntxUnpackedToInt8TensorQuantizeTensorToFloat8KwargsQuantizeTensorToInt8KwargsSparse2x4CUTLASSFloat8Tensor)_QUANTIZE_CONFIG_HANDLER register_quantize_module_handler)_fp8_mm_compat_linear_extra_repr_quantization_typeget_block_size)is_MI300is_sm_at_least_89is_sm_at_least_90) GranularityPerAxisPerGroupPerRow PerTensor)LinearActivationQuantizedTensorto_linear_activation_quantized)Int4WeightOnlyQuantizerInt8DynActInt4WeightQuantizer) intx_quantization_aware_training)_DTYPE_TO_QVALUE_BOUNDS MappingTypeZeroPointDomainquantize_affine) QuantizerTwoStepQuantizer)_get_per_token_block_size) swap_conv2d_1x1_to_linearrQrRrJ autoquant_get_subclass_inserter quantize_rLrK3Float8DynamicActivationFloat8SemiSparseWeightConfigModuleFqnToConfigrUz .autoquantcCsB|tvrddl}t|}||t}t||Stdtd|)Nrzmodule z has no attribute ) _lazy_imports importlib import_module __package__getattrAttributeError__name__)namer[ module_pathmodulerdR/home/ubuntu/.local/lib/python3.10/site-packages/torchao/quantization/quant_api.py __getattr__s   rfFrd extra_args.returnc Cs|||ddr|dur|j|d||g|R}|St|}|D] \}}t|||||d||} | |urE| durEt||| q%|durP|j|d|S)a' Recursively replaces each child module in `model` with the result of `replacement_fn(child)` if `filter_fn(child)` returns `True`. Args: model (torch.nn.Module): The model containing modules to be replaced. replacement_fn (Callable[[torch.nn.Module], torch.nn.Module]): The function to replace matching modules. filter_fn (Callable[[torch.nn.Module], bool]): The filter function to determine which modules to replace. cur_fqn (str, optional): The current fully qualified name of the module being processed. Defaults to "". device (device, optional): Device to move the model to before applying `filter_fn`. Defaults to None. extra_args (Tuple[Any, ...], optional): optional extra args to pass to `replacement_fn`. Returns: None Ndevice.)tolistnamed_children)_replace_with_custom_fn_if_matches_filtersetattr) modelreplacement_fn filter_fncur_fqnrlrhnamed_children_listrachild new_childrdrdrerqs*      rqcGszddlm}ddlm}t|tjjo.insert_subclass)pop)rrrrrdrrerVs rVcsBGdddtjjfdd}|durdd}t|||ddS) zi Changes all conv2d 1x1 modules to equivalent linear modules so that they can then be quantized. cs$eZdZfddZddZZS)z2swap_conv2d_1x1_to_linear..PermuteSandwichcst||_dSN)super__init__r)selfr __class__rdrer"s  z;swap_conv2d_1x1_to_linear..PermuteSandwich.__init__cWs&||dddddddddS)NrrB)rpermute)rrrdrdreforward&s&z:swap_conv2d_1x1_to_linear..PermuteSandwich.forward)r` __module__ __qualname__rr __classcell__rdrdrrePermuteSandwich!s rcsR|jdksJtjj|j|j|jdud}tj|j dd|_|j|_|S)NrBrB)biasrj) kernel_sizerrr in_channels out_channelsrrr|squeeze)convrrrdrereplace_conv2d_1x1)sz5swap_conv2d_1x1_to_linear..replace_conv2d_1x1NcWst|tjjo |jdkS)Nr)r~rrConv2dr)rrrdrdre3s z+swap_conv2d_1x1_to_linear..ru)rrModulerq)rsrurrdrrerTs   rTrrsinput_observerweight_observerrucs8dtjffdd }t|||durtdS|dS)a Converts the weight of a linear module to a LinearActivationWeightObservedTensor. This function wraps the weight of the given linear module with a LinearActivationWeightObservedTensor, which enables observation of both input and weight tensors during forward passes. The wrapped weight is then re-wrapped as a nn.Parameter to maintain compatibility with PyTorch's module system. Example:: ``` import torch import torch.nn as nn from torchao.quantization import PerTensor from torchao.quantization.linear_observer_tensor import insert_observers_ from torchao.quantization.observer import ( AffineQuantizedMinMaxObserver, MappingType ) # Create observers input_observer = AffineQuantizedMinMaxObserver( MappingType.SYMMETRIC, torch.float8_e4m3fn, granularity_type=PerTensor(), eps=torch.finfo(torch.float32).eps, scale_dtype=torch.float, zero_point_dtype=torch.int, zero_point_domain=ZeroPointDomain.NONE, ) # Create a linear module linear_module = nn.Linear(10, 20) # Convert the linear module's weight to an observed tensor insert_observers_(linear_module, input_observer, weight_observer=None) # The linear_module can now be used as usual, with observers calculating statistics output = linear_module(torch.randn(10, 10)) # Get the scale and zero point of the input observer scale, zero_point = linear_module.weight.input_observer.calculate_qparams() ``` Args: model (nn.Module): The nn.Module to convert. input_observer (Optional[AffineQuantizedObserverBase]): Observer for input tensor. weight_observer (Optional[AffineQuantizedObserverBase]): Observer for weight tensor. filter_fn (Optional[Callable[[torch.nn.Module, str], bool]]): Filter function to select which modules to convert. If not provided, all linear modules will be converted. This function should take a module and its fully qualified name. Returns: nn.Linear: The modified linear module with its weight wrapped in a LinearActivationWeightObservedTensor. linear_modulecs&tjtj|jd|jjd|_|S)Nrrr)rrr&rr|r)rrrdreconvert_to_linear_observerzsz5insert_observers_..convert_to_linear_observerN)rrrqr)rsrrrurrdrreinsert_observers_<s>  rcCs,d|jjdd|jjddt|jS)Nznum_embeddings=rz, embedding_dim=rBz , weight=)r|shaper=rrdrdre_embedding_extra_reprs,rcCsHg}|}t|dkr||||dtt||d|S)Nr=z, )lenappendr=r^join)roriginal_extra_reprparameter_namemodule_torchao_extra_reproriginal_extra_repr_strrdrdre_module_extra_reprs   r)allow_requires_gradpropagate_biasc sfdd}|S)zHelper function to apply the constructor that quantizes the weight Tensor (with additional kwargs) to the weight of linear module csPo|jj}dkr|jd<tjj|jfi|d|_tt||_ |S)NTrr) r|rrrrrtypes MethodTyper< extra_repr)rrrrrrrdrers  z6_get_linear_subclass_inserter..insert_subclassrd)rrrrrrdrre_get_linear_subclass_inserters rconfigrlc Cstjdt|tri|durtdt|}|D]I\}}t ||s5t |||s5d|j vrft |rft |||}|durE|j|d}||urf|dkrf|dd}||d} || } t| ||qdSt|tr|durtt n|}tt|} t|| |||fd dStd ) aConvert the weight of linear modules in the model with `config`, model is modified inplace Args: model (torch.nn.Module): input model config (AOBaseConfig): a workflow configuration object. filter_fn (Optional[Callable[[torch.nn.Module, str], bool]]): function that takes a nn.Module instance and fully qualified name of the module, returns True if we want to run `config` on the weight of the module device (device, optional): Device to move module to before applying `filter_fn`. This can be set to `"cuda"` to speed up quantization. The final model will be on the specified `device`. Defaults to None (do not change device). Example:: import torch import torch.nn as nn from torchao import quantize_ # quantize with some predefined `config` method that corresponds to # optimized execution paths or kernels (e.g. int4 tinygemm kernel) # also customizable with arguments # currently options are # Int8DynamicActivationInt8WeightConfig (optimized with int8 mm op and torch.compile) # Int4WeightOnlyConfig (optimized with int4 tinygemm kernel and torch.compile) # Int8WeightOnlyConfig (optimized with int8 mm op and torch.compile from torchao.quantization.quant_api import Int4WeightOnlyConfig m = nn.Sequential(nn.Linear(32, 1024), nn.Linear(1024, 32)) quantize_(m, Int4WeightOnlyConfig(group_size=32)) ztorchao.quantization.quantize_NzuCustom filter_fn and FqnToConfig were both specified. Only filter_fn=None is supported when FqnToConfig is specified._defaultrkrgrmrj)rlrhzPassing a generic Callable to `quantize_` is no longer recommended and will be deprecated at a later release. Please see https://github.com/pytorch/ao/issues/1690 for instructions on how to pass in workflow configuration instead.)r_C_log_api_usage_oncer~ FqnToConfig ValueErrordict named_modulesitemsfqn_matches_fqn_config _module_param_matches_fqn_config fqn_to_configr_fqn_to_config_handlerrnsplit removesuffixrrr r9typerqAssertionError) rsrrurlr module_fqnrc replacement child_name parent_fqn parent_modulehandlerrdrdrerWsH #         rWxc Cs@tj}tj}tj}ttjj}tj}t||t|||||dS)zGThis is defined here instead of local function to support serialization)eps scale_dtypezero_point_dtype) rN ASYMMETRICrint8float32finforrrS)r mapping_type target_dtyperrrrdrdre_int8_asymm_per_token_quantsrc CsPtj}tj}tj}ttjj}tj}d}d}t||t |||||||d }|S)Nr) quant_min quant_maxrrr) rNrruint8rrrint32rrS) rrrrrrrroutrdrdre_uint8_asymm_per_token_quants& rc Cs6tj}tj}d}d}d}t||t|||||tjdSNh㈵>rrrr)rN SYMMETRICrrrrSrrrrrrrrdrdre_int8_symm_per_token_quant%src@seZdZUdZejZejed<e dZ e ed<e j Ze ed<dZeejed<e jZe ed<ejZeed <ejZeed <d Zeed <d dZdS)%Int8DynamicActivationIntxWeightConfiga| Configuration for dynamically quantizing activations to torch.int8 and weights to torch.intx, with 1 <= x <= 8. More specifically, activations are dynamically quantized to 8-bits at a per-token granularity with scales/zeros. Weights are quantized with scales/zeros in a groupwise or channelwise manner using the number of bits specified by weight_dtype. This layout is identical to Int8DynamicActivationInt4WeightConfig when weight_dtype is torch.int4 and other args are the same. However, this layout is more general and supports other weight dtypes. args: `weight_dtype`: The dtype to use for weight quantization. Must be torch.intx, where 1 <= x <= 8. ` weight_granularity`: The granularity to use for weight quantization. Must be PerGroup or PerAxis(axis=0). `weight_mapping_type`: The type of mapping to use for the weight quantization. Must be one of MappingType.ASYMMETRIC or MappingType.SYMMETRIC. MappingType.SYMMETRIC requires ZeroPointDomain.NONE `weight_scale_dtype`: The dtype to use for the weight scale. `act_mapping_type`: The type of mapping to use for the activation quantization. Must be one of MappingType.ASYMMETRIC or MappingType.SYMMETRIC. `intx_packing_format`: The format to use for the packed weight tensor (version 2 only). - unpacked_to_int8: this format is the default and is intended for export applications like ExecuTorch. - opaque_torchao_auto: this format is optimized for CPU performance. `intx_choose_qparams_algorithm`: The algorithm to use for choosing the quantization parameters. `version`: version of the config to use, only subset of above args are valid based on version, see note for more details. Example: .. literalinclude:: ../../examples/inference/int8_dynamic_activation_intx_weight.py :language: python weight_dtype weight_granularityweight_mapping_typeNweight_scale_dtypeact_mapping_typeintx_packing_formatintx_choose_qparams_algorithmrversioncCstjd|jddtddDvsJd|jt|jttfs+Jd|jt|jtr@|jj dks@Jd |jj |j t j t j t jfvsSJd |j |jt j t j fvsdJd |jdS) Nz:torchao.quantization.Int8DynamicActivationIntxWeightConfigcSg|] }ttd|qSintr^r.0brdrdre gzGInt8DynamicActivationIntxWeightConfig.__post_init__..rB .rBrzBInt4ChooseQParamsAlgorithm.HQQ is not supported by packing format zF, it's only supported by Int4PackingFormat.TILE_PACKED_TO_4D currentlyactivation_dtype)r9z!Unsupported int4 packing format: )r'r9r8rloggerinfotuplerndimr ror+HQQr,TILE_PACKED_TO_4D PRESHUFFLEDr.r,rbfloat16r>r/ PLAIN_INT32r-r0r)r|rr'r9r8r/r1rdrdre!_int4_weight_only_quantize_tensorsT"       rOcCsT|jr tjjt|dsJdt|j|}tj j |dd|_t t ||_|S)Nr|gapplying int8 weight only quant requires module to have weight attribute but {module} does not have oneFr)r7torchao quantizationutils"recommended_inductor_config_setterrrOr|rrrrrr<r)rcrr1rdrdre_int4_weight_only_transform=s   rUc@seZdZUdZdZeed<dS)'Float8DynamicActivationInt4WeightConfigaConfiguration for apply float8 dynamic per row quantization and int4 per group weight quantization to linear (only group_size 128 is supported right now since underlying kernel used only supports 128 and above and no benefits of making it bigger) Args: `int4_packing_format`: how the weight is packed, only preshuffled is supported Example: .. literalinclude:: ../../examples/inference/float8_dynamic_activation_int4_weight.py :language: python preshuffledr8N)r`rrrr8r,rrdrdrdrerVNs rVcCst|ds Jd|j}|dksJd||j}d}tddt|jdD|g}tj|j|tj d }tj j |d d |_t t||_|S) Nr|rPrWz?only preshuffled int4_packing_format supported right now, got: r6cSr@rArdrBrdrdrerprCzD_float8_dynamic_activation_int4_weight_transform..rBrDFr)rr8r|rHrrIr.r,r float8_e4m3fnrrrrr<r)rcrr8r|r'r/r1rdrdre0_float8_dynamic_activation_int4_weight_transformas$  "rYc@sTeZdZUdZdZeeed<eZ ee ed<dZ e ed<dZ eed<d d ZdS) Int8WeightOnlyConfiga Configuration for applying int8 weight-only symmetric per-channel quantization to linear layers. Args: group_size (version 1) - Controls the granularity of quantization. If None, applies per-channel quantization. Otherwise, applies per-group quantization with the specified group size. granularity (version 2) - Quantization granularity. PerRow() for per-channel quantization, PerTensor() for per-tensor quantization. set_inductor_config: bool = True - If True, adjusts `torchinductor` settings to recommended values for better performance with this quantization scheme. Example: .. literalinclude:: ../../examples/inference/int8_weight_only.py :language: python Nr' granularityTr7rBr cCs8tjd|jdkr|jdusJd|jdSdS)Nz)torchao.quantization.Int8WeightOnlyConfigrz1Only support version 2 with group_size=None, got )rrrr r'rrdrdrers    z"Int8WeightOnlyConfig.__post_init__)r`rrrr'r r rrFr[rCr7r=r rrdrdrdrerZ{s    rZc Cs|jdkrDtdtj}tj}ttjj }tj }|j }|dur&|j d}t ddt|dD|g}t||||||d}|S|jdksQJd|jtj||jd }|S) NrBzConfig Deprecation: version 1 of Int8WeightOnlyConfig is deprecated and will no longer be supported in a future release, please use version 2, see https://github.com/pytorch/ao/issues/2752 for more detailsrjcSr@rArd)rrrdrdrerrCz5_int8_weight_only_quantize_tensor..)rrrUnexpected version: )r[)r warningswarnrNrrrrrrint64r'rrHrr&rr1r,r[) r|rrrrrr'r/r1rdrdre!_int8_weight_only_quantize_tensors0  $ r`r|rrcCsj|jr tjjt||sJdtt|||}t||t j j |ddt tt|j|d||_|S)Nzqapplying int8 weight only quant requires module to have {parameter_name} attribute but {module} does not have oneFrrr)r7rQrRrSrTrr`r^rrrrrrrrrrrcrrquantized_tensorrdrdre_int8_weight_only_transforms,   rec CsJtj}tj}d}d}d}t||t||||||jtjkr!tjdSddSr) rNrrrrrSrfloat16rrrdrdre(_int8_symm_per_token_reduced_range_quants"rgc Cs\tj}tj}d}d}d}|jddkr|St||t||||||jtjkr*tj dSddS)NrrrrBr) rNrrrrrrSrrfrrrdrdre4_int8_symm_per_token_reduced_range_quant_noop_decodes&rhrcCst|t|tj|tdddS)N mm_configr/rr_layout)rrSrrrrrrdrdre_float8_cutlass_quantsrncCst|t|tj|tdS)Nrk)rrSrrrrmrdrdre_float8_cutlass_quant_sparse sroc@speZdZUdZeZeeed<e j Z ee ed<dZ e ed<eZeed<dZe ed<d Zeed <d d Zd S)%Int8DynamicActivationInt8WeightConfigae Configuration for applying int8 dynamic symmetric per-token activation and int8 per-channel weight quantization to linear layers. Args: layout: Optional[Layout] = PlainLayout() - Tensor layout for the quantized weights. Controls how the quantized data is stored and accessed. act_mapping_type: Optional[MappingType] = MappingType.SYMMETRIC - Mapping type for activation quantization. SYMMETRIC uses symmetric quantization around zero. weight_only_decode: bool = False - If True, only quantizes weights during forward pass and keeps activations in original precision during decode operations. set_inductor_config: bool = True - If True, adjusts `torchinductor` settings to recommended values for better performance with this quantization scheme. version (int): the version of the config, version 1 is using AffineQuantizedTensor that we plan to deprecate/split, version 2 is using Int8Tensor Example: .. literalinclude:: ../../examples/inference/int8_dynamic_activation_int8_weight.py :language: python layoutrFweight_only_decoder[Tr7rBr cCr:)Nz:torchao.quantization.Int8DynamicActivationInt8WeightConfigr;rrdrdrer8z3Int8DynamicActivationInt8WeightConfig.__post_init__N)r`rrrrrqr rrrNrrrrr=rFr[rCr7r r rrdrdrdrerps     rpc Cs6|jdkrb|j}|j}|j}|jd}|dkr%td|jd||Stj}t j }dd}t j } t t jj} t j} |rAt} n |tjkrIt} nt} ||} t||| | | | ||d}t|| }|S|jtthvsoJd |j}|j}|jtjksJd |jd ksJd |jtj||t||jd d}|S)NrBrjzKSkipping applying Int8DynamicActivationInt8WeightConfig to weight of shape z because `in_feature` is <= 16: cSs*tddt|dD|jdgS)NcSr@rArdrBrdrdrerRrCzg_int8_dynamic_activation_int8_weight_quantize_tensor..get_weight_block_size..rBrj)rHrr&r)rrdrdreget_weight_block_sizeQs*zS_int8_dynamic_activation_int8_weight_quantize_tensor..get_weight_block_size)rrrlzero_point_domainz'Only PerRow and PerTensor are supportedz0asymmetric dynamic quant not supported currentlyrr\r[r)r[act_quant_kwargs)r rqrrrrrFrGrNrrONONErrrrrr_rhrgrrrIr[rFrGr1r,r7)r|rrqrrr in_featuresrweight_zero_point_domainrurrrinput_quant_funcr/r1quantized_weightract_granularityrdrdre4_int8_dynamic_activation_int8_weight_quantize_tensor>sl      rcCs`|jr tjjt|dsJdt|j|}tj j |dd|_t t t|j|d||_|S)Nr|zyapplying int8 dynamic activation int8 weight quant requires module to have weight attributebut {module} does not have oneFrrb)r7rQrRrSrTrrr|rrrrrrrr)rcrrr1rdrdre._int8_dynamic_activation_int8_weight_transforms$  rc@sreZdZUdZdZeejed<e Z e ed<e j Zee ed<dZeed<dZeed <d d Zd efd dZdS)$Int8StaticActivationInt8WeightConfigaM Configuration for applying int8 static symmetric quantization to both activation and weight Args: act_quant_scale (torch.Tensor): The scale tensor for activation quantization. granularity (Granularity): The granularity of quantization. PerRow() and PerTensor() are supported currently act_mapping_type (MappingType): The mapping type for activation quantization. only SYMMETRIC is supported currently set_inductor_config (bool): if True, adjusts `torchinductor` settings to recommended values. version (int): the version of the config Nact_quant_scaler[rTr7rBr cCs@tjdt|jtr|jjdkrtd|jjddSdS)Nz9torchao.quantization.Int8StaticActivationInt8WeightConfigrjznInt8StaticActivationInt8WeightConfig only supports PerRow(dim=-1) for activation quantization, got PerRow(dim=zS). Per-feature activation quantization is not supported due to slicing limitations.)rrrr~r[rFr&rrrdrdrersz2Int8StaticActivationInt8WeightConfig.__post_init__ricCst|j|jdS)zGet the activation quantization kwargs for static quantization. Returns: QuantizeTensorToInt8Kwargs with the configured granularity and mapping type. rw)r7r[rrrdrdreget_act_quant_kwargssz9Int8StaticActivationInt8WeightConfig.get_act_quant_kwargs)r`rrrrr rTensorrrFr[rCrNrrr7r=r r rr7rrdrdrdrers     rcCs|jtthvs Jd|jtjksJdt||s$Jd|d|jr-tj j |j}|j}t j t|||t||jd|jd}t||tjj|ddttt|j|d ||_|S) Nz0Only PerRow and PerTensor is supported currentlyz/asymmetric static quant not supported currentlyz#Expected module to have attribute `z` but not foundrw)r[rxrFrrb)r[rFrGrrNrrr7rQrRrSrTr1r,r^r7rdetachrrrrrrrrrr)rcrractivation_granularityrrdrdrdre-_int8_static_activation_int8_weight_transformsH    rcCstdttdS)z Applies int8 dnynamic symmetric per-token activation and int8 per-channel weight quantization + 2:4 sparsity to linear layers. aint8_dyanmic_activation_int8_semi_sparse_weight() will be deprecated at a later release. Please use the layout kwarg in Int8DynamicActivationInt8WeightConfig instead. from torchao.dtypes import SemiSparseLayout Int8DynamicActivationInt8WeightConfig(layout=SemiSparseLayout())rq)r]r^rprrdrdrdre/int8_dynamic_activation_int8_semi_sparse_weights rc@s@eZdZUdZeZejed<dZ e ed<dZ e ed<ddZ d S) Float8WeightOnlyConfiga Configuration for applying float8 weight-only symmetric per-channel quantization to linear layers. Args: weight_dtype (torch.dtype): The target data type for weight quantization. Default is torch.float8_e4m3fn. set_inductor_config (bool): if True, adjusts `torchinductor` settings to recommended values. version (int): the version of the config, version 1 is deprecated, version 2 is using Float8Tensor (default) Note: The actual matmul will be computed in original precision of the weight tensor. Example: .. literalinclude:: ../../examples/inference/float8_weight_only.py :language: python rTr7rr cCr:)Nz+torchao.quantization.Float8WeightOnlyConfigr;rrdrdrer'r<z$Float8WeightOnlyConfig.__post_init__N)r`rrrrrrrrr7r=r r rrdrdrdrers    rcCs6|jdks Jd|j|j}tj||td}|S)Nrr\) float8_dtyper[)r rr*r,rF)r|rrr1rdrdre _float8_weight_only_quant_tensor+s rcCs||jr tjjt||sJdt|trt|}t t |||}t ||t j j|ddttt|j|d||_|S)Nzsapplying float8 weight only quant requires module to have {parameter_name} attribute but {module} does not have oneFrrb)r7rQrRrSrTrr~r_unwrap_float8_linearrr^rrrrrrrrrrrcrdrdre_float8_weight_only_transform4s0    rrrEscale zero_pointcCs|dusJdt|tr|jtjksJdt|j|}|dur0t|||tjt ddd}|St|t s9Jdt ||||t ddd}|S)zThis function is used to quantize the input activation tensor for an aqt_float variant. If scale is not provided it will be dynamically calculate the scales otherwise it will use the provided scale. Nz8Zero point is not supported for dynamic FP8 quantizationzFPerRow quantization only works for bfloat16 precision input activationri) input_floatr/rrrlz7Static quantization only supports PerTensor granularity)rr/rrrl) r~rFrrrMr>rrrrrGr)rrrErrr/ activationrdrdre _input_activation_quant_func_fp8Zs8    rc@seZdZUdZeZejed<eZ ejed<dZ e e e ee fed<ejZe eed<dZe eed<dZe eed<dZe eed <ejZeed <d Zeed <d Zeed<ddZdS))Float8DynamicActivationFloat8WeightConfiga Configuration for applying float8 dynamic symmetric quantization to both activations and weights of linear layers. Args: activation_dtype (torch.dtype): The target data type for activation quantization. Default is torch.float8_e4m3fn. weight_dtype (torch.dtype): The target data type for weight quantization. Default is torch.float8_e4m3fn. granularity (Optional[Union[FP8Granularity, List[FP8Granularity]]]): The granularity for quantization. Can be either a single granularity (applied to both activations and weights) or a tuple of two granularities (one for activations, one for weights). If None, defaults to PerTensor for both. Currently both quantizations need to be the same type. And only PerTensor and PerRow are supported. mm_config (Float8MMConfig): Configuration for the matrix multiplication. Default uses fast accumulation. activation_value_lb (Optional[float]): the lower bound for activation value for calculating scale activation_value_ub (Optional[float]): the upper bound for activation value for calculating scale kernel_preference (KernelPreference): kernel preference for ops like matmul, grouped matmul etc. by defalut (KernelPreference.AUTO) it will be chosen for user based on hardware or other information, this only needs to be set in weight set_inductor_config (bool): if True, adjusts `torchinductor` settings to recommended values. version (int): the version of the config, version 1 is deprecated, version 2 is using Float8Tensor (default) Example: .. literalinclude:: ../../examples/inference/float8_dynamic_activation_float8_weight.py :language: python rErNr[packing_formatrjactivation_value_lbactivation_value_ubkernel_preferenceTr7rr cCstjdt|j\}}||g|_d}t|jr1|jtjtj fvs&Jd|j dks/Jdd}tj r8d}|j durEt|d|_ dSdS)Nz>torchao.quantization.Float8DynamicActivationFloat8WeightConfigT unimplementedrF)use_fast_accum)rrrr!r[r rr(AUTOTORCHr xpu is_availablerjr)rrrdefault_use_fast_accumrdrdrers,    z7Float8DynamicActivationFloat8WeightConfig.__post_init__)r`rrrrrErrrrr[r r rrr)r>rrjrrfloatrr(rrr7r=r r rrdrdrdrers    rcCsL|j}|j}|j}|j}|j}|j}|j}|j} t||\} } | dvrIt | t r0t | t s4Jd|j dddksF|j dddkrH|Snt |sO|S|jdks\Jd|j| tjkrpt | trp|jtjkspJdt|| |||d } | tjkrtj||| ||| d } | S| tjkrt | tsJd tj||| | d } | SdS) N)zH4D/5D tensor only supports per tensor activation and weight quantizationrrtrBrr\zBPerRow quantization only works for bfloat16 precision input weight) hp_value_lb hp_value_ubr)rr[rjrrxz8Sparse packing format only supports per-row quantization)rr[rx)rErr[rjrrrrrr&r~rGrr;r r)r>rFrrrMr6r*r,SPARSE_CUTLASSr8)r|rrErr[rjrrrrrrrxr}rdrdre8_float8_dynamic_activation_float8_weight_quantize_tensorsr  $    rcCstjrtstsJd|jrtjj t ||s+Jd|dd|dt |t r4t |}tt|||}t||tjj|ddttt|j|d||_|S) NzPFloat8 dynamic activation quantization is only supported on CUDA>=8.9 and MI300+zKapplying float8 dynamic activation quant requires module to have parameter z attributez but z does not have oneFrrb)rcudarr@r?r7rQrRrSrTrr~rrrr^rrrrrrrrrrcrdrdre2_float8_dynamic_activation_float8_weight_transform s<       rc@sDeZdZUdZeZeed<eZ e j ed<e Z e j ed<ddZdS)rXa Applies float8 dynamic quantization to activations and float8 quantization followed by compression to sparse semi-structured tensor to weights of linear layers. Args: `layout`: layout type for quantized weight tensor, only supports `CutlassSemiSparseLayout` at the moment. `activation_dtype`: data type for quantized activation tensor. `weight_dtype`: data type for quantized weight tensor. rqrErcCr:)NzHtorchao.quantization.Float8DynamicActivationFloat8SemiSparseWeightConfigr;rrdrdrer?rszAFloat8DynamicActivationFloat8SemiSparseWeightConfig.__post_init__N)r`rrrrrqrrrrErrrrrrdrdrdrerX0s   rXcCstdts Jdt|trt|}|j}|j}|j}|j }t|t s.t d|dt ||}t |td|id}tjj|dd|_tt||_|S) NaWConfig Deprecation: Float8DynamicActivationFloat8SemiSparseWeightConfig is deprecated and will no longer be supported in a future release. Please use Float8DynamicActivationFloat8WeightConfig with packing_format=Float8PackingFormat.SPARSE_CUTLASS and granularity=PerRow() instead. See https://github.com/pytorch/ao/issues/3594 for more detailsz2Float8 quantization is only supported on CUDA>=9.0z;Only CutlassSemiSparseLayout layout is supported. Received rmr) quant_kwargsFr)r]r^rAr~rrr|rrErqrNotImplementedErrorrorIrnrrrrrr<r)rcrr|rrErqrdrdre>_float8_dynamic_activation_float8_semi_sparse_weight_transformEs.    rintx_unpacked_tensor hp_tensorrc CsNt|tsJ|j||_t|j\}}t||j|j|jt j ||d|_ dS)ad Adjusts the scale_dtype on IntxUnpackedToInt8Tensor. Updating the scale dtype requires updating the qdata because qdata is calculated after the scale. This is used in IntxWeightOnlyConfig and Int8DynamicActivationIntxWeightConfig to make version=2 and version=1 numerically equivalent when the scale_dtype differs from the input dtype ) output_dtyperrN) r~r5rrnrMrrPr/rrrqdata)rrrqminqmaxrdrdrer-is  r-c@seZdZUdZejZejed<e dZ e ed<e j Ze ed<dZeejed<ejZeed<ejZeed <d Zeed <d d ZdS)IntxWeightOnlyConfigaE Configuration for quantizing weights to torch.intx, with 1 <= x <= 8. Weights are quantized with scales/zeros in a groupwise or channelwise manner using the number of bits specified by weight_dtype. args: `weight_dtype`: The dtype to use for weight quantization. Must be torch.intx, where 1 <= x <= 8. `granularity`: The granularity to use for weight quantization. Must be PerGroup or PerAxis(0). `mapping_type`: The type of mapping to use for the weight quantization. Must be one of MappingType.ASYMMETRIC or MappingType.SYMMETRIC. `scale_dtype`: The dtype to use for the weight scale. `intx_packing_format`: The format to use for the packed weight tensor (version 2 only). `intx_choose_qparams_algorithm`: The algorithm to use for choosing the quantization parameters. `version`: version of the config to use, only subset of above args are valid based on version, see note for more details. Example: .. literalinclude:: ../../examples/inference/intx_weight_only.py :language: python rrr[rNrrr rr cCstjd|jddtddDvsJd|jt|jttfs+Jd|jt|jtr@|jj dks@Jd |jj |j t j t j t jfvsSJd |j dS) Nz)torchao.quantization.IntxWeightOnlyConfigcSr r rrrdrdrerrz6IntxWeightOnlyConfig.__post_init__..rBrrz1granularity must be PerAxis or PerGroup, but got rr"zvmapping_type must be MappingType.ASYMMETRIC, MappingType.SYMMETRIC, or MappingType.SYMMETRIC_NO_CLIPPING_ERR, but got )rrrrrr~r[rDrErrrNrrrrrdrdrers$      z"IntxWeightOnlyConfig.__post_init__)r`rrrrrrrrrDr[rCrNrrrr r4rrr2rr r r rrdrdrdrers   rc Cs^|j}|j}|j}|j}|j}|j} |dkrd} n|dkr$d} n td|t|t r6|j } nt|t rN|j dksHJd|j |j | } ntd||dkr`d| f} n|dkshJd| ddf} |jdksuJ|jtjkr|dur|jtjkr|tj}tj|| ||||| d } |dur||jkrt| ||| Std |) NrrjrrBz>IntxWeightOnlyConfig only works for 2-d and 4-d Tensors, got: rr"z-granularity must be PerGroup or PerAxis, got )rr r!r r#)rr[rrrr r&rr~rEr'rDrrr r4rrrrrnrr5r,r-)r|rr r!rr[rrrr  input_dimr'r/r1rdrdre!_intx_weight_only_quantize_tensorsT              rcCsrt|ds Jdt|j|||d}tjj|dd|_t|tjr*t t ||_ |St|tj r7t t ||_ |S)Nr|zgapplying intx weight only quant requires module to have weight attribute but {module} does not have onerFr)rrr|rrrr~rrrr<r Embeddingr)rcrr r!r1rdrdre_intx_weight_only_transforms   rc@sjeZdZUdZeedZeee e fe d<eedZ eee e fe d<dZ ee d<ddZd d Zd S) ra Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). Args: `fqn_to_config`: typing.OrderedDict[str, Optional[AOBaseConfig]]: an ordered dictionary from (1). fully qualified name (fqn) of module or parameter (2). regex of fully qualified name (in python `re` module regex format), should start with prefix "re:" or (3). "_default" to the config that we want to apply to the module/param or None Config key ordered by precedence: * fully qualified parameter name, e.g. `language.layers.0.q_proj.weight` * fully qualified module name, e.g. `language.layers.0.q_proj` * regex for parameter names, must start with `re:`, e.g. `re:language\.layers\..+\.q_proj.weight`. The first regex that matches will be applied. * regex for module names, must start with `re:`, e.g. `re:language\.layers\..+\.q_proj`, whichever regex fully matches the module fqn first will be applied (order of keys for dictionary are kept consistent since we are using OrderedDict) * "_default", fallback if no match for all previous keys (Note, when using `_default`, the config is applied to all modules, to apply it to only a subset of modules, e.g. with some types, it's better to filter the modules that we don't want to quantize before hand and configure them to None, e.g. `{"re:.+norm.+": None, "_default": linear_config}`) "_default" is not supported when filter_fn is not specified. `module_fqn_to_config`: typing.OrderedDict[str, Optional[AOBaseConfig]]: To maintain BC with ModuleFqnToConfig, to be deprecated later `version`: int: Version of config to use. Note: - The order of patterns in the OrderedDict may matter as only the first matching pattern is applied - "_default" is ignored for parameter replacement. )default_factoryrmodule_fqn_to_configrBr cCstjdt|jdkrt|jdkr|j|jkrtdt|jdkr0t|jdkr0|j|_t|jdkrBt|jdkrB|j|_d|jvrNtddSdS)Nz torchao.quantization.FqnToConfigrzP`fqn_to_config` and `module_fqn_to_config` are both specified and are not equal!rzConfig Deprecation: _default is deprecated and will no longer be supported in a future release. Please see https://github.com/pytorch/ao/issues/3229 for more details.) rrrrrrrr]r^rrdrdrer<s   zFqnToConfig.__post_init__cCs$ddgdd|jDdS)N z FqnToConfig({css&|]\}}d|d|dVqdS)z 'z': ,Nrd)rkeyvaluerdrdre Ys  z&FqnToConfig.__str__..z}))rrrrrdrdre__str__UszFqnToConfig.__str__N)r`rrrrrrOrderedDictTypestrr r rrr r rrrdrdrdrers    rfqnc Cs6d}g}tt|D]$\}\}}|t|vr0t|dkr%|d|n|}|||||fq t|D]9\}}}}||jvrnd}|j|} | durQ||q5tt | } t | t vre| || |d}q5t t | dq5|s||jvr|j|} | durtt | } | || S|S|D]C\}}}}|jD]9} | drt | d d|rd}|j| } | durtt | } t | t vr| || |d}qt t | dqq|s|jD]'} | drt | d d|r|j| } | durtt | } | || Sq|s|jd d} | durtt | } | || S|S) awThis function expects a module that either is specified in FqnToConfig or has a parameter that is specified in FqnToConfig. Args: module (torch.nn.Module): The module to be processed. fqn (str): The fully qualified name of the module containing the parameters. config (FqnToConfig): Configuration object containing regex patterns / fqn mapped to quantization configurations. Returns: torch.nn.Module: The modified module with quantized parameters. Raises: NotImplementedError: If the quantization configuration is not yet supported for parameter quantization. FrrmTNrazs does not yet support parameter quantization! Please see https://github.com/pytorch/ao/issues/3252 for more detailsre:rr) enumerateronamed_parametersdirrrrrr9r+CUSTOM_PARAM_QUANTIZATION_SUPPORTED_CONFIGSr startswithre fullmatchget) rcrrparameter_config_foundtop_level_paramsirparam parameter_fqncrpatternrdrdrerosn                     rcCs\||jvr|drJd|ddS|jD]}|dr+t|dd|r+dSqdS)aeCheck if a given fqn matches the exact fqn or regex pattern specified in FqnToConfig. Args: fqn (str): The fully qualified name of the module. config (FqnToConfig): Configuration object containing regex patterns or raw FQNs for quantization. Returns: bool: True if the fqn is specified in FqnToConfig. False otherwise. rzError: Exact match but regex z specified.TrNF)rrrr)rr!maybe_module_or_param_fqn_patternrdrdrers    rcCsP|D]!\}}|t|vr%t|dkr|d|n|}t||r%dSqdS)aCheck if a given module contains top-level parameters that match the exact fqn or regex pattern specified in FqnToConfig. Args: module (nn.Module): The module to be checked. fqn (str): The fully qualified name of the module. config (FqnToConfig): Configuration object containing regex patterns or raw FQNs for quantization. Returns: bool: True if the module contains top-level parameters that match the fqn or regex pattern specified in FqnTo rrmTF)rrrr)rcrrrarrrdrdrers  rcCsNtdt|j|j}Wdn1swY|j|_|j|_|S)aY Unwrap a torchao Float8Linear by returning a nn.Linear with the same weights and bias. Torchao inference quantization techniques are generally only applicable to nn.Linear layers, so this helper is useful for unwrapping models trained with torchao float8 training, which replaces nn.Linear layers with Float8Linear layers. metaN)rrlrrrz out_featuresr|r)rc new_modulerdrdrers r)rgNrd)Fr)NN)rloggingrrr] collectionsr dataclassesrr functoolsrtypingrrrr r r rrtorch.nnrtorch.nn.utils.parametrizerSrrQtorchao.core.configr torchao.dtypesr rrrrrrrrrrMtorchao.dtypes.uintx.packed_linear_int8_dynamic_activation_intx_weight_layoutrtorchao.dtypes.utilsrtorchao.float8.configrrtorchao.float8.float8_linearrtorchao.float8.inferencerrrr r!(torchao.prototype.quantization.quant_apir"r#r$r%=torchao.quantization.linear_activation_weight_observed_tensorr&torchao.quantization.observerr'%torchao.quantization.quantize_.commonr((torchao.quantization.quantize_.workflowsr)r*r+r,r-r.r/r0r1r2r3r4r5r6r7r8%torchao.quantization.transform_moduler9r:torchao.quantization.utilsr;r<r=r> torchao.utilsr?r@rAr[rCrDrErFrG"linear_activation_quantized_tensorrHrIlinear_quant_modulesrJrKqatrLquant_primitivesrMrNrOrPunifiedrQrRrS getLoggerr`rF__all__rZrfFLOATINTLAYOUT_TO_ZERO_POINT_DOMAINLAYOUT_TO_PRESERVE_ZEROSrqrrVrTrrr=rrrrDevicerWrrrrrr3r4r5rOrUrVrYrZr`rergrhrrnrorprrrrrrrrrrrrrXrr-rrrrrYrrrrrr serializationadd_safe_globalsrdrdrdresZ       4     H       .  "% Q   KL J "9          #I + 0  ) )=H & # 8 > Q \