o iP@sUddlmZmZmZmZddlZddlmZddlm Z m Z ddl m Z m Z ddlmZejjZejjZejjZiZeeefed<   d=d ejd ejd ejd ejd ejdeejdeejdedejfddZddZddZeejjejjejjej jej!j"ej#jgd>ddZ$eej%jgd>ddZ&eej'jej(j)gd>ddZ*eej+jgd>ddZ,eej-jgd>d d!Z.eej/jgd>d"d#Z0eej1j2gd>d$d%Z3d&e d'e fd(d)Z4eej5jej6jgd>d*d+Z7eej8jgd>d,d-Z9eej:jgd>d.d/Z;eejd0d1Z=eej>jej>jgd>d2d3Z?eej@jej@jgd>d4d5ZAed6rCeejBjgd>d7d8ZCeejDjgd>d9d:ZEeejFjgd>d;d<ZGdS)?)AnyDictOptionalTupleN)tree_map)Float8TrainingTensorchoose_scaled_mm_config) is_row_majorpad_tensor_for_matmul)torch_version_at_leastFLOAT8_OPS_TABLEFa_dataa_scaleb_datab_scale output_dtype output_scalebiasuse_fast_accumreturnc Cs|}|} d} |j|jddfko|jd|jdfk} | r0|s0|| } |d}|d} |} | tjtjfvr?| r?tj}d} |tjkrJ|} d}tj|||| ||||d}| dur_|| 9}| durg|| 7}| tjtjfvrv| rv|| }|S)z This is the unwrapped version of addmm_float8, which does not take in Float8TrainingTensors as inputs. This is used to standardize the logic between subclassed and non subclassed versions of the linear module. Nr)scale_ascale_br scale_result out_dtyper) reciprocalshapenew_onestorchfloat16float32bfloat16 _scaled_mmto)r rrrrrrra_inverse_scaleb_inverse_scalepost_inverse_scaleis_rowwise_scaling orig_dtype post_biasoutputrrM/home/ubuntu/.local/lib/python3.10/site-packages/torchao/float8/float8_ops.pyaddmm_float8_unwrappedsF     r-cCs t|jdvsJ|ddS)N)rrz+ with axiswise scaling is not supported yet)lenr)aten_opscalerrr,_assert_tensorwise_scale_sr1csfdd}|S)z(Register aten ops to the float8 op tablecs8D]}|tvrtd|dt|j|t|<q|S)Nz Float8 op z is already registered to )r RuntimeError__name__)funcopaten_opsrr, decoratorjs zimplements..decoratorr)r7r8rr6r, implementsgs  r9cCs\t||dj||djg|ddRi|}t||dj|dj|dj|djSNrr)r1_scale_datar _orig_dtype_linear_mm_config_gemm_input_role)r/argskwargsnew_datarrr,float8_desugar_opvs $rCcCsj||djg|ddRi|}||djg|ddRi|}t|||dj|dj|djSr:)r<r;rr=r>r?)r/r@rArB new_scalerrr, float8_desugar_data_and_scale_ops$$rEcCs||djg|ddRi|}|djjdkr-||djg|ddRi|}n|dj}|tjjkr@t||dj|dj}|}|durX|dkrT|dkn|dkt|||dj |dj |dj |S)Nrr) r<r;ndimaten transposeintr1 _axiswise_dimrr=r>r?)r/r@rArBrDold_axiswise_dimnew_axiswise_dimrrr,float8_transposes($&    rNc Cs|d|d}}|t|jjkr;||djg|ddRi|}t||dj|dj|dj|dj|djSt |djjdkrKt |||S|j}t |dkr|dkr||j|fi|}d|dg}||j|fi|}t|||j|j|j|jS|dks|t |jdkr||j|fi|}|ddg}||j|fi|}d} t|||j|j|j| St |d|jd|jjd|jd|d ) NrrrFz# with axiswise scaling and t.shape z t._scale.shape z t._axiswise_dim z new_shape z is not supported yet.) listr<rrr;r=r>r?rKr.rCAssertionError) r/r@rAt new_shaperB axiswise_dimnew_scale_shaperDrMrrr, float8_viewsV$     (rVcsR|djgddRi|}t|djfdd}t||}t|S)Nrrcs(t|djdjdjdjS)Nr)rr;r=r>r?)datar@rr, make_float8sz!float8_split..make_float8)r<r1r;maprP)r/r@rAnew_data_tensorsrYoutrrXr, float8_splits $  r]c Cs|d}|dj}|dj}|dj}|djj}|dj}g} |D]I} t| ts-Jd| j|ks6Jd| j|us?Jd| j|usHJd| jj|ksRJd| j|us[Jdt|| j| | j t j q"|| g|ddRi|} | |} t| ||||S) Nrz7Expecting all chunks to be of type Float8TrainingTensorz,Expecting all chunks to be of the same dtypezCExpecting all chunks to have thee same scale as a result of a splitzGExpecting all chunks to have thee same mm config as a result of a splitzCExpecting all chunks to be of the same dtype as a result of a splitzLExpecting all chunks to have the same gemm_input_role as a result of a splitr) r=r;r>r<dtyper? isinstancerr1appendviewruint8) r/r@rAchunked_tensorsr)r0 mm_config fp8_dtypegemm_input_role chunk_datachunkrBrrr, float8_cat s>            ricCs:t||djdd}t||}t||}||i|S)a)Be careful with this function, this is a "fallback" op that casts the output of the op to the original precision. And performs the op. We currently need this to support the backward for admmm bias. "addmm" -> out "hp_gradBias" <-"sum" <- "identity" <- gradOut <- "hp_gradOut" rcSst|tr |S|SN)r_rto_original_precision)xrrr,unwrap9s z!float8_cast_up_op..unwrap)r1r;r)r/r@rArmnew_args new_kwargsrrr,float8_cast_up_op.s    rpabcCs|j}|j}|j}t|j|j|j|j}|jrA|jd|jdks5Jd|jdd|jdt|dd}t|dd}t| sK| }t| rY| }|j}|j durs|j durs| |jddd}n|j dur|j dur| |jddd}||||fS)Nrrz"Inner dims must match for mm, got z and )dimsrF)r<r;rr?r> pad_inner_dimsizer r stride contiguousrRrKrepeatrreshape)rqrrr rrscaled_mm_configrrrr,preprocess_addmmCs2      r{c Cs|d}|d}t|trt|tsJdt|t|t||\}}}}|j} t|j|j|j|j} | j rMt |j |j|j |j| St||||| dd| jd} | S)NrrzFExpecting both Float8TrainingTensor for mm inputs but found {} and {}rrr)r_rformattyper{r=rr?r>emulatermmr<floatr;r$r-r) r/r@rArqrrr rrrrrz tensor_outrrr, float8_mmls@  $ rc Cst|dtjrt|dtrt|dtsJ|d}|d}|d}t||\}}}} |j} |j| ks9Jdt|j|j |j|j } | j r`t |j |j|j |j| } | |St|||| | d|| jd} | S)NrrrOz"bias dtype must match output dtyper|)r_rTensorrr{r=r^rr?r>rrr<rr;r$r-r)r/r@rArrqrrr rrrrrzr\rrrr, float8_addmmsD  $ rcCs$t||dj|dj|djkSr:)r1r;rr/r@rArrr,float8_is_same_sizesrcCs~t|dts Jt|dkrd|vsJd|dtjtjhvs%Jdt|dj|dj|d|dj|dj |dj S)zThis gets called when running matmul under autocast when the input is a Float8TrainingTensor, presenting as a fp32 tensor. rrr^z%Only support dtype kwarg for autocastzKOnly support floating point conversion for autocast w/ Float8TrainingTensor) r_rr.rr r"r<r;r>r?rKrrrr,autocast_to_copys$rcCsxt||dj|d}t|tsJdt||j}|}||g|ddRi|}t||j|j|j|j S)z+ override funcol with FP8 handling rz9expecting a Float8TrainingTensor for allgather but found rN) r1r;r_rr~r<rwr=r>r?r/r@rA fp8_inputfp8_datafp8_outrrr, allgather_fp8s   rcCsbt||dj|d}t|tsJ|j}||g|ddRi|}t||j|j|j|jSr:r1r;r_rr<r=r>r?rrrr,wait_tensor_fp8srz 2.11.0.devcCsbt||dj|d}t|tsJ|j}||g|ddRi|}t||j|j|j|jS)z Handle _wrap_tensor_autograd for Float8TrainingTensor. This wraps the underlying fp8 data in AsyncCollectiveTensor while preserving the Float8TrainingTensor wrapper with its scale and metadata. rrNrrrrr,wrap_tensor_autograd_fp8srcCs|d}|d}t|tsJt|tsJt||dj|j|jks&J|j|jks.J|j|jks6J|j}|j}|||d|g|ddRi|}t||j|j|j|jS)NrrOr) r_rr1r;r^r=r<r>r?)r/r@rAfp8_self fp8_valuesrfp8_values_datarrrr, index_put_fp8s$&rcCs$|d}|d}t|ts,t|tr,|}t||j|||g|ddRi|St|trt|trt||j|j|jksFJd|j|jksPJd|j|jksZJd|jj|jjksfJd|j |j kspJd||j|jg|ddRi|}t||j|j|j|j St d ) NrrrOzr<r^r?r2)r/r@rAselfsrcsrc_hprrrr,copy_fp8/sH      $r)NNFrj)Htypingrrrrrtorch.utils._pytreer%torchao.float8.float8_training_tensorrrtorchao.float8.float8_utilsr r torchao.utilsr opsrHc10d_functional_c10d_functionalr __annotations__rr^boolr-r1r9 _unsafe_viewdefault as_stridedcloneslicefill_ScalarryrCdetachrErRrIrJrNrarVsplitr]catrisum dim_IntListrpr{rmatmulraddmmr is_same_sizer_to_copyrall_gather_into_tensorr wait_tensorr_wrap_tensor_autogradr index_put_rcopy_rrrrr,s   B        6    $ )  "  $