o Ii_o@sXddlZddlZddlZddlZddlZddlZddlZddlmZddl m Z m Z ddl Z ddl Z ddlZddlZddlmZmZddlZddlZddlZddlmZddlZddlmZmZmZmZe ddd d  Z!e!"Z#Wdn1szwYej$%ej$&e'Z(d Z)d Z*e+d ddkZ,e+dddkZ-e+dddkZ.e+dddkZ/e+dddkZ0e+dddkZ1e+dddkZ2e+dddkZ3e+dddkZ4e+dddkZ5e+dddkZ6e+dddkZ7e+dddkZ8e+dddkZ9e+dddkZ:e+dddkZ;e+dddkZe+d"ddkZ?e+d#ddkZ@dd$lmAZAmBZBmCZCmDZDmEZEmFZFmGZGmHZHmIZI % dd&d'ZJeJejKjL_Jd(d)ZMd*d+ZNd,eOd%dfd-d.ZPdd0eOd1eOd%eQfd2d3ZRd%eQfd4d5ZSd6d7ZTd8d9ZUd:d;ZVdd?iZXe Yd@ZZiZ[gZ\e]gdAe-se^dB_ej`eaej`bdCdZceaej`bdCdDZdePd eNe\ZeZfefe dEkregdFefe dEkr/eVd>dGdGeXd>dHdIdJeVd>dKdGeXd>dLdIdJej$%e'Zhej$iehejjdMdNdOdGZkej$iekd>eZZlekejmejndPejndP<elejndQ<eoeleeljpejqBgZrersdRersdSe.rBdTejt_uee(jvZwewdUdVZxge/rTdWgnge1r\dXgnge0rddYgnge2rldZgnge3rtd[gnge4r|d\gnge5rd]gnge6rd^gnge7rd_gnge8rd`gnge9rdagnge:rdbgnge;rdcgnge<rddgnge=rdegnge>rdfgnge?rdggnge@rdhgngZydige6sdjgngZzdige6sdjgnge7sdkgngZ{dige6sdjgngZ|ge:sdlgnge;sdmgnge<sdngnge=s'dognge>s/dpgngZ}dqgZ~e}Zd/ge0s?drgngZd/ge1sJdsgngZd/ge4sUdtgngZe4r^d/gndugZd/ge5sidvgngZdwdxeeezeeeDZdydxee~e{eeeeDZdzdxee}e|eDZd{dxee}e|eDZe/rgZgZd|ge?sengee?sengeZe0sed}g7Zgd~ZeMdkreddgee(exdgZe\sededdgeyeWeereydedddZddZGdddeZee)eedddgde#dgde\e\r eedndeidgdd dS)N)Path)parseVersion)setup find_packages) bdist_wheel)BuildExtension CppExtension CUDAExtension CUDA_HOMEz ../README.mdrzutf-8)encoding flash_attnzVhttps://github.com/Dao-AILab/flash-attention/releases/download/{tag_name}/{wheel_name}FLASH_ATTENTION_FORCE_BUILDFALSETRUEFLASH_ATTENTION_SKIP_CUDA_BUILDFLASH_ATTENTION_FORCE_CXX11_ABI FLASH_ATTENTION_DISABLE_BACKWARDFLASH_ATTENTION_DISABLE_SPLITFLASH_ATTENTION_DISABLE_PAGEDKV FLASH_ATTENTION_DISABLE_APPENDKVFLASH_ATTENTION_DISABLE_LOCALFLASH_ATTENTION_DISABLE_SOFTCAPFLASH_ATTENTION_DISABLE_PACKGQAFLASH_ATTENTION_DISABLE_FP16FLASH_ATTENTION_DISABLE_FP8FLASH_ATTENTION_DISABLE_VARLENFLASH_ATTENTION_DISABLE_CLUSTERFLASH_ATTENTION_DISABLE_HDIM64FLASH_ATTENTION_DISABLE_HDIM96FLASH_ATTENTION_DISABLE_HDIM128FLASH_ATTENTION_DISABLE_HDIM192FLASH_ATTENTION_DISABLE_HDIM256FLASH_ATTENTION_DISABLE_SM80 FLASH_ATTENTION_ENABLE_VCOLMAJOR) IS_HIP_EXTENSIONCOMMON_HIP_FLAGSSUBPROCESS_DECODE_ARGS IS_WINDOWSget_cxx_compiler_join_rocm_home_join_cuda_home _is_cuda_file _maybe_writereturnc ' Csdd} | |}| |}| |}| |}| |}| |}t|t|ks&Jt|dks.Jt} dg} | d| | s@|rjtrHtdd}ntdd}d tjvrXtd }n|}| d || d |trpt |}d d |g}|dd || r|dd ||dd |dd|D}|dd ||ddg}|dd ||dd ||dd |dd|D}dg}t r|d|dn|d|d|d| rHd g}d!}t j jd"urtd#d$d%kr|d|dd&}d'g|d(d"d)|d*g}d+g|d(d"d)|d,g}|d-|d.g}t||D]O\}}t|oY| }|rr|d/rfd0}n|d1rod2}nd3}nd4}t r|d5d6}|d5d6}|d d7}|d d7}|d8|d9|d |qO|rtj tj|dd:}d;g}|d<d8|d=d |g}||g7}ngg}}| d"ur d>g} t rtd?d@gjtdA}!t|!d(krtj|!dd5d6}"ntdB| dC|"dDn| dEd8| dFd |g}#dG| g}$nggg} }#}$| ||g}%| r?|%||%||%||%|| |||#|$g7}%dH dIdJ|%D}&|&dK7}&t||&d"S)LaWrite a ninja file that does the desired compiling and linking. `path`: Where to write this file `cflags`: list of flags to pass to $cxx. Can be None. `post_cflags`: list of flags to append to the $cxx invocation. Can be None. `cuda_cflags`: list of flags to pass to $nvcc. Can be None. `cuda_postflags`: list of flags to append to the $nvcc invocation. Can be None. `sources`: list of paths to source files `objects`: list of desired paths to objects, one per source. `ldflags`: list of flags to pass to linker. Can be None. `library_target`: Name of the output library. Can be None; in that case, we do no linking. `with_cuda`: If we should be compiling with CUDA. cSs|durgSdd|DS)NcSsg|]}|qS)strip).0flagr0r0@/home/ubuntu/.local/lib/python3.10/site-packages/hopper/setup.py psz=_write_ninja_file..sanitize_flags..r0)flagsr0r0r4sanitize_flagslsz)_write_ninja_file..sanitize_flagsrzninja_required_version = 1.3zcxx = binhipccnvcc PYTORCH_NVCCznvcc_from_env = znvcc = z cflags =  zpost_cflags = zcuda_cflags = zcuda_post_cflags = cSsg|] }|dkr |ndqS)arch=compute_90a,code=sm_90aarch=compute_80,code=sm_80r0)r2sr0r0r4r5sz%_write_ninja_file..zcuda_post_cflags_sm80 = -gencoder>zcuda_post_cflags_sm80_sm90 = zcuda_dlink_post_cflags = z ldflags = cSsg|]}tj|qSr0)ospathabspath)r2filer0r0r4r5sz rule compilez@ command = cl /showIncludes $cflags -c $in /Fo$out $post_cflagsz deps = msvczD command = $cxx -MMD -MF $out.d $cflags -c $in -o $out $post_cflagsz depfile = $out.dz deps = gcczrule cuda_compileN*TORCH_EXTENSION_SKIP_NVCC_GEN_DEPENDENCIES01z?--generate-dependencies-with-compile --dependency-output $out.dzrule cuda_compile_sm80z command = $nvcc z3 $cuda_cflags -c $in -o $out $cuda_post_cflags_sm80zrule cuda_compile_sm80_sm90z8 $cuda_cflags -c $in -o $out $cuda_post_cflags_sm80_sm90z command = $nvcc_from_env z. $cuda_cflags -c $in -o $out $cuda_post_cflags_sm90.cu cuda_compile_sm80.cucuda_compile_sm80cuda_compile_sm80_sm90compile:z$:z$ zbuild z: zdlink.ozrule cuda_devlinkz5 command = $nvcc $in -o $out $cuda_dlink_post_cflagsz: cuda_devlink z rule linkwhereclz z'MSVC is required to load C++ extensionsz command = "z)/link.exe" $in /nologo $ldflags /out:$outz% command = $cxx $in $ldflags -o $outz: link zdefault z css|]}d|VqdS) N)join)r2br0r0r4 sz$_write_ninja_file..rS)lenr*appendr&r+r,rAenvirongetenvr'rTr)torchversioncudazipr-endswithreplacerBdirname subprocess check_outputdecoder(split RuntimeErrorr.)'rBcflags post_cflags cuda_cflagscuda_post_cflagscuda_dlink_post_cflagssourcesobjectsldflagslibrary_target with_cudar7compilerconfigr: nvcc_from_envr6cuda_post_cflags_sm80cuda_post_cflags_sm80_sm90 compile_rulecuda_compile_rule nvcc_gendepscuda_compile_rule_sm80cuda_compile_rule_sm80_sm90build source_file object_fileis_cuda_sourcerule devlink_out devlink_ruledevlink link_rulecl_pathscl_pathlinkdefaultblockscontentr0r0r4_write_ninja_fileSs                              rcCsdtjdrdStjdkr#dtdddd}d|d Stjd kr*d Std tj) z? Returns the platform name as used in wheel filenames. linux linux_x86_64darwin.rNmacosx__x86_64win32 win_amd64zUnsupported platform: {})sysplatform startswithrTmac_verre ValueErrorformat) mac_versionr0r0r4 get_platforms     rcCsJtj|ddgdd}|}|dd}t||dd}||fS) Nz /bin/nvccz-VT)universal_newlinesreleaserI,r)rbrcreindexr)cuda_dir raw_outputoutput release_idxbare_metal_versionr0r0r4get_cuda_bare_metal_versions r global_optioncCs tdurdSt|ddS)Nz was requested, but nvcc was not found. Are you sure your environment has nvcc available? If you're installing within a container from https://hub.docker.com/r/pytorch/pytorch, only images whose names contain 'devel' will provide nvcc.)r warningswarn)rr0r0r4check_if_cuda_home_nones rrEnamercCst||dvS)N)ONrHYESrY)rArZupper)rrr0r0r4check_env_flag'srcCs tddS)am Downstream projects and distributions which bootstrap their own dependencies from scratch and run builds in offline sandboxes may set `FLASH_ATTENTION_OFFLINE_BUILD` in the build environment to prevent any attempts at downloading pinned dependencies from the internet or at using dependencies vendored in-tree. Dependencies must be defined using respective search paths (cf. `syspath_var_name` in `Package`). Missing dependencies lead to an early abortion. Dependencies' compatibility is not verified. Note that this flag isn't tested by the CI and does not provide any guarantees. FLASH_ATTENTION_OFFLINE_BUILDrE)rr0r0r0r4is_offline_build,s rcCsJtd}|stdptdptdpd}|stdtj|dS)NFLASH_ATTENTION_HOMEHOME USERPROFILEHOMEPATHz"Could not find user home directoryz .flashattn)rArZrfrBrT) user_homer0r0r4get_flashattn_cache_path=s "rcCs,d}d|i}tj|d|}tjj|ddS)NzFMozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/119.0z User-Agenti,)timeout)urllibrequestRequesturlopen)url user_agentheadersrr0r0r4open_urlFs rcCsntrdSt}tjt}t}z ddddt}Wn t y+t}Ynwddd} || |||} tj |d|} tj |tj ddd|}|dkrSd nd } t |r^|| |n|}tj | |}tj | } | rtd | d tjt| d d}|j| dtjtj|dddtd|d|d tj|rtj||dddSt||dS)N64aarch64)x86_64arm64rr)LinuxDarwinnvidia third_partybackendz sbsa-linuxz x86_64-linuxzdownloading and extracting z ...zr|*)fileobjmode)rBrT)exist_okzcopy z to ) dirs_exist_ok)rrrArBra__file__rsystemmachineKeyErrorrTpardircallableexistsprinttarfileopenr extractallmakedirsreisdirshutilcopytreecopy)rsrc_pathdst_pathr\url_funcflashattn_cache_pathbase_dirrarch supportedrtmp_path platform_namedownloadrDr0r0r4download_and_copyPs6      rcCstdpd}d|gS)N NVCC_THREADS4z --threads)rArZ) nvcc_threadsr0r0r4nvcc_threads_argsnsrr:z12.3.107EXE)git submoduleupdatez--initz../csrc/cutlassz torch.__version__ = {} rrIz12.3z9FlashAttention-3 is only supported on CUDA 12.3 and abover8cfdddS)Nc ddddd SNz&https://anaconda.org/nvidia/cuda-nvcc/z /download/-z /cuda-nvcc-z -0.tar.bz2r0 version_majorversion_minor1version_minor2rrr\r0r4..rrerrr\r0rr4rr)rrrr\rznvvm/bincr)Nc rrr0rrr0r4rrrrrrr0rr4rrrrrPATHr;r@r=Tcsrccutlassz!-DFLASHATTENTION_DISABLE_BACKWARDz -DFLASHATTENTION_DISABLE_PAGEDKVz-DFLASHATTENTION_DISABLE_SPLITz!-DFLASHATTENTION_DISABLE_APPENDKVz-DFLASHATTENTION_DISABLE_LOCALz -DFLASHATTENTION_DISABLE_SOFTCAPz -DFLASHATTENTION_DISABLE_PACKGQAz-DFLASHATTENTION_DISABLE_FP16z-DFLASHATTENTION_DISABLE_FP8z-DFLASHATTENTION_DISABLE_VARLENz -DFLASHATTENTION_DISABLE_CLUSTERz-DFLASHATTENTION_DISABLE_HDIM64z-DFLASHATTENTION_DISABLE_HDIM96z -DFLASHATTENTION_DISABLE_HDIM128z -DFLASHATTENTION_DISABLE_HDIM192z -DFLASHATTENTION_DISABLE_HDIM256z-DFLASHATTENTION_DISABLE_SM8xz!-DFLASHATTENTION_ENABLE_VCOLMAJORbf16fp16e4m3@`all_split_paged_softcap _softcapall_packgqac Cs4g|]\}}}}}d|d||||dqS)instantiations/flash_fwd_hdim_rLr0)r2hdimdtyperepagedsoftcapr0r0r4r5s "r5c CsFg|]\}}}}}}|r|s|sd|d|||||d qS)rrrJr0)r2rrrerrpackgqar0r0r4r5s(cC(g|]\}}}d|d||dqS)instantiations/flash_bwd_hdimrrLr0r2rrrr0r0r4r5cCr)rrrJr0rr0r0r4r5rz flash_api.cppzflash_fwd_combine.cu) -O3 -std=c++17z--ftemplate-backtrace-limit=0z--use_fast_mathz--resource-usagez -lineinfoz'-DCUTE_SM90_EXTENDED_MMA_SHAPES_ENABLEDz-DCUTLASS_DEBUG_TRACE_LEVEL=0z-DNDEBUGrz-D_USE_MATH_DEFINESz-Xcompiler=/Zc:__cplusplusincludeflash_attn_3_cudarr)cxxr:)rrlextra_compile_args include_dirscCs|tttdd}td|tj}Wdn1swYt| d}t j d}|r:|d|St |S)Nz __init__.pyr z^__version__\s*=\s*(.*)$rIFLASH_ATTN_LOCAL_VERSION+)rrthis_dirresearchread MULTILINEast literal_evalgrouprArYgetstr)f version_matchpublic_version local_versionr0r0r4get_package_versions r3c Csttjj}ttj}|jdkrtdntd}dtjjtjj}t }t }|j|j}|jd|j}t tj j }td|d|d|d |d|d|d|d }tjd ||d } | |fS) N z11.8z12.2cprrz+cur[cxx11abi.whlv)tag_name wheel_name)rr[r\r] __version__majorr version_infominorrr3r._C_GLIBCXX_USE_CXX11_ABIr PACKAGE_NAMEBASE_WHEEL_URLr) torch_cuda_versiontorch_version_rawpython_versionrpackage_version cuda_version torch_version cxx11_abiwheel_filename wheel_urlr0r0r4 get_wheel_urls  4rLcs eZdZdZfddZZS)CachedWheelsCommandai The CachedWheelsCommand plugs into the default bdist wheel, which is ran by pip when it cannot find an existing wheel (which is currently the case for all installs). We use the environment parameters to detect whether there is already a pre-built version of a compatible wheel available and short-circuits the standard full build pipeline. cstrtSt\}}td|zAtj||tj |j s&t |j | \}}}|jd|d|d|}tj |j |d}td|t||WdStjjygtdtYdSw)NzGuessing wheel URL: rr7zRaw wheel pathz4Precompiled wheel not found. Building from source...) FORCE_BUILDsuperrunrLrrr urlretrieverArBrdist_dirrget_tagwheel_dist_namerTrmoveerror HTTPError)selfrKrJimpl_tagabi_tagplat_tagarchive_basename wheel_path __class__r0r4rP9s"     zCachedWheelsCommand.run)__name__ __module__ __qualname____doc__rP __classcell__r0r0r^r4rM1srM)r{rrtestsdistdocs benchmarks)excludeflash_attn_interfacezFlashAttention-3z text/markdown)z#Programming Language :: Python :: 3z2License :: OSI Approved :: Apache Software LicensezOperating System :: Unix)r build_extrz>=3.8)r[einops packagingninja) rr\packages py_modules descriptionlong_descriptionlong_description_content_type classifiers ext_modulescmdclasspython_requiresinstall_requires)r/N)rE)rrrAstatr&rr*pathlibrpackaging.versionrrr sysconfigr itertools setuptoolsrrrburllib.requestr urllib.errorwheel.bdist_wheelr _bdist_wheelr[torch.utils.cpp_extensionrr r r rfhr(rrrBrarCrr%rArBrZrNSKIP_CUDA_BUILDFORCE_CXX11_ABIDISABLE_BACKWARD DISABLE_SPLITDISABLE_PAGEDKVDISABLE_APPENDKV DISABLE_LOCALDISABLE_SOFTCAPDISABLE_PACKGQA DISABLE_FP16 DISABLE_FP8DISABLE_VARLENDISABLE_CLUSTERDISABLE_HDIM64DISABLE_HDIM96DISABLE_HDIM128DISABLE_HDIM192DISABLE_HDIM256 DISABLE_SM8xENABLE_VCOLMAJORr&r'r(r)r*r+r,r-r.rutils cpp_extensionrrr.rboolrrrrrrNVIDIA_TOOLCHAIN_VERSIONget_config_var exe_extensionrvrurPrrr;intre TORCH_MAJOR TORCH_MINORrrrfrrTr ctk_path_new nvcc_path_newpathseprYchmodst_modeS_IEXECcc_flagrXr?r@parentrepo_dir cutlass_dir feature_argsDTYPE_FWD_SM80DTYPE_FWD_SM90 DTYPE_BWDHEAD_DIMENSIONS_BWDHEAD_DIMENSIONS_FWDHEAD_DIMENSIONS_FWD_SM80SPLITPAGEDKVSOFTCAP SOFTCAP_ALLPACKGQAproductsources_fwd_sm80sources_fwd_sm90sources_bwd_sm80sources_bwd_sm90rl nvcc_flagsextendr"r3rLrMr0r0r0r4s   ,   -                   &         "