o i*@sVUdZddlZddlZddlZddlmZddlmZddlm Z ddl m Z ddl m Z ddl m Z dd lmZddlZdd lmZeejed krRed ddlZdd lmZddlmZddlmZddlmZddlm Z dddddZ!e"e#e fe$d<eeejj%edkre!d=de!d<dZ&dZ'ej(ddZ)Gdd d Z*Gd!d"d"Z+ d@d#ej,j-d$e"e#e fdBfd%d&Z. 'dAd(e+d)ej/j0d*e#fd+d,Z1d)ej/j0fd-d.Z2ej( 'dAd)ej/j0e d/e fBd*e#fd0d1Z3Gd2d3d3Z4Gd4d5d5Z5d)ej/j0d6ej/j0fd7d8Z6d@d)ej/j0d9e#d:e#dBfd;d<Z7d=e#d:e#dBfd>d?Z8dS)B N) ContextVar)BytesIO)Path)Any)Callable)cast)patch)versionz2.8z"ZeroGPU AoTI reuqires PyTorch 2.8+)!unwrap_tensor_subclass_parameters) package_aoti)AOTICompiledModel)Weights)register_cleanupFT)z$aot_inductor.package_constants_in_so&aot_inductor.package_constants_on_diskzaot_inductor.packagealways_keep_tensor_constantsINDUCTOR_CONFIGS_OVERRIDESz2.10rpickle_weightsz-aot_inductor.package_constants_on_disk_formatz2/tmp/*/archive/data/aotinductor/model/*.wrapper.soz package.pt2ccst}td|d}dd|D}dV|D] }|j|vr=|}tr=t|jdd}t ||SqdS)z PyTorch already cleans-up extracted archives in /tmp But the GPU worker never terminates gracefully in ZeroGPU so cleanup must be done manually z/proc/z /map_filescSsh|]}|jqS)name).0frrJ/home/ubuntu/.local/lib/python3.10/site-packages/spaces/zero/torch/aoti.py 4sz)_register_aoti_cleanup..N) osgetpidriterdirrreadlinkmatchARCHIVE_SO_PATTERNpartsr)pid map_files maps_beforemap_filemapped package_pathrrr_register_aoti_cleanup,s  r)c@s2eZdZd deeejfdefddZddZ dS) ZeroGPUWeightsF constants_mapto_cudacCs&|rdd|D|_dS||_dS)NcSsi|] \}}||dqS)cuda)torrtensorrrr @z+ZeroGPUWeights.__init__..)itemsr+)selfr+r,rrr__init__>s zZeroGPUWeights.__init__cCsLi}|jD]\}}tj|dd}||||<qt|dffS)NcpudeviceT) r+r3torch empty_like pin_memorycopy_detach share_memory_r*)r4r+rr0tensor_rrr __reduce__Cs  zZeroGPUWeights.__reduce__N)F) __name__ __module__ __qualname__dictstrr9Tensorboolr5r@rrrrr*=s r*c@s2eZdZdejjdefddZddZddZ d S) ZeroGPUCompiledModel archive_fileweightscCs||_||_tddd|_dS)Ncompiled_modeldefault)rIrJrrK)r4rIrJrrrr5LszZeroGPUCompiledModel.__init__csj}durAttjj}Wdn1swYtt|}fdd| D}|j |dddj |||i|S)Ncsi|] }|jj|qSr)rJr+rrr4rrr1Usz1ZeroGPUCompiledModel.__call__..Tcheck_full_update user_managed) rKgetr)r9 _inductoraoti_load_packagerIrr get_constant_fqnsload_constantsset)r4argskwargsrK constant_maprrOr__call__Ps  zZeroGPUCompiledModel.__call__cCst|j|jffSN)rHrIrJrOrrrr@YszZeroGPUCompiledModel.__reduce__N) rArBrCr9typesFileLiker*r5r\r@rrrrrHKs rHexported_programinductor_configsc si|pit}ttjj|}|jdusJ|j\}}tjj||||d}tt t t B|}t }dd|D}t |t |dd|D\tt tfddD}t||S)N)optionscs|] }t|tr|VqdSr]) isinstancerE)rfilerrr hzaoti_compile..csrcr])rdr)rartifactrrrrfjrgcsi|] }||dqS)r) get_weightrNrJrrr1lr2z aoti_compile..)rrr9fx GraphModulemoduleexample_inputsrT aot_compilelistrErrr r*rH) r`ragmrYrZ artifactsrIfileszerogpu_weightsrrjr aoti_compile]s   ruforwardcompiledrm call_methodcCst|||t|dSr])setattrdrain_module_parameters)rwrmrxrrr aoti_applyps  r{cCspdd|D}dd|D}|j|dd|D]\}}||}tgjdi||_q!dS)NcSs i|] \}}||j|jdqS)r8dtyper|r/rrrr1zs z+drain_module_parameters..c Ss(i|]\}}|tjtj|ddqS)r6r7)r9nn Parameterr:r/rrrr1{s(T)assignr) state_dictr3load_state_dictr9rFr.data)rmstate_dict_metarrparammetarrrrzysrz.c #sGdddtGddd}|}fdd}tj|||d1z|VWnyA}z|j|_|j|_WYd}~n d}~wwWddSWddS1sUwYdS)NcseZdZfddZZS)z+aoti_capture..CapturedCallExceptioncst||_||_dSr])superr5rYrZ)r4rYrZ __class__rrr5s  z4aoti_capture..CapturedCallException.__init__)rArBrCr5 __classcell__rrrrCapturedCallExceptionsrc@seZdZddZdS)z"aoti_capture..CapturedCallcSsd|_i|_dS)NrrYrZrOrrrr5s z+aoti_capture..CapturedCall.__init__N)rArBrCr5rrrr CapturedCalls rcs|i|r]rrrrr capture_callsz"aoti_capture..capture_call)new) Exceptionr objectrYrZ)rmrxr captured_callrerrr aoti_captures"  "rc@sReZdZdejjfddZdeeej fde fddZ deeej ffdd Z d S) LazyAOTIModelrIcCs&||_tddd|_tddd|_dS)NrKrLloaded_weights)rIrrKr)r4rIrrrr5szLazyAOTIModel.__init__rJrQcOs|j}dur.ttj|j}Wdn1swYtt|}|j ||j }dus;||urI|j ||dd|j |||i|S)NTrP) rKrSr)r9rTrUrIrr rXrrW)r4rJrQrYrZrKrrrrr\s   zLazyAOTIModel.__call__cCs t||Sr])LazyAOTIModelWithWeights)r4rJrrr with_weightss zLazyAOTIModel.with_weightsN) rArBrCr9r^r_r5rDrErFrGr\rrrrrrs rc@s0eZdZdedeeejffddZddZ dS)rmodelrJcCs||_||_d|_dS)NT)rrJ first_call)r4rrJrrrr5s z!LazyAOTIModelWithWeights.__init__cOs(|j}d|_|j|j|g|Ri|S)NF)rrrJ)r4rYrZrQrrrr\sz!LazyAOTIModelWithWeights.__call__N) rArBrCrrDrEr9rFr5r\rrrrrs rreturncCsJt|j}|j|_|j|_|j|_dd|jD|_|S)NcSs"i|] \}}|dur|t|qSr])_shallow_clone_module)rkvrrrr1s"z)_shallow_clone_module..) r__new__r__dict__copy _parameters_buffers_modulesr3)rmclonerrrrs    rrepo_idvariantcs0ddlmddlm}t|dd}durtdz tdd}Wn |y2d}Ynwt|dur?t | nd }| d d}dur^|D]}dd l m } | di|qNttt|}fd d |D} | D]!\} } |D]} | jj| krt| }t|| || _q|qtdS)aq Loads AOTI-compiled blocks for a given module from the Hugging Face Hub. This function expects the module to expose a `_repeated_blocks` attribute. This attribute is present on most models from the diffusers library. Args: module (torch.nn.Module): The module containing repeated blocks to be replaced. repo_id (str): The Hugging Face Hub repository ID where the compiled blocks are stored. variant (str | None, optional): Variant suffix to append to block names. Defaults to None. Returns: None: The function mutates the given module in place. Example: >>> import spaces >>> import torch >>> from diffusers import FluxPipeline >>> pipeline = FluxPipeline.from_pretrained('black-forest-labs/FLUX.1-dev') >>> spaces.aoti_blocks_load(pipeline.transformer, 'zerogpu-aoti/FLUX.1') r)hf_hub_download)EntryNotFoundError_repeated_blocksNzEaoti_blocks_load only works with modules that expose _repeated_blocksconfigz.jsonz{}kernels) get_kernelc s&i|]}|ttt|dqS))rfilename subfolder)rPACKAGE_FILENAME_variantrNrrrrrr1s z$aoti_blocks_load..r)huggingface_hubrhuggingface_hub.errorsrgetattr RuntimeErrorrjsonloadsr read_textrSrrrrprEr3modulesrrArr rrrv)rmrrrrepeated_blocks config_pathrkernels_configkernels_kwargsr aoti_models block_name aoti_modelblockblock_rrraoti_blocks_loads6      rrcCs|r |d|S|S)N.r)rrrrrrsrr])rv)9__doc__ contextlibrr contextvarsriorpathlibrtypingrrr unittest.mockr r9 packagingr parse __version__rtorch._inductor.codecache7torch._functorch._aot_autograd.subclass_parametrizationr torch._inductor.package.packager !torch.export.pt2_archive._packager )torch.export.pt2_archive._package_weightsrutilsrrrDrE__annotations__ base_versionr!rcontextmanagerr)r*rHexportExportedProgramrur~Moduler{rzrrrrrrrrrrs|                    7