o ۷i@sddlZddlZddlZddlZddlZddlZddlmZddlm Z ddl m Z m Z ddl mZddlZddlmZddlmZddlmZdd lmZd d lmZmZd d lmZmZmZmZd d l m!Z!m"Z"m#Z#d dl$m%Z%m&Z&d dl'm(Z(m)Z)d dl*m+Z+ddl,m-Z-ddl.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;me?Z@deAfddZBdBddZCdBddZDdBddZEedeBdfdeDfd eEfd!eBd"fd#eBd$fd%eBd&fd'eCfd(eBd)fd*eBd+fd,eBd-fd.eBd/fd0eBd1fg ZFe Gd2d3d3ZGe Gd4d5d5ZHGd6d7d7ee!ZIGd8d9d9eIZJGd:d;d;eJZKGdd?d?eIZMGd@dAdAee!ZNdS)CN) OrderedDict)deepcopy) dataclassfield)Any) create_repo)validate_hf_hub_args)tqdm)Self) ConfigMixin FrozenDict)LOADABLE_CLASSES_fetch_class_library_tuple _unwrap_modelsimple_get_class_obj)PushToHubMixinis_accelerate_availablelogging)get_class_from_dynamic_moduleresolve_trust_remote_code)load_or_create_model_cardpopulate_model_card)is_compiled_module)ComponentsManager)MODULAR_MODEL_CARD_TEMPLATE ComponentSpec ConfigSpec InputParamInsertableDict OutputParam_validate_requirementscombine_inputscombine_outputsformat_componentsformat_configsformat_workflow#generate_modular_model_card_contentmake_doc_stringpipeline_class_namecsdfdd }|S)zFCreate a mapping function that always returns the same pipeline class.NcsSN config_dictr*r,b/home/ubuntu/vllm_env/lib/python3.10/site-packages/diffusers/modular_pipelines/modular_pipeline.py_map_fnIsz'_create_default_map_fn.._map_fnr+r,)r*r1r,r/r0_create_default_map_fnFsr2cCs$|durdSd|vr|drdSdS)NFlux2KleinModularPipeline is_distilledFlux2KleinBaseModularPipeliner,r-r,r,r0_flux2_klein_map_fnOs r6cC(|durdSd|vr|ddurdSdS)NWanModularPipelineboundary_ratioWan22ModularPipeliner,r-r,r,r0 _wan_map_fnY r;cCr7)NWanImage2VideoModularPipeliner9Wan22Image2VideoModularPipeliner,r-r,r,r0_wan_i2v_map_fncr<r?zstable-diffusion-xl StableDiffusionXLModularPipelinewanzwan-i2vfluxFluxModularPipelinez flux-kontextFluxKontextModularPipelineflux2Flux2ModularPipelinez flux2-klein qwenimageQwenImageModularPipelinezqwenimage-editQwenImageEditModularPipelinezqwenimage-edit-plus QwenImageEditPlusModularPipelinezqwenimage-layeredQwenImageLayeredModularPipelinezz-imageZImageModularPipelinec @seZdZUdZeedZeeefe d<eedZ eee efe d<ddededefd d Z dd ee eBd ed eeeefBfddZ ded eeeffddZd eeeffddZddZddZdS) PipelineStatezl [`PipelineState`] stores the state of a pipeline. It is used to pass data between pipeline blocks. )default_factoryvalueskwargs_mappingNkeyvalue kwargs_typecCsD||j|<|dur ||jvr|g|j|<dS|j||dSdS)z Add a value to the pipeline state. Args: key (str): The key for the value value (Any): The value to store kwargs_type (str): The kwargs_type with which the value is associated N)rOrPappend)selfrQrRrSr,r,r0sets  zPipelineState.setkeysdefaultreturncs,t|tr j|Sfdd|DS)aU Get one or multiple values from the pipeline state. Args: keys (str | list[str]): Key or list of keys for the values default (Any): The default value to return if not found Returns: Any | dict[str, Any]: Single value if keys is str, dictionary of values if keys is list csi|] }|j|qSr,)rOget).0rQrXrUr,r0 z%PipelineState.get..) isinstancestrrOrZ)rUrWrXr,r\r0rZs zPipelineState.getcCs|j|g}||S)z Get all values with matching kwargs_type. Args: kwargs_type (str): The kwargs_type to filter by Returns: dict[str, Any]: Dictionary of values with matching kwargs_type )rPrZ)rUrS value_namesr,r,r0 get_by_kwargss zPipelineState.get_by_kwargscCs i|jS)z8 Convert PipelineState to a dictionary. )__dict__rUr,r,r0to_dicts zPipelineState.to_dictc Csdzt|d}Wntytd|jjd|dw||vr%||Std|jjd|d)z Allow attribute access to intermediate values. If an attribute is not found in the object, look for it in the intermediates dict. rO'z' object has no attribute ')object__getattribute__AttributeError __class____name__)rUnamerOr,r,r0 __getattr__s zPipelineState.__getattr__csRdddfdd|jD}ddd|jD}d|d|d S) NcSst|drt|drd|jd|jdSt|tr=t|dkr=t|ddr=t|ddr=d|djd|djdSt|S) Nshapedtype Tensor(dtype=, shape=)rz[Tensor(dtype=z), ...])hasattrrornr_listlenrepr)vr,r,r0 format_values 2z,PipelineState.__repr__..format_value c3(|]\}}d|d|VqdSz z: Nr,r[krwrxr,r0 &z)PipelineState.__repr__..css$|] \}}d|d|VqdSr{r,r|r,r,r0rs"zPipelineState( values={ z }, kwargs_mapping={ z } ))joinrOitemsrP)rU values_strkwargs_mapping_strr,r~r0__repr__szPipelineState.__repr__r+)rk __module__ __qualname____doc__rdictrOr`r__annotations__rPrtrVrZrbrermrr,r,r,r0rMs ,  rMc@sHeZdZdZddZdefddZdedefdd Zd d Z d d Z dS) BlockStatez\ Container for block state data with attribute access and formatted representation. cKs"|D] \}}t|||qdSr+)rsetattr)rUkwargsrQrRr,r,r0__init__szBlockState.__init__rQcCs t||dSr+getattr)rUrQr,r,r0 __getitem__s zBlockState.__getitem__rRcCst|||dSr+)r)rUrQrRr,r,r0 __setitem__zBlockState.__setitem__cCt|jS)z Convert BlockState to a dictionary. Returns: dict[str, Any]: Dictionary containing all attributes of the BlockState )rrcrrdr,r,r0as_dictszBlockState.as_dictcs2dddfdd|jD}d|dS)NcSst|drt|drd|jd|jdSt|trCt|dkr?t|ddr?t|ddr?dd|D}d t|d |St|St|trqt|dkrmt|ddrmt|ddrmd d|D}d t|d |St|St|tri}| D]O\}}t|drt|drd |jd|jd||<q|t|trt|dkrt|ddrt|ddrdd|D}d t|d |||<q|t|||<q||St|S)NrnrorprqrrrcSg|]}|jqSr,rnr[tr,r,r0 z=BlockState.__repr__..format_value..zlist[z] of Tensors with shapes cSrr,rrr,r,r0rrztuple[z Tensor(shape=z, dtype=cSrr,rrr,r,r0rr) rsrornr_rtrurvtuplerr)rwshapesformatted_dictr}valr,r,r0rxs: ( (    z)BlockState.__repr__..format_valueryc3rzr{r,r|r~r,r0rrz&BlockState.__repr__..z BlockState( z ))rrcr)rU attributesr,r~r0rs( zBlockState.__repr__N) rkrrrrr`rrrrrr,r,r,r0rs rc @seZdZUdZdZdZdZeeefdBe d<dZ e ddZ ddZ ed efd d Zed eefd d Zed eefddZed eefddZddZed eefddZed eefddZddZed eefddZddZeddZd efd!d"Ze #d?d$ed%e fd&d'Z!d?d(d)Z"   d@d$ee#j$BdBd*e%dBd+edBd d,fd-d.Z&d/e'd efd0d1Z(d/e'd2e)fd3d4Z*ed eefd5d6Z+ed eefd7d8Z,ed eefd9d:Z-ed eefd;d<Z.ed=d>Z/dS)AModularPipelineBlocksaU Base class for all Pipeline Blocks: ConditionalPipelineBlocks, AutoPipelineBlocks, SequentialPipelineBlocks, LoopSequentialPipelineBlocks [`ModularPipelineBlocks`] provides method to load and save the definition of pipeline blocks. > [!WARNING] > This is an experimental feature and is likely to change in the future. zmodular_config.jsonN _requirementscCsPt|jj}dd|D}tdd|D}t|dh}||fS)NcSs"i|] \}}|jtjkr||qSr,rXinspect_emptyr|r,r,r0r]3"z=ModularPipelineBlocks._get_signature_keys..cSs h|] \}}|jtjkr|qSr,rr|r,r,r0 4 z.rU)r signaturer parametersrrVrW)clsobjrrequired_parametersoptional_parametersexpected_modulesr,r,r0_get_signature_keys0s z)ModularPipelineBlocks._get_signature_keyscCs t|_dSr+)r sub_blocksrdr,r,r0r9s zModularPipelineBlocks.__init__rYcCdS).auto_mapzoSelected model repository does not happear to have any custom code or does not have a valid `config.json` file. requirements.z.py) module_file class_namecs(i|]}|vs |vr||qSr,rZrexpected_kwargsroptional_kwargsr,r0r]s"r,) load_configrkr ValueErrorr"splitrr)rrrrhub_kwargs_names hub_kwargsconfighas_remote_code_ class_refrr block_cls block_kwargsr,rr0from_pretraineds:  z%ModularPipelineBlocks.from_pretrainedc Ks|jj}t|j}|ddddd}|jjj ddd}||d|i}|j |dt t |dd} | r@|j | d |j d ||d |t|j} t| |_dS) Nrr __dynamic__rr)rr)r)save_directory push_to_hubr,)rjrktyperrsplitreplacesave_pretrained__func__rrregister_to_configr"r save_configrrr _internal_dict) rUrrrcls_namefull_modmodule parent_modulerrrr,r,r0rs    z%ModularPipelineBlocks.save_pretrainedcomponents_manager collectionModularPipelinec CsDt|jtd}|}td}t||}|t||||d}|S)zm create a ModularPipeline, optionally accept pretrained_model_name_or_path to load from hub. r  diffusers)blocksrr r )MODULAR_PIPELINE_MAPPINGrZ model_namer2 importlib import_modulerr) rUrr r map_fnr*diffusers_modulepipeline_classmodular_pipeliner,r,r0 init_pipelines   z#ModularPipelineBlocks.init_pipelinestatec Csi}|j}|D][}|jr5||j}|jr"|dur"td|jd|dus/|dur4|j|vr4|||j<q|jrb|j|vrBi||j<||j}|rb|D]\}}|dura|||<|||j|<qNqtdi|S)z2Get all inputs and intermediates in one dictionaryNzRequired input 'z ' is missingr,) rrlrZrrrSrbrr) rUrdata state_inputsrrR inputs_kwargsr}rwr,r,r0get_block_states*     z%ModularPipelineBlocks.get_block_state block_statec Cs|jD] }t||jstd|jdt||j}||j||jq|jD]S}|jrLt||jrLt||j}||j}||urK||j||jq'|jrz| |j}| D] \}}|durbqYt||shqYt||}||ury||||jqYq'dS)NzIntermediate output 'z' is missing in block state) rrsrlrrrVrSrrZrbr) rUrr output_paramparamr current_valueintermediate_kwargs param_namer,r,r0set_block_states0         z%ModularPipelineBlocks.set_block_statecCdd|jDS)NcSg|] }|jdur|jqSr+rl)r[rr,r,r0rr^z5ModularPipelineBlocks.input_names..rrdr,r,r0rrz!ModularPipelineBlocks.input_namescCr#)NcSr$r+r%r[rr,r,r0rr^zCModularPipelineBlocks.intermediate_output_names..rrdr,r,r0intermediate_output_namesrz/ModularPipelineBlocks.intermediate_output_namescCr#)NcSr$r+r%r'r,r,r0rr^z6ModularPipelineBlocks.output_names..rrdr,r,r0 output_namesrz"ModularPipelineBlocks.output_namescCr#)NcSrr,r%r[ componentr,r,r0r#rz9ModularPipelineBlocks.component_names..)rrdr,r,r0component_names!rz%ModularPipelineBlocks.component_namescC"t|j|j|j|jj|j|jdSN)rrrr)rrrrjrkrrrdr,r,r0doc%zModularPipelineBlocks.doc)FNNN)0rkrrr config_namerrrr`r _workflow_map classmethodrrpropertyrrtrrrrrrrrr!rrrrrrboolrrosPathLikerrrMrrr"rr(r*r-r1r,r,r,r0r!sz      /  rc@s2eZdZdZgZgZgZdZddZe ddZ e ddZ e d d Z e d d Z e d eefddZe d eeeeffddZe d eefddZe d eefddZe d eeeffddZd efddZd edBfddZeded efddZd edBfdd Z d!d"Z!e d#d$Z"dS)%ConditionalPipelineBlocksa A Pipeline Blocks that conditionally selects a block to run based on the inputs. Subclasses must implement the `select_block` method to define the logic for selecting the block. Currently, we only support selection logic based on the presence or absence of inputs (i.e., whether they are `None` or not) This class inherits from [`ModularPipelineBlocks`]. Check the superclass documentation for the generic methods the library implements for all the pipeline blocks (such as loading or saving etc.) > [!WARNING] > This is an experimental feature and is likely to change in the future. Attributes: block_classes: List of block classes to be used. Must have the same length as `block_names`. block_names: List of names for each block. Must have the same length as `block_classes`. block_trigger_inputs: List of input names that `select_block()` uses to determine which block to run. For `ConditionalPipelineBlocks`, this does not need to correspond to `block_names` and `block_classes`. For `AutoPipelineBlocks`, this must have the same length as `block_names` and `block_classes`, where each element specifies the trigger input for the corresponding block. default_block_name: Name of the default block to run when no trigger inputs match. If None, this block can be skipped entirely when no trigger inputs are provided. NcCst}t|j|jD]\}}t|r|||<q |||<q ||_t|jt|jks5td|j j d|j durQ|j |jvrStd|j j d|j d|jdSdS)NIn z?, the number of block_classes and block_names must be the same.z, default_block_name 'z' must be one of block_names: ) r zip block_names block_classesrisclassrrurrjrkdefault_block_namerUr block_nameblockr,r,r0rLs   z"ConditionalPipelineBlocks.__init__cCtt|jjSr+)nextiterrrOrrdr,r,r0r]z$ConditionalPipelineBlocks.model_namecCrNrr,rdr,r,r0rarz%ConditionalPipelineBlocks.descriptioncC6g}|jD]}|jD] }||vr||q q|Sr+rrOrrTrUrrDr,r,r,r0re  z-ConditionalPipelineBlocks.expected_componentscCrJr+rrOrrTrUrrDrr,r,r0rnrMz*ConditionalPipelineBlocks.expected_configsrYcCsr|jdurgStt|j}tt|dt}t|jddD]}tt|dt}||q$t|SNrr) rArFrGrrOrVrrtintersection_update)rU first_blockrequired_by_allrDblock_requiredr,r,r0rws  z)ConditionalPipelineBlocks.required_inputscCsDdd|jD}t|}|D]}|j|jvrd|_qd|_q|S)NcSg|] \}}||jfqSr,r&r[rlrDr,r,r0rz4ConditionalPipelineBlocks.inputs..TF)rrr#rlrr)rU named_inputscombined_inputsrr,r,r0rs z ConditionalPipelineBlocks.inputscC dd|jD}t|}|S)NcSrUr,rrVr,r,r0rrWzBConditionalPipelineBlocks.intermediate_outputs..rrr$rU named_outputscombined_outputsr,r,r0rz.ConditionalPipelineBlocks.intermediate_outputscCrZ)NcSrUr,r)rVr,r,r0rrWz5ConditionalPipelineBlocks.outputs..r[r\r,r,r0rr_z!ConditionalPipelineBlocks.outputscC2i}|jD]\}}t|ddr|j||<q|SNrrrrrrUrrCrDr,r,r0r   z'ConditionalPipelineBlocks._requirementscs0fdddd|jD}||j|S)zi Returns a set of all unique trigger input values found in this block and nested blocks. cht}|dur2|D]&\}}t|dr$|jdur$|dd|jD|jr1|j}||q |S)Nblock_trigger_inputscs|] }|dur|VqdSr+r,rr,r,r0rzbConditionalPipelineBlocks._get_trigger_inputs..fn_recursive_get_trigger..rVrrsrfupdaterr trigger_valuesrlrDnested_triggersfn_recursive_get_triggerr,r0ro  zOConditionalPipelineBlocks._get_trigger_inputs..fn_recursive_get_triggercSsh|]}|dur|qSr+r,rr,r,r0rz@ConditionalPipelineBlocks._get_trigger_inputs..)rfrjr)rU all_triggersr,rnr0_get_trigger_inputss z-ConditionalPipelineBlocks._get_trigger_inputscKstd|jjd)a Select the block to run based on the trigger inputs. Subclasses must implement this method to define the logic for selecting the block. Note: When trigger inputs include intermediate outputs from earlier blocks, the selection logic should only depend on the presence or absence of the input (i.e., whether it is None or not), not on its actual value. This is because `get_execution_blocks()` resolves conditions statically by propagating intermediate output names without their runtime values. Args: **kwargs: Trigger input names and their values from the state. Returns: str | None: The name of the block to run, or None to use default/skip. z Subclass z* must implement the `select_block` method.rrr,r,r0 select_blocksz&ConditionalPipelineBlocks.select_blockrc sfdd|jD}|jdi|}|dur|j}|dur+td|jj|fS|j|}ztd|jj||WStyc}zd|jjdt |dt }t |d}~ww) Nc i|] }|dur||qSr+rrrr,r0r]rz6ConditionalPipelineBlocks.__call__..zskipping conditional block: zRunning block: z Error in block: z Error details: Traceback: r,) rfrtrAloggerinforjrkr Exceptionr` traceback format_excerror)rUpipelinertrigger_kwargsrCrDe error_msgr,rvr0__call__s,    z"ConditionalPipelineBlocks.__call__c slfdd|jD}|jdi|}|dur|j}|durdS|j|}|jr4t|ts4|jdiS|S)u Get the block(s) that would execute given the inputs. Recursively resolves nested ConditionalPipelineBlocks until reaching either: - A leaf block (no sub_blocks or LoopSequentialPipelineBlocks) → returns single `ModularPipelineBlocks` - A `SequentialPipelineBlocks` → delegates to its `get_execution_blocks()` which returns a `SequentialPipelineBlocks` containing the resolved execution blocks Args: **kwargs: Input names and values. Only trigger inputs affect block selection. Returns: - `ModularPipelineBlocks`: A leaf block or resolved `SequentialPipelineBlocks` - `None`: If this block would be skipped (no trigger matched and no default) crur+rrrr,r0r]rzBConditionalPipelineBlocks.get_execution_blocks..Nr,)rfrtrArr_LoopSequentialPipelineBlocksr)rUrrrCrDr,rr0rs z.ConditionalPipelineBlocks.get_execution_blocksc Cs|jj}|jjdj}|r|dkr|d|dn|d}|r>|d7}|d7}|d7}|dt|d7}|d 7}|jd}g}|d |dt|d krd| d d |d dDd |d}t |dg}t |ddd}t |dg}t |ddd} d} t|jD]I\} \} } | |jkrd}nd}| d| |d| jjd7} | jd}|d}t|d kr|dd dd |d dD7}| d|d7} q|d|}|r|d|7}| r|d| 7}|d| d7}|S)Nrrg ( Class: ry( g ==================================================================================================== N This pipeline contains blocks that are selected at runtime based on inputs.  Trigger Inputs: h ====================================================================================================  Description: rcs|]}d|VqdSz Nr,r[liner,r,r0r'z5ConditionalPipelineBlocks.__repr__..rr F indent_leveladd_empty_linesr Sub-Blocks: z [default]ru •  () cs|]}d|VqdSz Nr,rr,r,r0r? Description:  rr)rjrk __bases__rssortedrrrTruextendrrr%r& enumeraterrrAstrip)rUr base_classheader desc_linesdescrcomponents_strr configs_str blocks_strirlrD addtional_strblock_desc_lines indented_descresultr,r,r0rsL$       $z"ConditionalPipelineBlocks.__repr__cCr.r/r0rdr,r,r0r1Rr2zConditionalPipelineBlocks.doc)#rkrrrr?r>rfrArr7rrrrrtr`rrrrrrrrrVrsrttorchno_gradrMrrrrr1r,r,r,r0r;1sB     !@r;cs2eZdZdZfddZdedBfddZZS)AutoPipelineBlocksa A Pipeline Blocks that automatically selects a block to run based on the presence of trigger inputs. This is a specialized version of `ConditionalPipelineBlocks` where: - Each block has one corresponding trigger input (1:1 mapping) - Block selection is automatic: the first block whose trigger input is present gets selected - `block_trigger_inputs` must have the same length as `block_names` and `block_classes` - Use `None` in `block_trigger_inputs` to specify the default block, i.e the block that will run if no trigger inputs are present Attributes: block_classes: List of block classes to be used. Must have the same length as `block_names` and `block_trigger_inputs`. block_names: List of names for each block. Must have the same length as `block_classes` and `block_trigger_inputs`. block_trigger_inputs: List of input names where each element specifies the trigger input for the corresponding block. Use `None` to mark the default block. Example: ```python class MyAutoBlock(AutoPipelineBlocks): block_classes = [InpaintEncoderBlock, ImageEncoderBlock, TextEncoderBlock] block_names = ["inpaint", "img2img", "text2img"] block_trigger_inputs = ["mask_image", "image", None] # text2img is the default ``` With this definition: - As long as `mask_image` is provided, "inpaint" block runs (regardless of `image` being provided or not) - If `mask_image` is not provided but `image` is provided, "img2img" block runs - Otherwise, "text2img" block runs (default, trigger is `None`) cst|jdurtd|jjdt|jt|jkr't|j ks2ntd|jjdd|j vrE|j d}|j||_dSdS)Nr<z|, do not set `default_block_name` for AutoPipelineBlocks. Use `None` in `block_trigger_inputs` to specify the default block.zV, the number of block_classes, block_names, and block_trigger_inputs must be the same.) superrrArrjrkrur?r>rfindex)rUidxrjr,r0rs  (  zAutoPipelineBlocks.__init__rYNcKs:t|j|jD]\}}|dur||dur|SqdS)z@Select block based on which trigger input is present (not None).N)r=rfr>rZ)rUr trigger_inputrCr,r,r0rts zAutoPipelineBlocks.select_block)rkrrrrr`rt __classcell__r,r,rr0r^s "rc@sVeZdZdZgZgZeddZeddZeddZ edd Z ed d Z d e fd dZ e d.dee efde dBddfddZddZddZedeee effddZedee fddZedee fddZedee fdd Zed!edefd"d#Zd$d%Zd/d&d'Zd(d)Z ed*d+Z!edee e ffd,d-Z"dS)0SequentialPipelineBlocksa. A Pipeline Blocks that combines multiple pipeline block classes into one. When called, it will call each block in sequence. This class inherits from [`ModularPipelineBlocks`]. Check the superclass documentation for the generic methods the library implements for all the pipeline blocks (such as loading or saving etc.) > [!WARNING] > This is an experimental feature and is likely to change in the future. Attributes: block_classes: list of block classes to be used block_names: list of prefixes for each block cCrrIr,rdr,r,r0rrz$SequentialPipelineBlocks.descriptioncCstdd|jDdS)Ncss |] }|jdur|jVqdSr+)rr[rDr,r,r0rz6SequentialPipelineBlocks.model_name..)rFrrOrdr,r,r0rsz#SequentialPipelineBlocks.model_namecCrJr+rKrLr,r,r0rrMz,SequentialPipelineBlocks.expected_componentscCrJr+rNrOr,r,r0rrMz)SequentialPipelineBlocks.expected_configscCs*|jdurtd|jjt|jS)N@workflows is not supported because _workflow_map is not set for )r5rrjrkrtrWrdr,r,r0rs  z,SequentialPipelineBlocks.available_workflowsrcCs\|jdurtd|jj||jvrtd|d|jj|j|}|jdi|}|S)Nrz Workflow z not found in r,)r5rrjrkrr)rUrtrigger_inputsworkflow_blocksr,r,r0rs    z%SequentialPipelineBlocks.get_workflowN blocks_dictrrYcCsv|}t}|D]\}}t|r|||<q |||<q dd|D|_t||_||_ |dur9||_ |S)zCreates a SequentialPipelineBlocks instance from a dictionary of blocks. Args: blocks_dict: Dictionary mapping block names to block classes or instances Returns: A new SequentialPipelineBlocks instance cSrr,rrr,r,r0rrz=SequentialPipelineBlocks.from_blocks_dict..N) r rrr@rOr?rtrWr>rr)rrrinstancerrlrDr,r,r0from_blocks_dicts    z)SequentialPipelineBlocks.from_blocks_dictcCsnt}t|j|jD]\}}t|r|||<q |||<q ||_t|jt|jks5td|j j ddS)Nr<z?, the number of block_names and block_classes must be the same.) r r=r>r?rr@rrurrjrkrBr,r,r0rs   z!SequentialPipelineBlocks.__init__cCsg}t}|jD]9}|jD]}|j|vr%|jdd|Dvr%||qd}t|tr4|jdur4d}|rCdd|j D}| |q |S)NcSh|]}|jqSr,r%)r[inputr,r,r0rrz7SequentialPipelineBlocks._get_inputs..TFcSrr,r%r[outr,r,r0rrz8SequentialPipelineBlocks._get_inputs..) rVrrOrrlrTr_r;rArrj)rUrrrDinpshould_add_outputsblock_intermediate_outputsr,r,r0 _get_inputs s   z$SequentialPipelineBlocks._get_inputscCrr+rrdr,r,r0r#rzSequentialPipelineBlocks.inputscCsdtt|j}tt|dt}t|jddD]}tt|dt}||qt|SrP)rFrGrrOrVrrtrj)rUrRrequired_by_anyrDrTr,r,r0r's  z(SequentialPipelineBlocks.required_inputscCsLg}|jD]\}}dd|jD}||vr|||jfqt|}|S)NcSrr,r%r[rr,r,r0r8rz@SequentialPipelineBlocks.intermediate_outputs..)rrrrTrr$)rUr]rlrD inp_namesr^r,r,r0r4sz-SequentialPipelineBlocks.intermediate_outputscCrr+rrdr,r,r0rAsz SequentialPipelineBlocks.outputsrc Csz|jD]3\}}z |||\}}Wqty8}zd|d|jjdt|dt}t |d}~ww||fSNz Error in block: (, z) Error details: rw rrrzrjrkr`r{r|rxr})rUr~rrCrDrrr,r,r0rFs z!SequentialPipelineBlocks.__call__csfdd|jS)zW Returns a set of all unique trigger input values found in the blocks. cre)Nrfcsrgr+r,rr,r,r0rbrhzaSequentialPipelineBlocks._get_trigger_inputs..fn_recursive_get_trigger..rirkrnr,r0ro[rpzNSequentialPipelineBlocks._get_trigger_inputs..fn_recursive_get_trigger)rrdr,rnr0rsVs  z,SequentialPipelineBlocks._get_trigger_inputsc sNt|}fddt}|jD]\}}|||}||qt|S)a Get the blocks that would execute given the specified inputs. As the traversal walks through sequential blocks, intermediate outputs from resolved blocks are added to the active inputs. This means conditional blocks that depend on intermediates (e.g., "run img2img if image_latents is present") will resolve correctly, as long as the condition is based on presence/absence (None or not None), not on the actual value. Args: **kwargs: Input names and values. Only trigger inputs affect block selection. Returns: SequentialPipelineBlocks containing only the blocks that would execute cst}t|tr|jdi|}|dur|S|jr@t|ts@|jD]\}}|||}fdd|D}||q#|S||<t|drT|j D]}d||j <qL|S)Ncs i|] \}}d||qS)rr,r|rCr,r0r]rz`SequentialPipelineBlocks.get_execution_blocks..fn_recursive_traverse..rTr,) rr_r;rrrrrjrsrrl)rDrC active_inputs result_blockssub_block_name sub_block nested_blocksrfn_recursive_traverserr0rs        zLSequentialPipelineBlocks.get_execution_blocks..fn_recursive_traverse)rrrrrjrr)rUrr all_blocksrCrDrr,rr0rms    z-SequentialPipelineBlocks.get_execution_blocksc CsN|jj}|jjdj}|r|dkr|d|dn|d}|jdurY|rY|d7}|d7}|d7}|dd d |Dd7}td d |D}|d |d7}|d7}|j}|jdurnt|j}|jd|}|d}g}| d|dt |dkr| dd |ddDd |d}t |dg} t| ddd} t |dg} t| ddd} d} t|jD]@\}\}}| d|d|d|jjd7} |jd}|d}t |dkr|dd dd |ddD7}| d|d7} q|d|}| r|d| 7}| r|d| 7}|d| d 7}|S)!NrrgrryrrrrcSsg|]}|dur|qSr+r,rr,r,r0rrqz5SequentialPipelineBlocks.__repr__..csrgr+r,rr,r,r0rrhz4SequentialPipelineBlocks.__repr__..zR Use `get_execution_blocks()` to see selected blocks (e.g. `get_execution_blocks(z =...)`). rrrrcsrrr,rr,r,r0rrrr Frrr [] rrcsrrr,rr,r,r0rrrrr)rjrkrr5rsrFrr'rrTrurrrr%r&rrrr)rUrrr example_inputr workflow_strrrrrrrrrrlrDrrr,r,r0rsR$         $  z!SequentialPipelineBlocks.__repr__cCsJ|j}|jdurt|j}|jd|}t|j|j||jj|j|j dS)Nr)rrrr) rr5r'r)rrrjrkrr)rUrrr,r,r0r1s  zSequentialPipelineBlocks.doccCr`rarbrcr,r,r0rs   z&SequentialPipelineBlocks._requirementsr+)rYr)#rkrrrr?r>r7rrrrrr`rr6rrrrrrtrrrrrrrrMrrsrrr1rr,r,r,r0rsV           3E rc@seZdZdZdZgZgZedefddZ ede e fddZ ede e fdd Zede efd d Zede efd d Zede efddZeddZeddZddZeddZede efddZede efddZede efddZedeeeffddZd d!Zed"eeefddfd#d$Z d%e!fd&d'Z"d%e!de!fd(d)Z#ed*d+Z$d,d-Z%e&j'j(d2d.d/Z)d0d1Z*dS)3ra5 A Pipeline blocks that combines multiple pipeline block classes into a For Loop. When called, it will call each block in sequence. This class inherits from [`ModularPipelineBlocks`]. Check the superclass documentation for the generic methods the library implements for all the pipeline blocks (such as loading or saving etc.) > [!WARNING] > This is an experimental feature and is likely to change in the future. Attributes: block_classes: list of block classes to be used block_names: list of prefixes for each block NrYcCtd)rz4description method must be implemented in subclassesrrdr,r,r0rz(LoopSequentialPipelineBlocks.descriptioncCrr+r,rdr,r,r0loop_expected_componentsrz5LoopSequentialPipelineBlocks.loop_expected_componentscCrr+r,rdr,r,r0loop_expected_configsrz2LoopSequentialPipelineBlocks.loop_expected_configscCrrr,rdr,r,r0 loop_inputsrz(LoopSequentialPipelineBlocks.loop_inputscCrr+)rrrTrlrr,r,r0loop_required_inputs#s   z1LoopSequentialPipelineBlocks.loop_required_inputscCrrr,rdr,r,r0loop_intermediate_outputs+rz6LoopSequentialPipelineBlocks.loop_intermediate_outputscCTg}|jD]}|jD] }||vr||q q|jD] }||vr'||q|Sr+)rrOrrTrrLr,r,r0r1    z0LoopSequentialPipelineBlocks.expected_componentscCrr+)rrOrrTrrOr,r,r0r>rz-LoopSequentialPipelineBlocks.expected_configscCsg}||jt}|jD]%\}}|jD]}|j|vr'||vr'||qdd|jD}| |q|D]}|j|j vrDd|_ q8d|_ q8|S)NcSrr,r%rr,r,r0rVrz.TF) rrrVrrrrlrTrrjrr)rUrrrlrDrrrr,r,r0rJs     z(LoopSequentialPipelineBlocks._get_inputscCrr+rrdr,r,r0rarz#LoopSequentialPipelineBlocks.inputscCstt|j}tt|dt}tt|dt}||t|jddD]}tt|dt}||q+t|S)Nrrr)rFrGrrOrVrrjrt)rUrRrrequired_by_looprDrTr,r,r0rgs  z,LoopSequentialPipelineBlocks.required_inputscCsJdd|jD}t|}|jD]}|jdd|Dvr"||q|S)NcSrUr,rrVr,r,r0r{rWzELoopSequentialPipelineBlocks.intermediate_outputs..cSrr,r%)r[outputr,r,r0r~rzDLoopSequentialPipelineBlocks.intermediate_outputs..)rrr$rrlrT)rUr]r^rr,r,r0rys  z1LoopSequentialPipelineBlocks.intermediate_outputscCrEr+)rFreversedrrOrrdr,r,r0rrHz$LoopSequentialPipelineBlocks.outputscCr`rarbrcr,r,r0rrdz*LoopSequentialPipelineBlocks._requirementsc Cst}t|j|jD]\}}t|r|||<q |||<q ||_|jD]\}}|jr?td|j j d|d|j j dq&dS)Nr<z9, sub_blocks must be leaf blocks (no sub_blocks). Block 'z' (z) has sub_blocks.) r r=r>r?rr@rrrrjrkrBr,r,r0rs"    z%LoopSequentialPipelineBlocks.__init__rcCsh|}t}|D]\}}t|r|||<q |||<q dd|D|_t||_||_ |S)a Creates a LoopSequentialPipelineBlocks instance from a dictionary of blocks. Args: blocks_dict: Dictionary mapping block names to block instances Returns: A new LoopSequentialPipelineBlocks instance cSrr,rrr,r,r0rrzALoopSequentialPipelineBlocks.from_blocks_dict..) r rrr@rOr?rtrWr>r)rrrrrlrDr,r,r0rs    z-LoopSequentialPipelineBlocks.from_blocks_dictrc Ks|jD]7\}}z |||fi|\}}Wqty<}zd|d|jjdt|dt}t |d}~ww||fSrr)rU componentsrrrCrDrrr,r,r0 loop_steps z&LoopSequentialPipelineBlocks.loop_stepcCr)Nz9`__call__` method needs to be implemented by the subclassr)rUrrr,r,r0rsz%LoopSequentialPipelineBlocks.__call__cCr.r/r0rdr,r,r0r1r2z LoopSequentialPipelineBlocks.docc Cs|jj}|jjdj}|r|dkr|d|dn|d}|jd}g}|d|dt|dkrD|dd |ddDd|d}t |d g}t |d d d }t |dg}t |d d d } d} t |j D]@\} \} } | d| d| d| jjd7} | jd}|d}t|dkr|dddd |ddD7}| d|d7} qn|d|}|r|d|7}| r|d| 7}|d| d7}|S)Nrrgrryrrrcsrrr,rr,r,r0rrz8LoopSequentialPipelineBlocks.__repr__..rr Frrrrrrrcsrrr,rr,r,r0rrrrrr)rjrkrrrrTrurrrr%r&rrrr)rUrrrrrrrrrrrrlrDrrr,r,r0rs:$       $z%LoopSequentialPipelineBlocks.__repr__cCsrt|ds i|_nt|jtstdt|jd|dur't|fi|jS|dur5tdd|i|jStd)N_progress_bar_configz=`self._progress_bar_config` should be of type `dict`, but is rtotalz/Either `total` or `iterable` has to be defined.r,)rsrr_rrrr )rUiterablerr,r,r0 progress_bars  z)LoopSequentialPipelineBlocks.progress_barcKs ||_dSr+)rrr,r,r0set_progress_bar_config s z4LoopSequentialPipelineBlocks.set_progress_bar_configNN)+rkrrrrr?r>r7r`rrtrrrrrrrr!rrrrrrrrrrrr6rrrMrrr1rrcompilerdisablerrr,r,r,r0rsV       6  rc@seZdZdZdZdZdZ      dMdedBdee j BdBde dBdedBde ee fdBd e ee fdBf d d Zed e ee ffd dZedee j BdBfddZee   dNdee j BdBdedBde dBdedBfddZ    dOdee j BdededBdeeBdBdef ddZeddZed efdd Zd!d"Zed ejfd#d$Zed%d&Zed ejfd'd(Zed eefd)d*Z ed eefd+d,Z!ed eefd-d.Z"ed eefd/d0Z#ed e ee ffd1d2Z$d3ed e%fd4d5Z&d6d7Z'dPd8eeeBdBfd9d:Z( dQd;edZ+d e,fd?d@Z-e.dAe%d e fdBdCZ/e.d3edDe ee fd e%fdEdFZ0dGdHZ1dRdIe2dJeeeBfdKdLZ3dS)Sr z Base class for all Modular pipelines. > [!WARNING] > This is an experimental feature and is likely to change in the future. Args: blocks: ModularPipelineBlocks, the blocks to be used in the pipeline zmodular_model_index.jsonNr rr r modular_config_dictr.cKs&|durD|durD|durD|dd}|dd} |dd} |dd} |dd} |dd} || | | | | d }|j|fi|\}}|dur|durR|d }n|j}|durstd }t||d}|dusk|jss|j}t||}|dur{|}n t d |j j ||_ ||_ ||_d d|j jD|_dd|j jD|_|dur|D]5\}}||jvrt|ttfrt|dkr|\}}}|||}d|_||j|<q||jvr||j|_qnJ|dur+|D]@\}}||jvrt|ttfrt|dkr|\}}||||fd}|||}d|_||j|<q||jvr*||j|_qt|dkr=t d|di}|jD]\}}|jdkrS|}nd}|||<qD|jdi|i}|jD] \}}|j||<qj|jdi||j|j dur|j j j ndd||_dS)a Initialize a ModularPipeline instance. This method sets up the pipeline by: - creating default pipeline blocks if not provided - gather component and config specifications based on the pipeline blocks's requirement (e.g. expected_components, expected_configs) - update the loading specs of from_pretrained components based on the modular_model_index.json file from huggingface hub if `pretrained_model_name_or_path` is provided - create defaultfrom_config components and register everything Args: blocks: `ModularPipelineBlocks` instance. If None, will attempt to load default blocks based on the pipeline class name. pretrained_model_name_or_path: Path to a pretrained pipeline configuration. Can be None if the pipeline does not require any additional loading config. If provided, will first try to load component specs (only for from_pretrained components) and config values from `modular_model_index.json`, then fallback to `model_index.json` for compatibility with standard non-modular repositories. components_manager: Optional ComponentsManager for managing multiple component cross different pipelines and apply offloading strategies. collection: Optional collection name for organizing components in the ComponentsManager. **kwargs: Additional arguments passed to `load_config()` when loading pretrained configuration. Examples: ```python # Initialize with custom blocks pipeline = ModularPipeline(blocks=my_custom_blocks) # Initialize from pretrained configuration pipeline = ModularPipeline(blocks=my_blocks, pretrained_model_name_or_path="my-repo/modular-pipeline") # Initialize with components manager pipeline = ModularPipeline( blocks=my_blocks, components_manager=ComponentsManager(), collection="my_collection" ) ``` Notes: - If blocks is None, the method will try to find default blocks based on the pipeline class name - Components with default_creation_method="from_config" are created immediately, its specs are not included in config dict and will not be saved in `modular_model_index.json` - Components with default_creation_method="from_pretrained" are set to None and can be loaded later with `load_components()` (with or without specific component names) - The pipeline's config dict is populated with component specs (only for from_pretrained components) and config values, which will be saved as `modular_model_index.json` during `save_pretrained` - The pipeline's config dict is also used to store the pipeline blocks's class name, which will be saved as `_blocks_class_name` in the config dict NrrFrrrrrrrrrr_blocks_class_namer z6`blocks` is `None`, no default blocks class found for cSi|]}|jt|qSr,rlrr[specr,r,r0r]rqz,ModularPipeline.__init__..cSrr,rrr,r,r0r]rqrr )repor type_hintrUnexpected input ''' provided. This input will be ignored. from_config)rr,) r_load_pipeline_configrZdefault_blocks_namerrrr?rxwarningrjrk_blocks_components_manager _collectionr_component_specsr _config_specsrr_rrtru_dict_to_component_specdefault_creation_methodrXrWcreateregister_componentsr_pretrained_model_name_or_path)rUr rr r rr.rrrrrrrload_config_kwargsblocks_class_namer blocks_classrlrRlibraryrcomponent_spec_dictcomponent_specregister_components_dictr,default_configs config_specr,r,r0r7s<            $      *        zModularPipeline.__init__rYcCs"i}|jjD]}|j||j<q|S)z_ Returns: - Dictionary mapping input names to their default values )rrrXrl)rUparamsrr,r,r0default_call_parameterss z'ModularPipeline.default_call_parametersc Ksz|j|fi|}|dfWSty(}ztd|WYd}~nd}~wwztdddlm}|j|fi|}d|fWStyW}z td|d|d}~ww)Nz1 modular_model_index.json not found in the repo: z try to load model_index.jsonr)DiffusionPipelinezFailed to load config from 'zK'. Could not find or load 'modular_model_index.json' or 'model_index.json'.)rEnvironmentErrorrxdebugr r)rrr rrrr.r,r,r0rs(     z%ModularPipeline._load_pipeline_configrc Kspddlm}z tj|fd|i|}Wnty/}ztd|d}WYd}~nd}~ww|dd} |dd} |d d} |d d} |d d} |d d}| | | | | |d }|j|fi|\}}|durs|||d}n5|durddl m }td|||d}||j }t |td}||}td}t||}n|}d}|d||||||d|}|S)a Load a ModularPipeline from a huggingface hub repo. Args: pretrained_model_name_or_path (`str` or `os.PathLike`, optional): Path to a pretrained pipeline configuration. It will first try to load config from `modular_model_index.json`, then fallback to `model_index.json` for compatibility with standard non-modular repositories. If the pretrained_model_name_or_path does not contain any pipeline config, it will be set to None during initialization. trust_remote_code (`bool`, optional): Whether to trust remote code when loading the pipeline, need to be set to True if you want to create pipeline blocks based on the custom code in `pretrained_model_name_or_path` components_manager (`ComponentsManager`, optional): ComponentsManager instance for managing multiple component cross different pipelines and apply offloading strategies. collection (`str`, optional):` Collection name for organizing components in the ComponentsManager. r )_get_pipeline_classrzEnvironmentError: NrrFrrrrr)rr) _get_modelzB try to determine the modular pipeline class from model_index.jsonr r )r rr r rr.r,) pipelines.pipeline_loading_utilsrrrrrxrrr!diffusers.pipelines.auto_pipelinerrkrrZr2rrr)rrrr r rrr rrrrrrrr rr.rrstandard_pipeline_classrrr*rr~r,r,r0rsp                 zModularPipeline.from_pretrainedTFrsafe_serializationvariantmax_shard_sizerc% Ks|dd}|d|tjjd}|r<|dd} |dd} |dd} |d d} |d d} t|d | | d j}|jD]\}}|j d krKqAt ||d}|durVqA|j }t |rdt |}|j }d}tD]D\}}|tjvryt|}nt|d|d|d|qj|D]\}}t ||d}|durt||r|d}nq|durnqj|durtd|d|dt|dqAt ||}t|}d|jv}d|jv}d|jv}i}|r||d<|r||d<|r|dur||d<tj||}||fi|||jvr qAt|d p|jdk} |s| r@|j|\}}!}"|r+|n||"d<||"d<|jd#i|||!|"fiqA|j |d|r}t!|j"}#t#|| d t$j%d#i|#d | d}$t&|$|#dd }$|$'tj|d!|j(||| | | d"dSdS)$a Save the pipeline and all its components to a directory, so that it can be re-loaded using the [`~ModularPipeline.from_pretrained`] class method. Args: save_directory (`str` or `os.PathLike`): Directory to save the pipeline to. Will be created if it doesn't exist. safe_serialization (`bool`, *optional*, defaults to `True`): Whether to save the model using `safetensors` or the traditional PyTorch way with `pickle`. variant (`str`, *optional*): If specified, weights are saved in the format `pytorch_model..bin`. max_shard_size (`int` or `str`, defaults to `None`): The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size lower than this size. If expressed as a string, needs to be digits followed by a unit (like `"5GB"`). If expressed as an integer, the unit is bytes. push_to_hub (`bool`, *optional*, defaults to `False`): Whether to push the pipeline to the Hugging Face model hub after saving it. **kwargs: Additional keyword arguments: - `overwrite_modular_index` (`bool`, *optional*, defaults to `False`): When saving a Modular Pipeline, its components in `modular_model_index.json` may reference repos different from the destination repo. Setting this to `True` updates all component references in `modular_model_index.json` so they point to the repo specified by `repo_id`. - `repo_id` (`str`, *optional*): The repository ID to push the pipeline to. Defaults to the last component of `save_directory`. - `commit_message` (`str`, *optional*): Commit message for the push to hub operation. - `private` (`bool`, *optional*): Whether the repository should be private. - `create_pr` (`bool`, *optional*, defaults to `False`): Whether to create a pull request instead of pushing directly. - `token` (`str`, *optional*): The Hugging Face token to use for authentication. overwrite_modular_indexFrepo_idrcommit_messageNprivate create_prrupdate_model_cardT)exist_okr$rrz is not installed. Cannot save z as  from rzself.=z of type z cannot be saved.rrr _diffusers_load_idnullrr)r)r is_pipelinemodel_description is_modularr&tags)r/z README.md)rr#r%r,))rrr9pathseprr"rrrrrjrrrsysmodulesrrrxry issubclassrrrrrrrrsr*rrr(r rrformatrsave_upload_folder)%rUrrrr rrr!r"r#r$r%rr&component_namerr, model_clssave_method_name library_namelibrary_classesrrsave_load_methodsclass_candidate save_methodsave_method_signaturesave_method_accept_safesave_method_accept_variant!save_method_accept_max_shard_size save_kwargscomponent_save_pathhas_no_load_idrr card_content model_cardr,r,r0rRs *          "           zModularPipeline.save_pretrainedcCs|jjS)zM Returns: - The docstring of the pipeline blocks )rr1rdr,r,r0r1szModularPipeline.doccCs t|jS)zF Returns: - A copy of the pipeline blocks )rrrdr,r,r0r s zModularPipeline.blocksc Ks|D]\}}|j|}|durtd|dqt||}|jdk}|dur1t|\}}nd\}}||} |||| fi} |sh|rM|j di| t ||||durg|rg|j durg|j |||j qt||d} | |ur|td|dq| dur|durtd|d | jjd n!| dur|durt|| jr| |krtd |d t| jd |r|j di| t ||||dur|r|j dur|j |||j qdS)a Register components with their corresponding specifications. This method is responsible for: 1. Sets component objects as attributes on the loader (e.g., self.unet = unet) 2. Updates the config dict, which will be saved as `modular_model_index.json` during `save_pretrained` (only for from_pretrained components) 3. Adds components to the component manager if one is attached (only for from_pretrained components) This method is called when: - Components are first initialized in __init__: - from_pretrained components not loaded during __init__ so they are registered as None; - non from_pretrained components are created during __init__ and registered as the object itself - Components are updated with the `update_components()` method: e.g. loader.update_components(unet=unet) or loader.update_components(guider=guider_spec) - (from_pretrained) Components are loaded with the `load_components()` method: e.g. loader.load_components(names=["unet"]) or loader.load_components() to load all default components Args: **kwargs: Keyword arguments where keys are component names and values are component objects. E.g., register_components(unet=unet_model, text_encoder=encoder_model) Notes: - When registering None for a component, it sets attribute to None but still syncs specs with the config dict, which will be saved as `modular_model_index.json` during `save_pretrained` - component_specs are updated to match the new component outside of this method, e.g. in `update_components()` method NzAModularPipeline.register_components: skipping unknown component 'rfrrz%ModularPipeline.register_components: z1 is already registered with same object, skippingz.ModularPipeline.register_components: setting 'z' to None (was rrz9ModularPipeline.register_components: replacing existing 'z ' (same type z, new instance)r,)rrrZrxrrsrr_component_spec_to_dictrrraddrrryrjrkr_rr) rUrrlrr is_registeredis_from_pretrainedrrr register_dictcurrent_moduler,r,r0r s`         z#ModularPipeline.register_componentscCs4|j}dd|D}|D]}|jStdS)zi Returns: `torch.device`: The torch device on which the pipeline is located. cSg|] }t|tjjr|qSr,r_rnnModuler[mr,r,r0r[z*ModularPipeline.device..cpu)rrOdevicerrUr3rr,r,r0rWTs   zModularPipeline.devicecCs|jD]9\}}t|tjjsqt|ds|jS|D]}t|dr=t|j dr=|j j dur=t|j j Sqq|jS)z Returns the device on which the pipeline's models will be executed. After calling [`~DiffusionPipeline.enable_sequential_cpu_offload`] the execution device can only be inferred from Accelerate's module hooks. _hf_hookexecution_deviceN) rrr_rrQrRrsrWr3rYrZ)rUrlmodelrr,r,r0_execution_devicebs     z!ModularPipeline._execution_devicecCs0|j}dd|D}|D]}|jStjS)zg Returns: `torch.dtype`: The torch dtype on which the pipeline is located. cSrOr,rPrSr,r,r0rrUz)ModularPipeline.dtype..)rrOrorfloat32rXr,r,r0roys  zModularPipeline.dtypecfddjDS)z] Returns: - list of names for components that needs to be loaded cs(g|]}t|rt|dur|qSr+rsrrrdr,r0rs(z8ModularPipeline.null_component_names..rrWrdr,rdr0null_component_namessz$ModularPipeline.null_component_namescCr)zI Returns: - list of names for all components )rtrrWrdr,r,r0r-szModularPipeline.component_namescr^)zU Returns: - list of names for from_pretrained components c g|] }j|jdkr|qS)rrrrrdr,r0r z>ModularPipeline.pretrained_component_names..r`rdr,rdr0pretrained_component_names z*ModularPipeline.pretrained_component_namescr^)zQ Returns: - list of names for from_config components crb)rrcrrdr,r0rrdz:ModularPipeline.config_component_names..r`rdr,rdr0config_component_namesrfz&ModularPipeline.config_component_namescr^)z Returns: - Dictionary mapping component names to their objects (include both from_pretrained and from_config components) cs"i|] }t|r|t|qSr,r_rrdr,r0r]rz.ModularPipeline.components..r`rdr,rdr0rszModularPipeline.componentsrlcCst|j|S)zh Returns: - a copy of the ComponentSpec object for the given component name )rr)rUrlr,r,r0get_component_specsz"ModularPipeline.get_component_specc sfdd|jD}fdd|jD}|D]z\}}|j|}|jdur>t||js>td|d|jjd|jj|durZ|}t ||rYt ||durYt d|d n|j d krpt |d ddurpt |t|d }nt ||}|j |j krtd |d|j d|j d||j|<qtdkrt d|jdi|i}|D]\}} | |j|_| ||<q|jdi|dS)a Update components and configuration values and specs after the pipeline has been instantiated. This method allows you to: 1. Replace existing components with new ones (e.g., updating `self.unet` or `self.text_encoder`) 2. Update configuration values (e.g., changing `self.requires_safety_checker` flag) In addition to updating the components and configuration values as pipeline attributes, the method also updates: - the corresponding specs in `_component_specs` and `_config_specs` - the `config` dict, which will be saved as `modular_model_index.json` during `save_pretrained` Args: **kwargs: Component objects or configuration values to update: - Component objects: Models loaded with `AutoModel.from_pretrained()` or `ComponentSpec.load()` are automatically tagged with loading information. ConfigMixin objects without weights (e.g., schedulers, guiders) can be passed directly. - Configuration values: Simple values to update configuration settings (e.g., `requires_safety_checker=False`) Examples: ```python # Update pre-trained model pipeline.update_components(unet=new_unet_model, text_encoder=new_text_encoder) # Update configuration values pipeline.update_components(requires_safety_checker=False) ``` Notes: - Components loaded with `AutoModel.from_pretrained()` or `ComponentSpec.load()` will have loading specs preserved for serialization. Custom or locally loaded components without Hub references will have their `modular_model_index.json` entries updated automatically during `save_pretrained()`. - ConfigMixin objects without weights (e.g., schedulers, guiders) can be passed directly. crr,rr[r}rr,r0r]rz5ModularPipeline.update_components..crr,rrirr,r0r]rNz*ModularPipeline.update_components: adding z with new type: z, previous type: z+ModularPipeline.update_components: setting z to None (spec unchanged)rr*rlrzKModularPipeline.update_components: changing the default_creation_method of r(z to rrz/Unexpected keyword arguments, will be ignored: r,)rrrrr_rxryrjrkrsrrrrrfrom_componentrurWr rXr) rUrpassed_componentspassed_config_valuesrlr,current_component_specnew_component_specconfig_to_register new_valuer,rr0update_componentss@%         z!ModularPipeline.update_componentsnamesc s|durfddjD}nt|tr|g}nt|ts(tdt|fdd|D}fdd|D}t|dkrHt d |i}|D]}j|}i}| D]#\} } t| t sg| || <qY|| vrr| ||| <qYd | vr|| d || <qYd } d |vrj dur|j j kr|d d } |j std|dqLz |jdi|||<WqLtyt} | rd | vrd|d|j d|dj d|d|j d|d|d} nd|d|d|d| } t | YqLwjdi|dS)a Load selected components from specs. Args: names: list of component names to load. If None, will load all components with default_creation_method == "from_pretrained". If provided as a list or string, will load only the specified components. **kwargs: additional kwargs to be passed to `from_pretrained()`.Can be: - a single value to be applied to all components to be loaded, e.g. torch_dtype=torch.bfloat16 - a dict, e.g. torch_dtype={"unet": torch.bfloat16, "default": torch.float32} - if potentially override ComponentSpec if passed a different loading field in kwargs, e.g. `pretrained_model_name_or_path`, `variant`, `revision`, etc. - if potentially override ComponentSpec if passed a different loading field in kwargs, e.g. `pretrained_model_name_or_path`, `variant`, `revision`, etc. Ncs@g|]}j|jdkrj|jdurt|ddur|qS)rN)rrrrrrdr,r0r# sz3ModularPipeline.load_components..zInvalid type for names: csh|] }|jvr|qSr,rrrdr,r0r/ rWz2ModularPipeline.load_components..csh|] }|jvr|qSr,rtrrdr,r0r0 rWrz$Unknown components will be ignored: rXFrTzSkipping component `z&`: no pretrained model path specified.zFailed to load component `z` from external repository `z3`. `trust_remote_code=True` was not forwarded to `zC` because it comes from a different repository than the pipeline (`a`). For safety, `trust_remote_code` is only forwarded to components from the same repository as the pipeline. You need to load this component manually with `trust_remote_code=True` and pass it to the pipeline via `pipe.update_components()`. For example, if it is a custom model: z = AutoModel.from_pretrained("z4", trust_remote_code=True) pipe.update_components(r)rzFailed to create component z: - Component spec: z - load() called with kwargs: ze If this component is not required for your workflow you can safely ignore this message. Traceback: r,)rrWr_r`rtrrrurxrrrr rrryloadrzr{r|r )rUrsrcomponents_to_load unknown_namescomponents_to_registerrlrcomponent_load_kwargsrQrRtrust_remote_code_strippedtb warning_msgr,rdr0load_components s               zModularPipeline.load_components raise_errorrcCsZddlm}|dur|jn|g}dd|D}|D]}||r*|r'tddSqdS)Nr )_is_group_offload_enabledcSrOr,rPr+r,r,r0ry rUzNModularPipeline._maybe_raise_error_if_group_offload_active..zYou are trying to apply model/sequential CPU offloading to a pipeline that contains components with group offloading enabled. This is not supported. Please disable group offloading for components of the pipeline to use other offloading methods.TF)hooks.group_offloadingrrrOr)rUr~rrrr,r,r,r0*_maybe_raise_error_if_group_offload_actives s z:ModularPipeline._maybe_raise_error_if_group_offload_activec s2ddlmddlmmm}m}|dd}|dd}|dd}d}d} t|d krKt |d t j r;|d }n@|d durHt |d nd} n0t|dkrqt |d t j r]t d |d durjt |d nd} |d }n t|dkr{t d |dur|durt d |p|}|dur| durt d|p| }|durt |jnd} tfdd|jD} fddfddtfdd|jD} |jduot|jd k} | rt d| dvr| r| st d| rddrt dtfdd|jD}|r | dvr tdd|jd| d krX|d!d"rX|rXd#tjd$<td%d dl}tt d rJt jsNt d&d#tjd'<td(|j}d)d*|D}|ph| }|D]}|\}}}|j|d+}|s|r|durtd,|j j!d-|rd.nd/d0|d1|rd.nd/d2 |r|durtd,|j j!d3|d4|j d5|jd|d6r|durtd,|j j!d7|d8|r|dur|d9d:r|j"|d;n|s|s|s|"|||j t j#krt$|duM Performs Pipeline dtype and/or device conversion. A torch.dtype and torch.device are inferred from the arguments of `self.to(*args, **kwargs).` > [!TIP] > If the pipeline already has the correct torch.dtype and torch.device, then it is returned as is. Otherwise, > the returned pipeline is a copy of self with the desired torch.dtype and torch.device. Here are the ways to call `to`: - `to(dtype, silence_dtype_warnings=False) → DiffusionPipeline` to return a pipeline with the specified [`dtype`](https://pytorch.org/docs/stable/tensor_attributes.html#torch.dtype) - `to(device, silence_dtype_warnings=False) → DiffusionPipeline` to return a pipeline with the specified [`device`](https://pytorch.org/docs/stable/tensor_attributes.html#torch.device) - `to(device=None, dtype=None, silence_dtype_warnings=False) → DiffusionPipeline` to return a pipeline with the specified [`device`](https://pytorch.org/docs/stable/tensor_attributes.html#torch.device) and [`dtype`](https://pytorch.org/docs/stable/tensor_attributes.html#torch.dtype) Arguments: dtype (`torch.dtype`, *optional*): Returns a pipeline with the specified [`dtype`](https://pytorch.org/docs/stable/tensor_attributes.html#torch.dtype) device (`torch.Device`, *optional*): Returns a pipeline with the specified [`device`](https://pytorch.org/docs/stable/tensor_attributes.html#torch.device) silence_dtype_warnings (`str`, *optional*, defaults to `False`): Whether to omit warnings if the target `dtype` is not compatible with the target `device`. Returns: [`DiffusionPipeline`]: The pipeline converted to specified `dtype` and/or `dtype`. r _check_bnb_status)ris_accelerate_versionis_hpu_availableis_transformers_versionroNrWsilence_dtype_warningsFrrzbWhen passing two arguments, make sure the first corresponds to `device` and the second to `dtype`.zPPlease make sure to pass at most two arguments (`device` and `dtype`) `.to(...)`zgYou have passed `dtype` both as an argument and as a keyword argument. Please only pass one of the two.zhYou have passed `device` both as an argument and as a keyword argument. Please only pass one of the two.c3s |] \}}t|VqdSr+)anyr[rrrr,r0r rz%ModularPipeline.to..csfrddr dS|\}}}|rdSt|do2t|jtjjp2t|jdo2t|jjdtjjS)N.module_is_sequentially_offloadedcs.rddr dSt|dot|jtjjS)Nrz 0.17.0.dev0FrY)rsr_rYrr CpuOffloadr)rrr,r0module_is_offloaded sz/ModularPipeline.to..module_is_offloadedc3|] \}}|VqdSr+r,r)rr,r0r s  zIt seems like you have activated a device mapping strategy on the pipeline which doesn't allow explicit device placement using `to()`. You can call `reset_device_map()` to remove the existing device map from the pipeline.)cudaxpua7It seems like you have activated sequential model offloading by calling `enable_sequential_cpu_offload`, but are now attempting to move the pipeline to GPU. This is not compatible with offloading. Please, move your pipeline `.to('cpu')` or consider removing the move altogether if you use sequential offloading.rz 1.1.0.dev0zYou are trying to call `.to('cuda')` on a pipeline that has models quantized with `bitsandbytes`. Your current `accelerate` installation does not support it. Please upgrade the installation.c3rr+r,r)rr,r0r rha5It seems like you have activated model offloading by calling `enable_model_cpu_offload`, but are now manually moving the pipeline to GPU. It is strongly recommended against doing so as memory gains from offloading are likely to be lost. Offloading automatically takes care of moving the individual components rz to GPU when needed. To make sure offloading works as expected, you should consider moving the pipeline back to CPU: `pipeline.to('cpu')` or removing the move altogether if you use offloading.hpu hpu_migrationT1PT_HPU_GPU_MIGRATIONz0Environment variable set: PT_HPU_GPU_MIGRATION=1zBYou are trying to call `.to('hpu')` but HPU device is unavailable.PT_HPU_MAX_COMPOUND_OP_SIZEz7Environment variable set: PT_HPU_MAX_COMPOUND_OP_SIZE=1cSrOr,rPrSr,r,r0r rUz&ModularPipeline.to..rz The module 'z$' has been loaded in `bitsandbytes` 4bit8bitz and conversion to z& is not supported. Module is still in z precision.z:' has been loaded in `bitsandbytes` 8bit and moving it to z2 via `.to()` is not supported. Module is still on r)r~rz&' is group offloaded and moving it to z via `.to()` is not supported.>z4.44.0)rW)rVaPipelines loaded with `dtype=torch.float16` cannot run with `cpu` device. It is not recommended to move them to `cpu` as running them will fail. Please make sure to use an accelerator to run the pipeline in inference, due to the lack of support for`float16` operations on this device in PyTorch. Please, remove the `torch_dtype=torch.float16` argument, or use another device for inference.)%pipelines.pipeline_utilsrutilsrrrrrrur_rrorWrrrrr hf_device_maprxrrrWr9environrhabana_frameworks.torchrsr is_availablerOrrjrktofloat16r`)rUargsrrrrorWr dtype_arg device_arg device_typepipeline_has_bnb"pipeline_is_sequentially_offloadedis_pipeline_device_mappedpipeline_is_offloadedhabana_frameworksr3 is_offloadedrris_loaded_in_4bit_bnbris_group_offloadedr,)rrrrrr0r s                   4   zModularPipeline.torcsVjdkrdSjdurtj\}}nd}d}fddD}d||fi|S)u Convert a ComponentSpec into a JSON‐serializable dict for saving as an entry in `modular_model_index.json`. If the `default_creation_method` is not `from_pretrained`, return None. This dict contains: - "type_hint": tuple[str, str] Library name and class name of the component. (e.g. ("diffusers", "UNet2DConditionModel")) - All loading fields defined by `component_spec.loading_fields()`, typically: - "pretrained_model_name_or_path": str | None The model pretrained_model_name_or_pathsitory (e.g., "stabilityai/stable-diffusion-xl"). - "subfolder": str | None A subfolder within the pretrained_model_name_or_path where this component lives. - "variant": str | None An optional variant identifier for the model. - "revision": str | None A specific git revision (commit hash, tag, or branch). - ... any other loading fields defined on the spec. Args: component_spec (ComponentSpec): The spec object describing one pipeline component. Returns: dict[str, Any]: A mapping suitable for JSON serialization. Example: >>> from diffusers.pipelines.modular_pipeline_utils import ComponentSpec >>> from diffusers import UNet2DConditionModel >>> spec = ComponentSpec( ... name="unet", ... type_hint=UNet2DConditionModel, ... config=None, ... pretrained_model_name_or_path="path/to/pretrained_model_name_or_path", ... subfolder="subfolder", ... variant=None, ... revision=None, ... default_creation_method="from_pretrained", ... ) >>> ModularPipeline._component_spec_to_dict(spec) { "type_hint": ("diffusers", "UNet2DConditionModel"), "pretrained_model_name_or_path": "path/to/repo", "subfolder": "subfolder", "variant": None, "revision": None, "type_hint": ("diffusers", "UNet2DConditionModel"), "pretrained_model_name_or_path": "path/to/repo", "subfolder": "subfolder", "variant": None, "revision": None, } rNcsi|]}|t|qSr,rrirr,r0r]q rqz;ModularPipeline._component_spec_to_dict..r)rrrloading_fields)rlib_namerload_spec_dictr,rr0rIA s ( z'ModularPipeline._component_spec_to_dict spec_dictcCsJ|}|d\}}|dur|durt||}nd}td||d|S)a Reconstruct a ComponentSpec from a loading specdict. This method converts a dictionary representation back into a ComponentSpec object. The dict should contain: - "type_hint": tuple[str, str] Library name and class name of the component. (e.g. ("diffusers", "UNet2DConditionModel")) - All loading fields defined by `component_spec.loading_fields()`, typically: - "pretrained_model_name_or_path": str | None The model repository (e.g., "stabilityai/stable-diffusion-xl"). - "subfolder": str | None A subfolder within the pretrained_model_name_or_path where this component lives. - "variant": str | None An optional variant identifier for the model. - "revision": str | None A specific git revision (commit hash, tag, or branch). - ... any other loading fields defined on the spec. Args: name (str): The name of the component. specdict (dict[str, Any]): A dictionary containing the component specification data. Returns: ComponentSpec: A reconstructed ComponentSpec object. Example: >>> spec_dict = { ... "type_hint": ("diffusers", "UNet2DConditionModel"), ... "pretrained_model_name_or_path": "stabilityai/stable-diffusion-xl", ... "subfolder": "unet", ... "variant": None, ... "revision": None, ... } >>> ModularPipeline._dict_to_component_spec("unet", spec_dict) ComponentSpec( name="unet", type_hint=UNet2DConditionModel, config=None, pretrained_model_name_or_path="stabilityai/stable-diffusion-xl", subfolder="unet", variant=None, revision=None, default_creation_method="from_pretrained" >>> spec_dict = { ... "type_hint": ("diffusers", "UNet2DConditionModel"), ... "pretrained_model_name_or_path": "stabilityai/stable-diffusion-xl", ... "subfolder": "unet", ... "variant": None, ... "revision": None, ... } >>> ModularPipeline._dict_to_component_spec("unet", spec_dict) ComponentSpec( name="unet", type_hint=UNet2DConditionModel, config=None, pretrained_model_name_or_path="stabilityai/stable-diffusion-xl", subfolder="unet", variant=None, revision=None, default_creation_method="from_pretrained" ) rNrjr,)copyrrr)rlrrrrr,r,r0rw s. z'ModularPipeline._dict_to_component_speccKs4|jjD]\}}t|dr|jdi|qdS)Nrr,)rrrrsr)rUrrrr,r,r0r s  z'ModularPipeline.set_progress_bar_configrrc Ks|durt}nt|}|}|jjD]H}|j}|j}|j}||vr.||| ||q|durL||vrL| |} | D] \} } || | |q?q|dur\||j vr\||||qt |dkrnt d|dt)z |||\} }Wntyd|jjjd} t| wWdn1swY|dur|St|tr||St|ttfr||Std|d) a Execute the pipeline by running the pipeline blocks with the given inputs. Args: state (`PipelineState`, optional): PipelineState instance contains inputs and intermediate values. If None, a new `PipelineState` will be created based on the user inputs and the pipeline blocks's requirement. output (`str` or `list[str]`, optional): Optional specification of what to return: - None: Returns the complete `PipelineState` with all inputs and intermediates (default) - str: Returns a specific intermediate value from the state (e.g. `output="image"`) - list[str]: Returns a dictionary of specific intermediate values (e.g. `output=["image", "latents"]`) Examples: ```python # Get complete pipeline state state = pipeline(prompt="A beautiful sunset", num_inference_steps=20) print(state.intermediates) # All intermediate outputs # Get specific output image = pipeline(prompt="A beautiful sunset", output="image") # Get multiple specific outputs results = pipeline(prompt="A beautiful sunset", output=["image", "latents"]) image, latents = results["image"], results["latents"] # Continue from previous state state = pipeline(prompt="A beautiful sunset") new_state = pipeline(state=state, output="image") # Continue processing ``` Returns: - If `output` is None: Complete `PipelineState` containing all inputs and intermediates - If `output` is str: The specific intermediate value from the state (e.g. `output="image"`) - If `output` is list[str]: Dictionary mapping output names to their values from the state (e.g. `output=["image", "latents"]`) NrrrzError in block: (z): zOutput 'z' is not a valid output type)rMrrrrrlrXrSrVrrrOruwarningswarnrWrrrzrjrkrxr}r_r`rZrtrr)rUrrr passed_kwargsexpected_input_paramrlrXrS kwargs_dictr}rwrrr,r,r0r sJ(         zModularPipeline.__call__)NNNNNNr3)TNNFr+)FNr)4rkrrrr4rrrr`r9r:rrrrr7rr6rrr8rintrr1r r rrWr\rortrar-rergrrrhrrr}rQrRrr r staticmethodrIrrrMrr,r,r,r0r (s       W   h     Qc  <5 < r r+)Orrr9r2r{r collectionsrrr dataclassesrrtypingrrhuggingface_hubrhuggingface_hub.utilsr tqdm.autor typing_extensionsr configuration_utilsr r rrrrrrrrrutils.dynamic_modules_utilsrrutils.hub_utilsrrutils.torch_utilsrr rmodular_pipeline_utilsrrrrr r!r"r#r$r%r&r'r(r)r get_loggerrkrxr`r2r6r;r?rrMrrr;rrrr r,r,r,r0s~          @           YG/=e,