o
    #iU                     @   s   d dl Z d dlmZmZmZmZmZmZ d dlm	Z	 d dl
Z
d dlZd dlmZ d dlmZ d dlmZ ddgZe
eZed	d
G dd dZed	d
			ddedejjdeegef deeeef  dee dee fddZdS )    N)AnyCallableDictListOptionalSet)OrderedDict)compatibility)GraphModule)Node	Partitionsplit_moduleT)is_backward_compatiblec                   @   s(   e Zd ZdefddZdefddZdS )r   namec                 C   sN   || _ d| | _g | _i | _i | _i | _i | _tjj	
 | _	i | _i | _d S )Nsubmod_)r   submod_name
node_namesinputsoutputsdependencies
dependentstorchfxgraphGraphenvironmenttargets)selfr    r   Z/home/ubuntu/SoloSpeech/.venv/lib/python3.10/site-packages/torch/fx/passes/split_module.py__init__   s   
zPartition.__init__returnc                 C   s4   d| j  d| j d| j d| j d| j d| j S )Nzname: z
,
 nodes: z,
 inputs: z,
 outputs: z,
 partitions depended on: z,
 partition dependents: )r   r   r   r   r   r   )r   r   r   r   __repr__   s   
zPartition.__repr__N)__name__
__module____qualname__strr    r"   r   r   r   r   r      s    Fmroot_msplit_callbackqualname_mapkeep_original_orderkeep_original_node_namec           5   
      s	  dt dttt f dtttjjjf ffdd}ddl}i 
i i dt d	tt  f
fd
d
fdd}tj	j
tj	jtjjg}	t }
t }i }d}t }jjD ]Љjdv rjdkrjd }durt|tjrt|jj|jr|jj< qU| jdkrj|	v rjtjjkrtjdksJ tjd tsJ }th|
|< nRjtj	j
krtdd jD sJ | th|< d|< n,jtj	jkrtjdksJ |jd   | jd  |jd < |dur|
|  |D ]}||  qqUtdd |! D s6J ddd |" D }dd |
" D }
t#$t%j&r[t#'d| t#'d|
 t|pct|
}d}jjD ]aj(< jdv ryqjjdkrtjj)jd fdd  qj|r}||ksJ d!| d"| |}j|	vrtjj)jfd#d  tjj)j*fd$d  qjt+
, }g }
" D ]\}	t	j-s|.| qg }|r|/ }|.| 
| j0D ]}
| j-/| 
| j-s|.| q|st|t
kr#t1d%||
fD ]R}|" D ]J\}t|dks:J 
t|d  j2< |dd D ]*}
t| 		jj3jjt4d&d jD i j5d'}j6 |_|	j2< qKq-q'|D ](}
| 		j7D ]}	jj8|| j5d(}| j6 |_|	j2| < qq|jjD ]t9d)rL
j: 		j2tjj)jfd*d } tjj)j*fd+d }!jd,vr݈j}"n>j;d-}#}$|#D ]}%t9|$|%st<d.j d/t=|$|%}$qd0>|#}"|$	j?|"< |dur	j@ d-|" }&j||&< t| t4s#J t|!tAs+J r1j(nd}'	jj3j|"| |!j5|'d1}j6 |_|	j2< q|fD ]M}tB|D ]E| }t|dksfJ |dd D ].}
t| 	| }(|(dusJ d2	jj3|(j|(j	j2 fi |(j5d'}|(j6 |_qlqWqQi i  tjjC i })|sjjD ]| |)\ })qnjjD ]j(< q|s|n|}*t }+d3d4 jjD |*D ]}
| 	t4	fd5d	jDD },t|,}-|-dkr	jE|,d  n|-dkr	jE|, |rSfd6d4	j7D }.D ]|+v r*q!| |)\ }/|+ q!|.D ]|+v rDq;| |)\ })|+ q;tjj	j?	j|)	j@< F	j@t4 fd7d	j7D }0t	jD}1|1dkrtjjGH|0}2tI	jDD ]\}3}4|2|3 j |4< qq|1dkr|0 tJtK	jD< qjjD ]jdkrĈEtjj)jd  fd8d  qtjj|)S )9a  
    Creates subgraphs out of main graph

    Args:
        m (GraphModule): Graph module to split
        root_m (torch.nn.Module): root nn module. Not currently used. Included
            because the root nn module is usually transformed via
            torch.fx._symbolic_trace.symbolic_trace (see example below)
        split_callback (Callable[[Node], int]): Callable function
            that maps a given Node instance to a numeric partition identifier.
            split_module will use this function as the policy for which operations
            appear in which partitions in the output Module.
        qualname_map: Optional[Dict[str, str]]: optional output parameter that returns a
            mapping from new target names in the module after split to old target
            names in the original module.
        keep_original_order: Optional[bool]: keep the original order of the GraphModule
            or use the Topological order of the new constructed GraphModule


    Returns:
        GraphModule: the module after split.

    Example:

        This is a sample setup:

            import torch
            from torch.fx.symbolic_trace import symbolic_trace
            from torch.fx.graph_module import GraphModule
            from torch.fx.node import Node
            from torch.fx.passes.split_module import split_module

            class MyModule(torch.nn.Module):
                def __init__(self):
                    super().__init__()
                    self.param = torch.nn.Parameter(torch.rand(3, 4))
                    self.linear = torch.nn.Linear(4, 5)

                def forward(self, x, y):
                    z = self.linear(x + self.param).clamp(min=0.0, max=1.0)
                    w = self.linear(y).clamp(min=0.0, max=1.0)
                    return z + w

            # symbolically trace model
            my_module = MyModule()
            my_module_traced = symbolic_trace(my_module)

            # random mod partitioning
            partition_counter = 0
            NPARTITIONS = 3

            def mod_partition(node: Node):
                global partition_counter
                partition = partition_counter % NPARTITIONS
                partition_counter = (partition_counter + 1) % NPARTITIONS
                return partition

            # split module in module with submodules
            module_with_submodules = split_module(
                my_module_traced, my_module, mod_partition
            )

        Output looks like this. Original graph is broken into partitions

            > print(module_with_submodules)
            GraphModule(
                (submod_0): GraphModule(
                    (linear): Linear(in_features=4, out_features=5, bias=True)
                )
                (submod_1): GraphModule(
                    (linear): Linear(in_features=4, out_features=5, bias=True)
                )
                (submod_2): GraphModule()
            )

            def forward(self, x, y):
                param = self.param
                submod_0 = self.submod_0(x, param, y);  x = param = y = None
                getitem = submod_0[0]
                getitem_1 = submod_0[1];  submod_0 = None
                submod_1 = self.submod_1(getitem, getitem_1);  getitem = getitem_1 = None
                getitem_2 = submod_1[0]
                getitem_3 = submod_1[1];  submod_1 = None
                submod_2 = self.submod_2(getitem_2, getitem_3);  getitem_2 = getitem_3 = None
                return submod_2

        Output of split module is the same as output of input traced module.
        This is an example within a test setting:

            > orig_out = my_module_traced(x, y)
            > submodules_out = module_with_submodules(x, y)
            > self.assertEqual(orig_out, submodules_out)
            True
    nodebase_mod_envbase_mod_attrsc                    s  | j dkrKt| jdkr| jd ntjj}r1|tjju rdn|f} jd| j|| jd|| j< n j	| j
| j|d|| j< | j || j _||fS | j dkr | j
|| j< | j || j _}| j
dD ]}t||sztd| j
 d	t||}qj||| j
< ||fS )
Nplaceholderr   r   )args	type_expr)r2   default_valueget_attr.zNode target  not found!)oplenr1   inspect	Signatureemptycreate_noder   typer0   targetmetacopyr4   splithasattrAttributeErrorgetattr)r-   r.   r/   r3   r1   attr_valatom)base_mod_graphr,   r'   r   r   construct_graph   s*   





z%split_module.<locals>.construct_graphr   Ndef_nodeuse_nodec           	         s   ddl m} t| dd }t|dd }||krh|d ur/ | }|j| j |d ur/|j| |d urj | }|j| j | j	d }d ur\t
||tdD ]}|j| j qP|d url|j| d S d S d S d S )Nr   )free_symbols_fx_partitionexample_value)key)%torch.fx.experimental.symbolic_shapesrK   rD   r   
setdefaultr   r   r   r?   getsortedr&   r   )	rI   rJ   rK   defineduseddef_partitionuse_partitiondef_vals)
partitionssymbol_to_noder   r   record_cross_partition_use   s(   z0split_module.<locals>.record_cross_partition_usec                    sF   t | } |}|d u rt|  |< }|j| j || _d S N)r&   rQ   r   r   appendr   rL   )r-   partition_name	partition)rY   r)   r   r   "instantiate_node_partition_mapping   s   

z8split_module.<locals>.instantiate_node_partition_mapping)r0   r4   outputr0   rM   call_function   c                 s   s    | ]	}t |t V  qd S r\   )
isinstancer   .0argr   r   r   	<genexpr>  s    zsplit_module.<locals>.<genexpr>c                 s   s    | ]}|d uV  qd S r\   r   )rf   vr   r   r   rh         zautocast must exitc                 S      i | ]	\}}|t |qS r   rR   rf   kri   r   r   r   
<dictcomp>      z split_module.<locals>.<dictcomp>c                 S   rk   r   rl   rm   r   r   r   ro     rp   zautocast_regions: %szgrad_regions: %s)r0   r4   ra   c                    s
    | d S r\   r   n)r[   r   r   <lambda>,     
 zsplit_module.<locals>.<lambda>zRautocast or set_grad_enabled require monotonically increasing partitions:highest: z, this node's: c                    
   |  S r\   r   rI   r-   r[   r   r   rt   ;  ru   c                    rv   r\   r   rw   rx   r   r   rt   >  ru   z cycle exists between partitions!c                 s   s    | ]}|V  qd S r\   r   re   r   r   r   rh   ^  s    )r7   r>   r1   kwargsr2   )r2   rL   c                        |  S r\   r   rr   r   r   r   rt   w      c                    rz   r\   r   rr   r{   r   r   rt   y  r|   )call_moduler4   r5   zOperator target r6   _)r7   r>   r1   ry   r2   r   zMissing exit nodec                 S   s   g | ]	}|j d kr|qS )r0   )r7   )rf   r-   r   r   r   
<listcomp>  rp   z split_module.<locals>.<listcomp>c                 3   s    | ]
}j  |  V  qd S r\   r{   rf   r   )
orig_nodesr_   r   r   rh     s    
c                    s   g | ]
}|vr | qS r   r   )rf   rN   )orig_mod_envoriginal_orderr   r   r     s    c                 3   s    | ]} | V  qd S r\   r   r   r.   r   r   rh     rj   c                    s
    | j  S r\   )r   rr   r   r   r   rt     ru   )Lr   r   r&   r   r   graph_moduler
   sympyr   amp_enter_autocast_exit_autocast_C_set_grad_enabledr   setr   nodesr7   r?   rQ   rd   SymIntr-   exprSymbolr>   r8   r1   boolalladdremovevaluesitems_LOGGERisEnabledForloggingDEBUGdebugr   map_argry   listkeysr   r]   popr   RuntimeErrorr   r<   tupler=   r@   r   r0   rB   rL   rA   rC   rD   joinr   r   dictreversedr   r   ra   r}   proxyProxy	enumeratenextiter)5r'   r(   r)   r*   r+   r,   rH   r   r`   GLOBAL_STATE_NODESgrad_regionsautocast_regionsautocast_exitsactive_gradactive_autocastsvalaassert_monotonically_increasinghighest_partitionpidoriginal_partition_orderroot_partitionsr^   sorted_partitionsroot_partition	dependentregions_mappingregionsrnew_nodeinpr0   gathered_argsgathered_kwargsr>   target_atomstarget_attrrF   qualnamer   	exit_noder/   construct_order_partitionsalready_constructed_attr_nodesoutput_valsnum_output_valsorig_mod_attr_nodesbased_mod_attrs
output_valnum_outputsoutput_val_proxyioutput_namer   )r.   rG   r   r,   r'   r-   r   r   r   r_   rY   r[   r)   rZ   r   r   *   s  h

	





 




	







	







)NFF)r9   typingr   r   r   r   r   r   collectionsr   r   r   torch.fx._compatibilityr	   torch.fx.graph_moduler
   torch.fx.noder   __all__	getLoggerr#   r   r   nnModuleintr&   r   r   r   r   r   r   <module>   s:    
