o
    "i,                  	   @   sH  U d dl mZmZmZ d dlZd dlmZ d dlm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d
dlmZmZmZmZmZmZmZ d
dlmZm Z m!Z!m"Z"m#Z#m$Z$ d
dl%m&Z&m'Z' dgZ(ee) e*d< dej+defddZ,dedefddZ-dedede"fddZ.de)dede#fddZ/de)dedede#fddZ0de)dej+de#fdd Z1de)d!efd"d#Z2d$d% Z3d&d' Z4de)d(ed)ee dee  fd*dZ5d+ede!fd,d-Z6de)d.edee# fd/d0Z7dedefd1d2Z8dej+dee fd3d4Z9de)d5ed6edee  fd7d8Z:d+eddfd9d:Z;d;edefd<d=Z<dS )>    )AnycastListN)_get_device_module)ShardMetadata)ShardedTensor)DTensor)%compute_local_shape_and_global_offset)_Checkpointable)tree_map_only_   )BytesStorageMetadataChunkStorageMetadataMetadataIndexSTATE_DICT_TYPESTORAGE_TYPESTensorPropertiesTensorStorageMetadata)LoadItemTypeReadItemSavePlanTensorWriteData	WriteItemWriteItemType)"_check_shard_metadata_pair_overlap+_shards_get_overlap_region_wrt_saved_tensor create_read_items_for_chunk_list__all__tensorreturnc                 C   s$   t tdgt|   |  dS )Nr   offsetssizes)r   torchSizelensize)r    r'   j/home/ubuntu/SoloSpeech/.venv/lib/python3.10/site-packages/torch/distributed/checkpoint/planner_helpers.py_create_chunk_from_tensor)   s   r)   shard_mdc                 C   s   t t| jt| jdS Nr    )r   r#   r$   shard_offsetsshard_sizes)r*   r'   r'   r(   _chunk_for_shard/   s   

r.   sharded_tensorc                 C   s>   |   j}t|j|j|j|j|jd}tt	|||   j
dS )N)dtypelayoutrequires_gradmemory_format
pin_memorychunk
propertiesr&   )metadatatensor_propertiesr   r0   r1   r2   r3   r4   r   r.   r&   )r/   r*   shard_propertiesr7   r'   r'   r(   _sharded_tensor_metadata6   s   
r;   fqnc              	   C   sb   t |j|j|j\}}t|t|}}tt| |tj	t
t||dt| | ddS )Nr    r5   indextypetensor_data)r	   shapedevice_mesh
placementsr#   r$   r   r   r   SHARDr   r   r   create_from_tensorto_localr&   )r<   r   r"   r!   r'   r'   r(   _create_write_items_for_dtensorJ   s    rG   c                 C   s(   t |j}tt| |tjt||dS )Nr=   )r#   r$   r,   r   r   r   rD   r;   )r<   r/   r*   r!   r'   r'   r(   _create_write_item_for_shard^   s   rH   c                 C   sN   t dgt|  }tt| |tjtt	|| dt
|| ddS )Nr   r    r5   r=   )r#   r$   r%   r&   r   r   r   TENSORr   r   r   rE   )r<   r   r!   r'   r'   r(   _create_write_item_for_tensori   s   rJ   bytesc                 C   s   t t| tjdS )N)r>   r?   )r   r   r   BYTE_IO)r<   rK   r'   r'   r(   _create_write_item_for_bytesiov   s   rM   c              	   C   s.   t tj| t|f|t|ft|fdS N)r?   
dest_indexdest_offsetsstorage_indexstorage_offsetslengths)r   r   rL   r#   r$   rO   dest_offsetrQ   storage_offsetlengthr'   r'   r(   _create_read_item_for_byteio}   s   


rX   c              	   C   s(   t tj| t||t|t|dS rN   )r   r   rI   r#   r$   rO   rP   rQ   rR   rS   r'   r'   r(   _create_read_item_for_tensor   s   rZ   checkpoint_mdlocal_chunksc                 C   s   g }t |D ]L\}}t |jD ]B\}}t||sqg }g }	g }
t||dD ]\}}}}|| |	| |
| q%|tt| |j||	t| |j|||
d qq|S )aW  
    Create a list of ``ReadItem`` based on the checkpoint and local chunks.

    This applies the resharding algorithm and computes the reads needed
    to satisfy ``local_chunks`` with a checkpoint described by ``checkpoint_md``.

    Args:
        fqn (str) : The state_dict FQN to pass to ``ReadItem``.
        checkpoint_md (TensorStorageMetadata): metadata for a given tensor
            from a checkpoint.
        local_chunks (List[ChunkStorageMetadata]): Local chunks that needs to be
            loaded.

    Returns:
        A list of ``ReadItem`` that will satisfy all input chunks.
    )saved_shardcurrent_shardrY   )	enumeratechunksr   r   appendrZ   r   r!   )r<   r[   r\   
read_itemsidxshardstorage_idx
storage_mdrR   rP   rS   dimoffset_for_saved_tensoroffset_for_current_tensorrW   r'   r'   r(   r      s<   



state_dictc                 C   s   g }|   D ]@\}}t|tr|t|| qt|tr/| jD ]}|t||| q"qt|t	j
r>|t|| q|t|| qt|S N)items
isinstancer   ra   rG   r   r8   shards_metadatarH   r#   TensorrJ   rM   r   )rj   requestsr<   objr*   r'   r'   r(   "_create_default_metadata_only_plan   s   

rr   objectc                    s   t tr S t tr%t  tr  S t gS t tr6 fdd D S t tj	rBt
 gS t gS )Nc                    s   g | ]	}t  |jqS r'   )rH   r8   .0rd   r<   rs   r'   r(   
<listcomp>   s    z'_create_write_items.<locals>.<listcomp>)rm   r
   _create_write_itemsr   rF   rG   r   local_shardsr#   ro   rJ   rM   rv   r'   rv   r(   rx      s   


rx   c                 C   s8   t | j| j| j\}}t|t|}}t||dS r+   )r	   rA   rB   rC   r#   r$   r   )r   r"   r!   r'   r'   r(   _create_chunk_from_dtensor   s   rz   c                 C   s   t | tr| | }|S t | tr&t |  tr|  | S t| g}|S t | tr6dd |  D }|S t | tj	rCt
| g}|S tdt|  )Nc                 S   s   g | ]}t |jqS r'   )r.   r8   rt   r'   r'   r(   rw     s    
z&_create_chunk_list.<locals>.<listcomp>zMUnsupported Type, expecting one of [Tensor, DTensor, ShardedTensor] ,but got )rm   r
   _create_chunk_listr   rF   rz   r   ry   r#   ro   r)   
ValueErrorr?   )r   r\   r'   r'   r(   r{      s*   





r{   mdrq   c              
   C   sx   t |ts.zt|}W n ty' } ztd|  ddt|  |d }~ww t| ||S tt| dt| dddgS )Nz Invalid checkpoint metadata for z, z(expected BytesStorageMetadata but found r   rT   )rm   r   r{   r|   r?   r   rX   r   )r<   r}   rq   r\   exr'   r'   r(   _create_read_items  s,   

r   c                 C   s   t tjt|  d S rk   )r   r#   ro   _init_meta_tensor)rj   r'   r'   r(   _init_state_dict)  s   r   valuec                 C   s   t | dd}|tdkrVtj j}ttjt|	 }t
| tr=tj|  |d}tj|| j| j|  |  d}|S t
| tjrLtj| |d}|S tdt|  d| S )zY
    Initializes tensor, moves it to device for torch.Tensor/DTensor on meta device.
    deviceNmeta)r   )rB   rC   rA   stridezFound unsupported type z for meta device loading.)getattrr#   r   distdistributed_c10d_get_pg_default_devicer?   r   r   current_devicerm   r   
empty_likerF   
from_localrB   rC   r&   r   ro   RuntimeError)r   r   device_typenew_local_tensordtensorr   r'   r'   r(   r   -  s*   
r   )=typingr   r   r   r#   torch.distributeddistributedr   torch._utilsr   !torch.distributed._shard.metadatar   'torch.distributed._shard.sharded_tensorr   torch.distributed._tensorr    torch.distributed._tensor._utilsr	   $torch.distributed.checkpoint.plannerr
   torch.utils._pytreer   r8   r   r   r   r   r   r   r   plannerr   r   r   r   r   r   
reshardingr   r   r   str__annotations__ro   r)   r.   r;   rG   rH   rJ   rM   rX   rZ   r   rr   rx   rz   r{   r   r   r   r'   r'   r'   r(   <module>   sj   $ 	


7