o
    }oi&                     @   s   d dl Z d dlZd dlmZ d dlmZ G dd dejZG dd dejZdd Z	dd
dZ
dddZG dd dZG dd dZG dd dZdS )    N)fnc                       s$   e Zd Z fddZdd Z  ZS )	CustomMLPc                    s*   t    tdd| _tdd| _d S )N
   )super__init__nnLinearlinear1linear2)self	__class__ V/home/ubuntu/.local/lib/python3.10/site-packages/tests/collections/llm/fn/test_base.pyr      s   
zCustomMLP.__init__c                 C      ||  | | S Nr
   r	   r   xr   r   r   forward      zCustomMLP.forward)__name__
__module____qualname__r   r   __classcell__r   r   r   r   r      s    r   c                       s,   e Zd Zdejf fddZdd Z  ZS )	SharedMLPsharedc                    s   t    || _|| _d S r   )r   r   r	   r
   )r   r   r   r   r   r       s   

zSharedMLP.__init__c                 C   r   r   r   r   r   r   r   r   %   r   zSharedMLP.forward)r   r   r   r   Moduler   r   r   r   r   r   r   r      s    r   c                 C   s    t | tjrt| t S | S r   
isinstancer   r   
SequentialReLUr   r   r   r   add_relu)   s   r#   r	   c                 C   s(   ||krt | tjrt| t S | S r   r   )r   name
to_replacer   r   r   add_relu_named/      r&   c                 C   s(   |dkrt | tjrt| t S | S )Nr   r   )r   ir   r   r   add_relu_first5   r'   r)   c                   @   s4   e Zd Zdd Zdd Zdd Zdd Zd	d
 ZdS )TestWalkModulec                 C   s,   t dd}dd }t|||u sJ d S )Nr   c                 S      | S r   r   r"   r   r   r   <lambda>?       z2TestWalkModule.test_map_identity.<locals>.<lambda>)r   r   r   map)r   moduleidentityr   r   r   test_map_identity<   s   z TestWalkModule.test_map_identityc                 C   sD   t dd}t|t}t|d t jsJ t|d t js J d S )Nr   r      )r   r   r   r.   r#   r   r!   )r   r/   transformed_moduler   r   r   test_map_transformB   s   z!TestWalkModule.test_map_transformc                 C   sz   t  }t|t}t|jtjsJ t|jtjsJ t	t
fD ]}tt  |}t|jtjs1J t|jtjs:J qd S r   )r   r   walkr#   r   r	   r   r    r
   r&   r)   r   )r   mlp	with_reluwalk_fnwith_relu_firstr   r   r   test_walk_custom_moduleI   s   z&TestWalkModule.test_walk_custom_modulec                 C   s   dt jfdd}t dd}t|}|jj }|jj }t	||}t
|jjj|d s1J t
|jjj|d s>J t
|jjj|d sKJ t
|jjj|d sXJ |j|ju s`J d S )Nr/   c                 S   s0   t | tjr| j jd9  _| j jd9  _| S )N   )r   r   r   weightdatabiasr/   r   r   r   double_linearU   s   z=TestWalkModule.test_walk_shared_module.<locals>.double_linearr   r;   )r   r   r   r   r<   r=   cloner>   r   r5   torchallcloser	   r
   )r   r@   shared_linearr6   initial_weightinitial_biastransformed_mlpr   r   r   test_walk_shared_moduleT   s   z&TestWalkModule.test_walk_shared_modulec                 C   s&   dt jfdd}tjt |dd d S )Nr/   c                 S   s   t | tjsJ | S r   )r   r   r   r?   r   r   r   	is_linearm   s   z0TestWalkModule.test_leaf_only.<locals>.is_linearT)	leaf_only)r   r   r   r5   r   )r   rI   r   r   r   test_leaf_onlyl   s   zTestWalkModule.test_leaf_onlyN)r   r   r   r1   r4   r:   rH   rK   r   r   r   r   r*   ;   s    r*   c                   @   sf   e Zd Zejdejejgdd Z	ejdejejgdd Z
ejdejejgdd ZdS )	TestWalkListModulemodule_containerc                 C   s   t ddt ddg}|t ju r||nt j| }dd }t||}t||s+J t|dks3J t	|d j
t|d j
sDJ t	|d j
t|d j
sUJ d S )Nr   c                 S      t | tjr| jjd | S Ng      ?r   r   r   r<   r=   fill_r?   r   r   r   r8   {      z>TestWalkListModule.test_walk_module_container.<locals>.walk_fnr;   r   r2   r   r   
ModuleListr    r   r5   r   lenrB   rC   r<   	ones_liker   rM   modulesr/   r8   walked_moduler   r   r   test_walk_module_containerv   s   "&z-TestWalkListModule.test_walk_module_containerc                 C   s   t ddt ddg}|t ju r||nt j| }dd }tj||dd}t||s-J t|dks5J t	|d j
dt|d j
 sHJ t	|d j
dt|d j
 s[J d S )	Nr   c                 S   s   t | tjr| jj| | S r   rP   )r/   valuer   r   r   r8      rR   zJTestWalkListModule.test_walk_module_container_with_kwargs.<locals>.walk_fng       @)r[   r;   r   r2   rS   rW   r   r   r   &test_walk_module_container_with_kwargs   s   &*z9TestWalkListModule.test_walk_module_container_with_kwargsc                 C   s   t t ddt ddt t ddt ddg}|t ju r%||nt j| }dd }t||}t||s;J t|dksCJ |D ]4}t|t jsOJ t|dksWJ t	|d j
t|d j
shJ t	|d j
t|d j
syJ qEd S )Nr   c                 S   rN   rO   rP   r?   r   r   r   r8      rR   zMTestWalkListModule.test_walk_module_container_with_recursion.<locals>.walk_fnr;   r   r2   )r   r    r   rT   r   r5   r   rU   rB   rC   r<   rV   )r   rM   rX   r/   r8   rY   seqr   r   r   )test_walk_module_container_with_recursion   s   "$z<TestWalkListModule.test_walk_module_container_with_recursionN)r   r   r   pytestmarkparametrizer   rT   r    rZ   r\   r^   r   r   r   r   rL   u   s    

rL   c                   @   s   e Zd Zdd Zdd ZdS )TestWalkDictModulec                 C   s   t t ddt dddd}dd }t||}t|t js#J d|v r/t|d t js1J d	|v r=t|d	 t js?J d
S )z
        Test walking through an nn.ModuleDict without applying any transformations,
        essentially testing the identity operation.
        r   r2         linearconvc                 S   r+   r   r   r"   r   r   r   r,      r-   zCTestWalkDictModule.test_walk_module_dict_identity.<locals>.<lambda>rf   rg   N)r   
ModuleDictr   Conv2dr   r5   r   )r   rX   r0   walked_modulesr   r   r   test_walk_module_dict_identity   s   " z1TestWalkDictModule.test_walk_module_dict_identityc                 C   s   t t ddt dddd}d
dt jfdd	}t||}t|t js(J | D ]}t|t j	s6J t|d t j
s@J q,dS )z
        Test walking through an nn.ModuleDict and applying a transformation to each module.
        In this case, we'll add a ReLU activation after each module.
        r   r2   rc   rd   re   Nr/   c                 S   s   |dv rt | t  S | S )Nre   )r   r    r!   )r/   r$   r   r   r   r#      s   zDTestWalkDictModule.test_walk_module_dict_transform.<locals>.add_relur   )r   rh   r   ri   r   r   r5   r   valuesr    r!   )r   rX   r#   rj   r/   r   r   r   test_walk_module_dict_transform   s   "z2TestWalkDictModule.test_walk_module_dict_transformN)r   r   r   rk   rm   r   r   r   r   rb      s    rb   )Nr	   r   )r_   rB   torch.nnr   nemo.collections.llmr   r   r   r   r#   r&   r)   r*   rL   rb   r   r   r   r   <module>   s   



:;