o
    }oim2                     @   s   d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dl	m
Z
 d dlmZ d dlmZ d dlmZ G dd deZdS )	    N)Txt2NerfBase)LaplacianSmoothLoss)NormalConsistencyLoss)ShadingEnum)optimc                       sd   e Zd Z fddZdd Zdd Zdd Zd	d
 Zdd Zdd Z	dd Z
dd Zdd Z  ZS )DreamFusionc                    s   t t| | |j| _|j| _|j| _|j| _|j| _|j| _|j	j
| _
|j	j| _|j	j| _|j	j| _|j	j| _|j	j| _|j	j| _|j	j| _| jdkrQt | _| jdkrZt | _g | _g | _d S )Nr   )superr   __init__guidance_scaleiterslatent_iter_ratioalbedo_iter_ratiomin_ambient_ratiotextureless_ratioloss
lambda_sdslambda_opacitylambda_entropylambda_orientationlambda_2d_normal_smoothlambda_3d_normal_smoothlambda_mesh_normallambda_mesh_laplacianr   normal_consistency_loss_fnr   laplacian_smooth_loss_fntest_imagestest_depths)selfcfg	__class__ g/home/ubuntu/.local/lib/python3.10/site-packages/nemo/collections/multimodal/models/nerf/dreamfusion.pyr	      s*   










zDreamFusion.__init__c           #         sl  | j | j }|| jk rd}tj}d}n*|| jkrd}d }n| jd| j t   }t }|d| j kr8tj	}ntj
}d}t| j}t| j}	t| j}
t| jpUt| j}t| j}| |d |d |d d||d||	|
||d |rtj d  d	 gd
ddddd }n d dddd }|d }| jd g|jd  }t|jd D ]d}|| dkr|| dk r|| dkrd|| d  }nd|| d  }| jd }| jd }n%|| dkrd|| d d  }n
d|| d d  }| jd }| jd }|| d| |  }|| qtj|dd}i }| jj|||| jd}|| j |d< | jdkrEd	 v rE d	 d  }| j| |d< | jdkrd v r d dd}| t | d| t d|    }| jt!dd| j  | j  }|| |d< | jdkrd v r߈ d }|d d dd d d d d f |d d d d
d d d d f  "  |d d d d dd d d f |d d d d d d
d d f  "   }| j| |d< | j#dkrt$ fd d!d"D r d %  d#  d$  &d
jdd%d  }| }| j#| |d&< | jdkr8t$ fd'd!d(D r8 d)  d#  '  }| j| |d*< | jdkr\t$ fd+d!d,D r\| j( d-  d. d/}| j| |d0< | jdkrt$ fd1d!d2D r| j) d3  d. d2} | j|  |d4< t&|* }!| j+|ddd5 | j,d6|!ddd5 | j-j.d d7 }"| j,d7|"ddd5 | j,d8| j d ddd5 |!S )9N      ?TFrays_orays_dmvpr$   r%   r&   perturbambient_ratioshading_typebinarizereturn_normal_imagereturn_normal_perturbreturn_verticesreturn_facesreturn_faces_normalsimageopacity)dimr            azimuthuncondiZ   frontsideback)	as_latentr
   r   loss_opacityweightsgh㈵>gwJ?loss_entropynormal_imageloss_smoothc                 3       | ]}| v V  qd S Nr!   .0keyoutputsr!   r"   	<genexpr>       z,DreamFusion.training_step.<locals>.<genexpr>)r@   normalsdirsrM   rN   )minloss_orientationc                 3   rD   rE   r!   rF   rI   r!   r"   rK      rL   )rM   normal_perturbrQ   loss_normal_smoothc                 3   rD   rE   r!   rF   rI   r!   r"   rK      rL   )face_normalsfacesrS   rT   )rS   	t_pos_idxnormal_consistency_lossc                 3   rD   rE   r!   rF   rI   r!   r"   rK      rL   )vertsrT   rW   laplacian_loss)prog_barrank_zero_onlyr   lrglobal_step)/r\   r   r   r   NORMALr   r   randomr   TEXTURELESS
LAMBERTIANboolr   r   r   r   torchcatpermute
contiguoustext_zshaperangeappendguidance
train_stepr
   r   r   meanr   clamplog2rO   squarer   alldetachsumabsr   r   valueslog_dictlog
_optimizerparam_groups)#r   batch	batch_idxexp_iter_ratior)   r*   r>   randr,   r-   r.   r/   r0   pred_rgbr8   rf   brstart_zend_zpos_z	loss_dictguidance_lossr?   alphasrA   r   pred_normalrC   rP   loss_normal_perturbrV   rX   r   r[   r!   rI   r"   training_step=   s   





(



*F>$,$$$zDreamFusion.training_stepc           	      C   s   |  |\}}tj| jjd}tj|dd tt||D ]O\}\}}t	
tj|| jdd| jdd| jdd|dd|dd
t	|t	j t	
tj|| jdd| jdd| jdd|dd|dd
| qd S )N
validationTexist_ok04d_z_rgb.pngz
_depth.png)_shared_predictospathjointrainerlog_dirmakedirs	enumeratezipcv2imwritecurrent_epochr\   global_rankcvtColorCOLOR_RGB2BGR)	r   ry   rz   imagesdepths	save_pathir1   depthr!   r!   r"   validation_step   s&   00zDreamFusion.validation_stepc                 C   s*   |  |\}}| j| | j| d S rE   )r   r   ri   r   )r   ry   rz   r   r   r!   r!   r"   	test_step   s   zDreamFusion.test_stepc              
   C   s   t j| jjd}t j|dd tj| jdd}t	j
t jt j|| jdd| jdd|d	d
dd tj| jdd}t	j
t jt j|| jdd| jdd|d	d
dd | j  | j  d S )NtestTr   r   )axisr   r   z_rgb.mp4      r6   )fpsqualitymacro_block_sizez
_depth.mp4)r   r   r   r   r   r   npconcatenater   imageiomimwriter   r\   r   clear)r   r   r   r   r!   r!   r"   on_test_epoch_end   s(   ((
zDreamFusion.on_test_epoch_endc                 C   s   |  | |S rE   )r   )r   ry   rz   r!   r!   r"   predict_step   s   zDreamFusion.predict_stepc                 C   s&   | j |||||||||	|
||d}|S )Nr'   )renderer)r   r$   r%   r&   r(   r)   r*   r+   r,   r-   r.   r/   r0   rJ   r!   r!   r"   forward   s   zDreamFusion.forwardc                 C   s   | |d |d |d dd|v r|d ndd|v r|d nd ddddddd}|d	     }|d
 tj}|d     }||  t|d  }|d
 tj}||fS )Nr$   r%   r&   Fr)   r#   r*   r'   r1      r   gư>)rq   cpunumpyastyper   uint8rO   ptp)r   datarJ   	images_np	depths_npr!   r!   r"   r     s(   zDreamFusion._shared_predictc                 C   sH   | j j}t|}|dd  t|j}|dd|  i|}|| _d S )Nnameparamsr!   )_cfgr   dictpopget_optimizerr   
parametersrw   )r   r   optimizer_args	optimizerr!   r!   r"   setup_optimization8  s   
zDreamFusion.setup_optimizationc                 C   s   |    | jS rE   )r   rw   )r   r!   r!   r"   configure_optimizersC  s   z DreamFusion.configure_optimizers)__name__
__module____qualname__r	   r   r   r   r   r   r   r   r   r   __classcell__r!   r!   r   r"   r      s     	r   )r   r^   r   r   r   r   rb   5nemo.collections.multimodal.models.nerf.txt2nerf_baser   Cnemo.collections.multimodal.modules.nerf.loss.laplacian_smooth_lossr   Enemo.collections.multimodal.modules.nerf.loss.normal_consistency_lossr   Anemo.collections.multimodal.modules.nerf.materials.materials_baser   	nemo.corer   r   r!   r!   r!   r"   <module>   s   