o }oiD#@sddlZddlmZmZmZmZmZmZddlZ ddl Z ddl m m ZddlmZddlmZddlmZmZmZmZdededed efd d Zd d gddgddgdddddddddf dedeedeedeedededededed ed!ed"ed#e jd ee je je je jee jffd$d%Zded&e jd#e jd ee je je jffd'd(Z d-ded&e jde j de j d#e jd ee je je jff d)d*Z!Gd+d,d,eZ"dS).N)AnyDictIteratorListOptionalTuple)IterableDataset) PinholeCamera)compute_look_at_vectorsconstruct_posesget_raysget_view_directionx lower_bound upper_boundreturncCstdtd||||S)aD Linearly normalize a value between lower_bound and upper_bound to a value between 0 and 1. Parameters: x: The value to normalize. lower_bound: The lower bound of the range of x. upper_bound: The upper bound of the range of x. Returns: The normalized value between 0 and 1. r)minmaxrrrrf/home/ubuntu/.local/lib/python3.10/site-packages/nemo/collections/multimodal/data/nerf/random_poses.pylinear_normalization s rrg?xh<g?F皙?{Gz?cudasize radius_range theta_range phi_rangeangle_overhead angle_frontuniform_sphere_ratejitter jitter_center jitter_target jitter_up return_dirsdevicec Cs*t|}t|}t|}t|}tj|| d|d|d|d} t|kr8t|| | d\}}}n t|| ||| d\}}}t|}|r_|t|||d7}t || }t |||rg| nd| d\}}}t ||||| d}| rt ||||nd }t |t |}}|||| |fS) aa Generate random poses from an orbit camera. Args: size (int): Number of poses to generate. radius_range (List[float]): Min and max radii for camera [min, max]. theta_range (List[float]): Elevation angle range in degrees [min, max]. phi_range (List[float]): Azimuth angle range in degrees [min, max]. angle_overhead (float): Overhead angle in degrees. angle_front (float): Front angle in degrees. uniform_sphere_rate (float): The probability of sampling from a uniform sphere. jitter (bool): Whether to add noise to the poses. jitter_center (float): Noise range for the camera center. jitter_target (float): Noise range for the camera target. jitter_up (float): Noise range for the camera up vector. return_dirs (bool): Whether to return the view directions. device (torch.device): The device on which to allocate tensors. Returns: Tuple: Contains the following: - poses (torch.Tensor): Generated poses, shape [size, 4, 4]. - thetas (torch.Tensor): Elevation angles in degrees, shape [size]. - phis (torch.Tensor): Azimuth angles in degrees, shape [size]. - radius (torch.Tensor): Radii of the camera orbits, shape [size]. - dirs (torch.Tensor, optional): View directions, if requested. r,rr)r radiusr,)r r.r"r#r,g@)centersr*r,)r/ right_vector up_vectorforward_vectorr,N)npradianstorchrandrandomsample_uniform_sphere sample_orbit zeros_like rand_like randn_liker r r rad2deg)r r!r"r#r$r%r&r'r(r)r*r+r,r.r/thetasphistargetsr2r1r0posesdirsrrr rand_poses/s. +   &     rCr.c Cstjtjtj||dttj||dtj||dgddddd}||d}t|dddf}t|dddf|dddf}||dkdt j 7<|||fS) a Sample points uniformly on a sphere. Args: size (int): Number of points to sample. device (torch.device): Device to allocate tensors on. radius (torch.Tensor): Radii for the points. Returns: Tuple: Contains the following: - centers (torch.Tensor): The Cartesian coordinates of the sampled points. - thetas (torch.Tensor): Elevation angles in radians. - phis (torch.Tensor): Azimuth angles in radians. r-dimr)prFNr) F normalizer5stackrandnabs unsqueezeacosatan2r3pi)r r.r, unit_centersr/r>r?rrrr8s    $ r8c Cstj||d|d|d|d}tj||d|d|d|d}||dkdtj7<|t|t|}|t|}|t|t|} tj||| gdd} | ||fS)a Sample points on a spherical orbit. Args: size (int): Number of points to sample. radius (torch.Tensor): Radii for the points. theta_range (np.ndarray): Elevation angle range in radians [min, max]. phi_range (np.ndarray): Azimuth angle range in radians [min, max]. device (torch.device): Device to allocate tensors on. Returns: Tuple: Contains the following: - centers (torch.Tensor): The Cartesian coordinates of the sampled points. - thetas (torch.Tensor): Elevation angles in radians. - phis (torch.Tensor): Azimuth angles in radians. r-rrrGrDrE)r5r6r3rQsincosrK) r r.r"r#r,r>r?ryzr/rrrr9s&& r9c1sbeZdZdZdddddgddgdd gd d gd d d gd d ddddddd dddddfdedededeeefdeeefdeeefdeeefdedeeefd ed!ed"ed#ed$ed%ed&ed'ed(ed)ed*ed+ed,ed-ej d.d/f0fd0d1 Z d2ed3ed.d/fd4d5Z d3ed.d/fd6d7Z d.e eeejffd8d9Zd:d;ZdZZS)?RandomPosesDatasetz3 A dataset class to generate random poses. dg@g @gF@g@Z@gfgf@g$@g>@g4@g @gV@gFrrgN@g{Gz?g@@cpuinternal_batch_sizeheightwidthr!r"r# fovx_range default_fovx fovy_range default_fovydefault_radius default_polardefault_azimuthr'r(r)r*r$r%r&nearfarr,rNcst||_||_||_d|_d|_d|_||_| |_ | |_ | |_ | |_ d|_ ||_||_||_||_| |_||_||_||_||_| |_||_||_||_||_||_||_||_||_||_ ||_!t"|j|j|j|j |j!d|_#dS)a. Initializes a new RandomPosesDataset instance. Parameters: internal_batch_size (int): Number of samples to pre-generate internally. height (int): Height of the image. width (int): Width of the image. radius_range (Tuple[float, float]): Range of generated radii. theta_range (Tuple[float, float]): Range of generated theta angles. phi_range (Tuple[float, float]): Range of generated phi angles. fovx_range (Tuple[float, float]): Range of generated field of view in x-direction. default_fovx (float): Default field of view in x-direction. fovy_range (Tuple[float, float]): Range of generated field of view angles in y-direction. default_fovy (float): Default field of view in y-direction. default_radius (float): Default radius of the circle. default_polar (float): Default polar angle. default_azimuth (float): Default azimuth angle. jitter (bool): Whether to jitter the poses. jitter_center (float): Jittering center range. jitter_target (float): Jittering target range. jitter_up (float): Jittering up range. angle_overhead (float): Overhead angle. angle_front (float): Frontal angle. uniform_sphere_rate (float): Rate of sampling uniformly on a sphere. near (float): Near clipping distance. far (float): Far clipping distance. device (torch.device): Device to generate data on. FriT)r]r\rerfr,N)$super__init__r\r]r[progressive_viewprogressive_view_start_stepprogressive_view_end_stepr_rarbrcrdsame_fov_randomr!r"r#r^r`current_radius_rangecurrent_theta_rangecurrent_phi_rangecurrent_fovx_rangecurrent_fovy_ranger$r%r&r'r(r)r*rerfr,r camera)selfr[r\r]r!r"r#r^r_r`rarbrcrdr'r(r)r*r$r%r&rerfr, __class__rrrhsH 7 zRandomPosesDataset.__init__epoch global_stepcCs|jr |j|ddSdS)z Update the dataset at the beginning of each epoch. Parameters: epoch (int): Current epoch. global_step (int): Current global step. )rwN)riprogressive_view_update_step)rsrvrwrrr update_step8s zRandomPosesDataset.update_stepcCst||j|jd}d||j||jdd||j||jdg|_d||j||jdd||j||jdg|_d||j ||j dd||j ||j dg|_ d||j ||j dd||j ||j dg|_dS)z| progressively relaxing view range Parameters: global_step (int): Current global step. rrrN)rrjrkrdr#rorcr"rnrbr!rmrar`rq)rsrwrrrrrxDs  z/RandomPosesDataset.progressive_view_update_stepc cst |\}}}}}t|jD](}|j|j||d||d||d||d||ddVqq)z~ Returns an iterator over the dataset. Returns: Iterator: An iterator over the dataset. Tr)r\r]rays_orays_ddirmvpazimuth)generate_samplesranger[r\r]rN)rsr{r|rBr~ delta_azimuthirrr__iter__`s      zRandomPosesDataset.__iter__cCs*t|j|j|j|j|j|j|j|j|j |j |j d|j d \}}}}}|j r,t}|}nt}t}||jd|jd|jd}||jd|jd|jd} |jj|| d} |jj| d| dd} | t|} t|| |j|j|j d\} }||j}||dkd 8<| ||| |fS) a Generate a batch of random poses. Returns: Tuple[Dict[str, torch.Tensor], Dict[str, torch.Tensor], Dict[str, torch.Tensor], Dict[str, torch.Tensor], Dict[str, torch.Tensor]]: A tuple containing: - rays (Dict[str, torch.Tensor]): A dictionary containing the origin and direction of the rays. - dirs (torch.Tensor): A tensor containing the directions of the rays. - mvp (torch.Tensor): A tensor containing the model-view-projection matrix. - azimuth (torch.Tensor): A A tensor containing the azimuth angle. T) r r!r"r#r$r%r&r'r(r)r*r+r,rr)fovxfovy)focal_xfocal_y)rA intrinsicsr\r]r,r)rCr[rmrnror$r%r&r'r(r)r*r,rlr7rprqrrcompute_intrinsicscompute_projection_matrixr5inverser r\r]rd)rsrArBr>r?r. fovx_random fovy_randomrrr projectionr~r{r|rrrrrws> "" z#RandomPosesDataset.generate_samplesbatchc Cs||j|jtjdd|Dddtjdd|Dddtjdd|Dddtjdd|Dddtjdd|Dddd S) z Collate function to bundle multiple samples into a single batch. Args: batch (List[Dict]): List of samples to collate. Returns: Dict: A dictionary containing the collated batch. cSg|]}|dqS)r{r.0itemrrr z1RandomPosesDataset.collate_fn..rrEcSr)r|rrrrrrrcSr)r~rrrrrrrcSr)r}rrrrrrrcSr)rrrrrrrr)r\r]r{r|r~r}r)r\r]r5cat)rsrrrr collate_fns zRandomPosesDataset.collate_fn)__name__ __module__ __qualname____doc__intrfloatboolr5r,rhryrxrrstrTensorrrrrr __classcell__rrrtrrWs         f .9rW)r)#r7typingrrrrrrnumpyr3r5torch.nn.functionalnn functionalrItorch.utils.datar-nemo.collections.multimodal.data.nerf.camerasr +nemo.collections.multimodal.data.nerf.utilsr r r r rrrrr,rrCr8ndarrayr9rWrrrrs         V )