o X۷i=@sPddlmZddlZddlZddlZddlZddlmZddl m Z m Z m Z m Z mZmZmZdZdZdZed  d0ddZejd dd1ddZd2ddZd3ddZejd dddZd4ddZddZddZd5d d!Zejd dd6d#d$Zd%d&Zd5d'd(Z ejd dd)d*Z!ejd dd7d+d,Z" " "d8d"ddd-d.d/Z#dS)9) annotationsN) _get_inttype)_check_distances_check_indices_distance_tranform_arg_check_generate_indices_ops_generate_shape_get_block_sizelcmz_ #define MARKER {marker} #define MAX_INT {max_int} #define BLOCKSIZE {block_size_3d} as // Sites : ENCODE(x, y, z, 0, 0) // Not sites : ENCODE(0, 0, 0, 1, 0) or MARKER #define ENCODED_INT_TYPE int #define ZERO 0 #define ONE 1 #define ENCODE(x, y, z, a, b) (((x) << 20) | ((y) << 10) | (z) | ((a) << 31) | ((b) << 30)) #define DECODE(value, x, y, z) x = ((value) >> 20) & 0x3ff; y = ((value) >> 10) & 0x3ff; z = (value) & 0x3ff #define NOTSITE(value) (((value) >> 31) & 1) #define HASNEXT(value) (((value) >> 30) & 1) #define GET_X(value) (((value) >> 20) & 0x3ff) #define GET_Y(value) (((value) >> 10) & 0x3ff) #define GET_Z(value) ((NOTSITE((value))) ? MAX_INT : ((value) & 0x3ff)) a // Sites : ENCODE(x, y, z, 0, 0) // Not sites : ENCODE(0, 0, 0, 1, 0) or MARKER #define ENCODED_INT_TYPE long long #define ZERO 0L #define ONE 1L #define ENCODE(x, y, z, a, b) (((x) << 40) | ((y) << 20) | (z) | ((a) << 61) | ((b) << 60)) #define DECODE(value, x, y, z) x = ((value) >> 40) & 0xfffff; y = ((value) >> 20) & 0xfffff; z = (value) & 0xfffff #define NOTSITE(value) (((value) >> 61) & 1) #define HASNEXT(value) (((value) >> 60) & 1) #define GET_X(value) (((value) >> 40) & 0xfffff) #define GET_Y(value) (((value) >> 20) & 0xfffff) #define GET_Z(value) ((NOTSITE((value))) ? MAX_INT : ((value) & 0xfffff)) T cCstj|||d}|dkr|t7}n|t7}tjtjtd}t tj|d}d| }Wdn1s;wY||7}|S)N) block_size_3dmarkermax_intrcudazpba_kernels_3d.h ) pba3d_defines_templateformatpba3d_defines_encode_64bitpba3d_defines_encode_32bitospathjoindirname__file__open readlines)rrrsize_max pba3d_codekernel_directoryf pba3d_kernelsr%Q/home/ubuntu/vllm_env/lib/python3.10/site-packages/cupyx/scipy/ndimage/_pba_3d.py get_pba3d_srcFs r')for_each_devicecCsj|dkrd}nd}|dkrd}nd}d|d|d|d |d |d |d |d |d}tjdd|dS)z-Pack array coordinates into a single integer.r ptrdiff_tintz!(((x) << 40) | ((y) << 20) | (z))z!(((x) << 20) | ((y) << 10) | (z))z$ if (arr[i]) { out[i] = z; } else { z# shape_2 = arr.shape()[2]; z# shape_1 = arr.shape()[1]; z _i = i; z2 x = _i % shape_2; _i /= shape_2; z2 y = _i % shape_1; _i /= shape_1; z z = _i; out[i] = z ; } z raw B arrz raw I out in_params out_params operation)cupyElementwiseKernel)r rint_typevaluecoder%r%r&_get_encode3d_kernelWs8   r4cCsj|jdkr td|dvrtd|dkrtj}ntj}tj|j|dd}t||d}||||j d |S) Nzonly 3d arr supported)r @z'only bit_depth of 32 or 64 is supportedrC)dtypeorder)r)size) ndim ValueErrornpint64int32r/zerosshaper4r:)arrr bit_depthr r8imagekernr%r%r&encode3d{s  rFcCs@|dkrd|d|d|d}|Sd|d|d|d}|S) Nrz z( x = (encoded >> 40) & 0xfffff; z( y = (encoded >> 20) & 0xfffff; z z = encoded & 0xfffff; z& x = (encoded >> 20) & 0x3ff; z& y = (encoded >> 10) & 0x3ff; z z = encoded & 0x3ff; r%)r r1r3r%r%r&_get_decode3d_codes" rHcCt|dd}tjdd|dS)z1Unpack 3 coordinates encoded as a single integer.rGr1 E encodedz I x, I y, I zr+)rHr/r0)r r3r%r%r&_get_decode3d_kernels rLcCsV|dkrtjntj}tj||d}t|}t|}t|}||||||||fS)Nr8)r/r?r> empty_likerL)encodedr coord_dtypexyzrEr%r%r&decode3ds   rUcCst||||||}|\}} } d} | |dkr"d} |t| |} n| } | |dkr6d} |t| |} n| } ||dkrJd} |t||}n|}| | koU|kn }|s]| rvt|| | }d||fd|| fd|| ff}|Sd}|S)NFrT)r mathceilmax)rA block_sizem1m2m3blockxblockyLCMorig_szorig_syorig_sxround_upsxsyszanisosmax padding_widthr%r%r&_determine_paddings.     rjcCsd|d|dS) Compute euclidean distance from current coordinate (ind_0, ind_1, ind_2) to the coordinates of the nearest point (z, y, x).z z tmp = z - ind_0; z sq_dist = tmp * tmp; tmp = y - ind_1; sq_dist += tmp * tmp; tmp = x - ind_2; sq_dist += tmp * tmp; dist[i] = sqrt(static_cast(sq_dist)); r%)r1 dist_int_typer%r%r&_generate_distance_computations rmcCs2td|d|d}|td|d7}|t||7}|SNr5distr;r1var_nameraw_varr;r1)r rrm)r1rl raw_out_varr3r%r%r&_get_distance_kernel_codes ruFcCs*|rdnd}t||dd}tjdd|dS)zAReturns kernel computing the Euclidean distance from coordinates.r)r*Trtz I z, I y, I x raw F distr+)rur/r0)r1 large_distrlr.r%r%r&_get_distance_kernels rycCsdS)rka F tmp = static_cast(z - ind_0) * sampling[0]; F sq_dist = tmp * tmp; tmp = static_cast(y - ind_1) * sampling[1]; sq_dist += tmp * tmp; tmp = static_cast(x - ind_2) * sampling[2]; sq_dist += tmp * tmp; dist[i] = sqrt(static_cast(sq_dist)); r%r%r%r%r&$_generate_aniso_distance_computationsrzcCs.td|d|d}|td|d7}|t7}|Srn)r rrz)r1rtr3r%r%r&_get_aniso_distance_kernel_code s  r{cCrI)z\Returns kernel computing the Euclidean distance from coordinates with axis spacing != 1.TrvzI z, I y, I x, raw F samplingrwr+)r{r/r0)r1r.r%r%r&_get_aniso_distance_kernelsr|cCs|dur |rdnd}d}t||d}|td|ddd7}|td|d 7}|dur2|t||7}d }n|t7}d }tj|d |d S)amFused decode3d and distance computation. This kernel is for use when `return_distances=True`, but `return_indices=False`. It replaces the separate calls to `_get_decode3d_kernel` and `_get_distance_kernel`, avoiding the overhead of generating full arrays containing the coordinates since the coordinate arrays are not going to be returned. Nr)r*rJr5roTrprsrKzE encoded, raw F samplingrwr+)rHr rrmrzr/r0)r rxsamplingrlr1r3r,r%r%r&_get_decode_as_distance_kernel#s$   r~)float64_distances distancesindicesc,Cst|tj} t|tj} t| | |||jdkr!td|jd|dur,d} d} d} n|\} } } t|j}|dkr=d}n|dkrDd}n |dkrKd}nd }d}t|}|j\}}}t |j|| | | ||}|durqtj ||d dd }|jd }t |j}t ||d }d }t |}||g}||df}||d ||ddf}tjt||dd}|d}|dur|d} |d}!d}"n|d} |d}!ttt|}|d|d|d f}"||||||d||fd|}||df}||d ||ddf}| |||||d||f|"|| df}||d |df}|!|||d||||f|"|dur?|d|d|d f}"||df}||d ||ddf}| |||||d||f|"|| df}||d |df}|!|||d||||f|"||}|r|||f}#|rtjntj}$| rt||#|$ntj|#|$d}tdd|#D}%|%dk}&|st||&|d}'|dur|'|d|d|d|f||fStj||jd}|'|d|d|d|f|||fS|r t|d|d|d|f|d \}(})}*d}+|r?|dur#tt||&d}'|'|*|)|(|ntt|d}'tj||jd}|'|*|)|(|||+|f}+|rn| r_t ||jf|j|(jj!|*|d<|)|d<|(|d<n tj"|*|)|(fd d }|+|f}+|+S)!Nr5zexpected a 3D array, got Drr constant)modeconstant_valuesr)r )rr )r3 kernelFloodZkernelMaurerAxiskernelColorAxisr%kernelMaurerAxisWithSpacingkernelColorAxisWithSpacingrNcss|] }|ddVqdS)rrNr%).0sr%r%r& sz_pba_3d..rM)r rxr})r1rxrJ)r.)r.)r.)axis)# isinstancer/ndarrayrr;r<minrAr rjpadrXrF zeros_like RawModuler' get_functiontuplemapfloatfloat64float32rr@sumr~asarrayr8rUryrr|ritemsizestack),rBr}return_distancesreturn_indices block_paramscheck_warp_sizerrrindices_inplace dt_inplacerZr[r\s_minr]r^rYr`rarbrir:r input_arr buffer_idxoutput pba_imagesblockgridpba3drrr sampling_args out_shape dtype_outmax_possible_distrxrErRrSrTvalsr%r%r&_pba_3dFs                         "     r)r r rr)r )r r r)rG)r)T)F)FN)NTFNF)$ __future__rrVrr/numpyr=_utilr_pba_2drrrrr r r rrrmemoizer'r4rFrHrLrUrjrmruryrzr{r|r~rr%r%r%r&sJ  $    #     $     "