o X۷i@sddlmZddlZddlZddlZddlZ ddl m Z ddl m ZddlmZddlmZddlmZddlmZdd lmZddlZGd d d Zd d ZddZddZiddddddddddddddddddddd d!dfd"d#d d!dfd$dd%dd&d'd(d'd)d'd*d'd'ddd+d+d'd'd'd,d'd-d-d.d/d0Z idd1dej!dfdd1dd1dd1dd1dd1dd1dd1dd2d"d2d$d1d%d1d&d1d(d1d)d1d*d1d1d1d1ej!dfej!dfd1d1d1d1d1d2d2ej!dfd3d0Z"d4d5Z#e$d6d7Z%dd:d;Z&dd>dZ'dd?d@Z(ddBdZ)ddCdZ*ddFdZ+ddGdZ,ddHdZ-ddIdZ.ddJdZ/ddKd$Z0ddLd%Z1dMd&Z2dNd(Z3ddPd)Z4dQd*Z5dRdSZ6dTdUZ7dVdWZ8dXdYZ9dZd[Z:d\d]Z;d^d_ZdddeZ?dfdgZ@GdhdidiZAGdjdkdkeAZBGdldmdmeAZCGdndodoeAZDGdpdqdqeAZEddrdZFddsd"ZG = =ddtduZHddvdwZIddydzZJdd{d|ZKdd}d~ZLdddZMdS)) annotationsN)driver)runtime)cusparse)_dtype)device)stream)_utilc@sVeZdZddZeddZddZejfddZ d d Z d d Z d dZ ddZ dS) MatDescriptorcCs ||_dSN) descriptor)selfr rD/home/ubuntu/vllm_env/lib/python3.10/site-packages/cupyx/cusparse.py__init__ zMatDescriptor.__init__cCst}t|Sr ) _cusparsecreateMatDescrr )clsdescrrrrcreateszMatDescriptor.createcCs |jdfS)Nr)r)r rrr __reduce__rzMatDescriptor.__reduce__cCs*|rdS|jrt|jd|_dSdSr )r rdestroyMatDescrr is_shutting_downrrr__del__ s   zMatDescriptor.__del__cCt|j|dSr )r setMatTyper )r typrrr set_mat_type'zMatDescriptor.set_mat_typecCrr )rsetMatIndexBaser )r baserrrset_mat_index_base*r z MatDescriptor.set_mat_index_basecCrr )rsetMatFillModer )r fill_moderrrset_mat_fill_mode-r zMatDescriptor.set_mat_fill_modecCrr )rsetMatDiagTyper )r diag_typerrrset_mat_diag_type0r zMatDescriptor.set_mat_diag_typeN)__name__ __module__ __qualname__r classmethodrrr rrrr#r&r)rrrrr s  r cs.dd|D}ttj|fdd|DS)NcSsg|] }|dur|jqSr dtype.0xrrr 5sz%_cast_common_type..cs,g|]}|dur|jkr|n|qSr )r/astyper0r.rrr37s$) _functoolsreduce_numpy promote_types)xsdtypesrr.r_cast_common_type4s  r;cCs|rtjStjSr )rCUSPARSE_OPERATION_TRANSPOSE CUSPARSE_OPERATION_NON_TRANSPOSE)transrrr_transpose_flag;sr?cGsR|dkrd}n|dkrd}n|dkrd}n |dkrd}nttt||}||S)NfsdFcDz) TypeErrorgetattrr)namer/argsprefixr@rrr_call_cusparseBsrLcsrmv)@*csrmvExcsrmmcsrmm2csrgeamcsrgeam2)<#Ncsrgemmcsrgemm2)rN.gthrspmv'rO)LinuxWindowsspmmi=( csr2dense csc2densecsrsort)rNNcscsortcoosortcoo2csr)r[N)rUrX)i$,N)i\+N)iP-N)csr2coocsr2csccsc2csr csr2cscEx2 csc2csrEx2 dense2csc dense2csrcsr2csr_compresscsrsm2csrilu02 denseToSparse sparseToDensespgemmspsm)i1N)N)iNcCs8t|trt}||vrd|}t|||S|S)Nz/No version information specified for the OS: {}) isinstancedict _platformsystemformat ValueError)r2os_namemsgrrr_get_avail_version_from_specs r|cCstjs t}t}nt}t}||vrd|}t|||\}}t |}t |}|dur5||kr5dS|dur?||kr?dSdS)Nz1No available version information specified for {}FT) _runtimeis_hip_available_cusparse_versionrget_build_version_available_hipsparse_version_driverrxryr|)rIavailable_versionversionr{ version_addedversion_removedrrrcheck_availabilitys    rreturnintcCsttSr )r getVersion_deviceget_cusparse_handlerrrrrsrFc Cs tdstd|dus|jjsJ|s|jn|jddd}|dt|kr+tdt}|\}} t |||\}}}|j } |durIt || }t || j}t || j}td| |t||jd|jd|j|j|jj|jjj|jjj|jjj|jj|j|jj|S)a\Matrix-vector product for a CSR-matrix and a dense vector. .. math:: y = \alpha * o_a(A) x + \beta y, where :math:`o_a` is a transpose function when ``transa`` is ``True`` and is an identity function otherwise. Args: a (cupyx.cusparse.csr_matrix): Matrix A. x (cupy.ndarray): Vector x. y (cupy.ndarray or None): Vector y. It must be F-contiguous. alpha (float): Coefficient for x. beta (float): Coefficient for y. transa (bool): If ``True``, transpose of ``A`` is used. Returns: cupy.ndarray: Calculated ``y``. rMzcsrmv is not available.Nrdimension mismatchr)r RuntimeErrorflags f_contiguousshapelenryrrr;r/_cupyzerosr7arrayctypesrLr?nnzdata_descrr ptrindptrindices) ar2yalphabetatransaa_shapehandlemnr/rrrrMs, cCsv|jjjddkr dS|jjjddkrdS|jjjddkr!dS|jjddkr+dS|dur9|jjddkr9dSdS)aCheck if the pointers of arguments for csrmvEx are aligned or not Args: a (cupyx.cusparse.csr_matrix): Matrix A. x (cupy.ndarray): Vector x. y (cupy.ndarray or None): Vector y. Check if a, x, y pointers are aligned by 128 bytes as required by csrmvEx. Returns: bool: ``True`` if all pointers are aligned. ``False`` if otherwise. rFNT)rrrr)rr2rrrrcsrmvExIsAlignedsrTcCstdstd|dus|jjsJ|jdt|krtdt}|j\}}t |||\}}}|j } |dur>t || }t | } |rHtjntj} tj} t|| j}t|| j}t|||sdJt|| | |jd|jd|j|j| |jj|jjj| |jjj|jjj|jj| |j| |jj| | } t | d}|jjddksJt || | |jd|jd|j|j| |jj|jjj| |jjj|jjj|jj| |j| |jj| | |jj|S) aMatrix-vector product for a CSR-matrix and a dense vector. .. math:: y = \alpha * A x + \beta y, Args: a (cupyx.cusparse.csr_matrix): Matrix A. x (cupy.ndarray): Vector x. y (cupy.ndarray or None): Vector y. It must be F-contiguous. alpha (float): Coefficient for x. beta (float): Coefficient for y. merge_path (bool): If ``True``, merge path algorithm is used. All pointers must be aligned with 128 bytes. Returns: cupy.ndarray: Calculated ``y``. rPzcsrmvEx is not available.Nrrrbr)!rrrrrrryrrr;r/rrr to_cuda_dtyperCUSPARSE_ALG_MERGE_PATHCUSPARSE_ALG_NAIVEr=r7rrrcsrmvEx_bufferSizerrrr rrremptyrP)rr2rrr merge_pathrrrr/datatypealgmode transa_flag bufferSizebufrrrrPsN     c CsXtdstd|j|jkrdksJJ|jjsJ|dus(|jjs(J|s-|jn|jddd}|d|jdkrBtdt}|\}} |jd} t |||\}}}|durft || f|j d }| } |} t ||j j}t ||j j}td|j |t||jd| |jd|j|j|jj|jjj|jjj|jjj|jj| |j|jj| |S) a|Matrix-matrix product for a CSR-matrix and a dense matrix. .. math:: C = \alpha o_a(A) B + \beta C, where :math:`o_a` is a transpose function when ``transa`` is ``True`` and is an identity function otherwise. Args: a (cupyx.scipy.sparse.csr): Sparse matrix A. b (cupy.ndarray): Dense matrix B. It must be F-contiguous. c (cupy.ndarray or None): Dense matrix C. It must be F-contiguous. alpha (float): Coefficient for AB. beta (float): Coefficient for C. transa (bool): If ``True``, transpose of A is used. Returns: cupy.ndarray: Calculated C. rQzcsrmm is not available.NrrrrrC)rrndimrrrryrrr;rrr/r7rrrLr?rrrr rrr) rrrDrrrrrrkrldbldcrrrrQVs6   ?cCstdstd|j|jkrdksJJ|jsJ|jjs#J|dus-|jjs-J|r3|r3J|s8|jn|jddd}|sE|jn|jddd}|d|dkrYtdt } |\} } |d} t |||\}}}|dur|t | | f|j d }|jd} |jd}t|}t|}t||j j}t||j j}td|j | |||jd| |jd|j|j|jj|jjj|jjj|jjj|jj| |j|jj||S) aVMatrix-matrix product for a CSR-matrix and a dense matrix. .. math:: C = \alpha o_a(A) o_b(B) + \beta C, where :math:`o_a` and :math:`o_b` are transpose functions when ``transa`` and ``tranb`` are ``True`` respectively. And they are identity functions otherwise. It is forbidden that both ``transa`` and ``transb`` are ``True`` in cuSPARSE specification. Args: a (cupyx.scipy.sparse.csr): Sparse matrix A. b (cupy.ndarray): Dense matrix B. It must be F-contiguous. c (cupy.ndarray or None): Dense matrix C. It must be F-contiguous. alpha (float): Coefficient for AB. beta (float): Coefficient for C. transa (bool): If ``True``, transpose of A is used. transb (bool): If ``True``, transpose of B is used. Returns: cupy.ndarray: Calculated C. rRzcsrmm2 is not available.rNrrrrrC)rrrhas_canonical_formatrrrryrrr;rrr/r?r7rrrLrrrr rrr)rrrDrrrtransbrb_shaperrrrrrop_aop_brrrrRs>      c Cstdstdt|tjjjstdt |t|tjjjs*tdt ||j s/J|j s4J|j |j kr>t dt }|j \}}t||\}}tdd}t|tjt}t|dd} t||||jj|j|jjj|jjj|jj|j|jjj|jjj|j| jj|j jtt!|d} tt!||j"} t#||j"j }t#||j"j }t$d|j"||||j|jj|j|jjj|jjj|jjj|j|jj|j|jjj|jjj|jjj|j| jj| jj| jjtjjj| | | f|j d} d | _%| S) eMatrix-matrix addition. .. math:: C = \alpha A + \beta B Args: a (cupyx.scipy.sparse.csr_matrix): Sparse matrix A. b (cupyx.scipy.sparse.csr_matrix): Sparse matrix B. alpha (float): Coefficient for A. beta (float): Coefficient for B. Returns: cupyx.scipy.sparse.csr_matrix: Result matrix. rSzcsrgeam is not available.unsupported type (actual: {})inconsistent shapesrirrT)&rrrtcupyxscipysparse csr_matrixrGrxtyperrryrrr;r7rrsetPointerModeCUSPARSE_POINTER_MODE_HOSTr rr xcsrgeamNnzrr rrrrrrrr/rrL_has_canonical_format) rrrrrrrrc_descrc_indptr c_indicesc_datarDrrrrSsX       cCstdstdt|tjjjstdt |t|tjjjs*tdt ||j s/J|j s4J|j |j kr>t dt }|j \}}t||\}}tdd}t|tjt||jj}t||jj}t}t|dd} d} td |j||||j|jj|j|jjj |j!jj |j"jj |j|jj|j|jjj |j!jj |j"jj |j| | jj | } t| tj#} t$||||jj|j|j!jj |j"jj |jj|j|j!jj |j"jj |j| jj |jj| jj tt%|d} tt%||j}td|j||||j|jj|j|jjj |j!jj |j"jj |j|jj|j|jjj |j!jj |j"jj |j|jj | jj | jj | jj tjjj|| | f|j d }d |_&|S) rrTzcsrgeam2 is not available.rrrrrrcsrgeam2_bufferSizeExtrT)'rrrtrrrrrGrxrrrryrrr;r7rrrrrr/rr rrrLrrr rrrrint8 xcsrgeam2Nnzrr)rrrrrrrrrrnull_ptr buff_sizebuffrrrDrrrrT sf        cCs6tdstd|j|jkrdksJJ|jsJ|js"J|s'|jn|jddd}|s4|jn|jddd}|d|dkrHtdt}|\}}|d} t||\}}|j dkse|j dkrqt j j j || f|jd St|} t|} td d } t|tjt} t|dd }t|| | || ||jj|j |jjj|jjj|jj|j |jjj|jjj| j|jj| jjtt | d }tt | |j}t!d|j|| | || ||jj|j |jjj|jjj|jjj|jj|j |jjj|jjj|jjj| j|jj|jj|jjt j j j |||f|| fd }d |_"|S)aMatrix-matrix product for CSR-matrix. math:: C = op(A) op(B), Args: a (cupyx.scipy.sparse.csr_matrix): Sparse matrix A. b (cupyx.scipy.sparse.csr_matrix): Sparse matrix B. transa (bool): If ``True``, transpose of A is used. transb (bool): If ``True``, transpose of B is used. Returns: cupyx.scipy.sparse.csr_matrix: Calculated C. rVzcsrgemm is not available.rNrrrrr.rrrT)#rrrrrryrrr;rrrrrr/r?r7rrrrr rr xcsrgemmNnzrr rrrrrrrLr)rrrrrrrrrrrrrrrrrrDrrrrVRsZ      cCstdstd|j|jkrdksJJt|tjjjs)td t |t|tjjjs:td t ||j s?J|j sDJ|j d|j dkrRt d|dur|jdks]Jt|tjjjsntd t ||j ssJ|j d|j dks|j d|j dkrt dtjrtd krtd t}|j \}}|j \}} |durt||\}}n t|||\}}}t} t||jj}d} |dur| } t} d}| }| }| }n t||jj}|j} |j} |j}|jjj }|j!jj }|j"jj }t#d |j||| ||j|jj$|j|j!jj |j"jj |jj$|j|j!jj |j"jj | | j$|||| }t%&|tj'}t&d d }t(|tj)t}t%&|dd }t*||| ||jj$|j|j!jj |j"jj |jj$|j|j!jj |j"jj | j$||||j$|jj |jj| |jj t%&t+|d }t%&t+||j}t#d|j||| ||j|jj$|j|jjj |j!jj |j"jj |jj$|j|jjj |j!jj |j"jj | | j$|||||j$|jj |jj |jj | |jj tjjj|||f|| fd}d|_,t-| |S)aMatrix-matrix product for CSR-matrix. math:: C = alpha * A * B + beta * D Args: a (cupyx.scipy.sparse.csr_matrix): Sparse matrix A. b (cupyx.scipy.sparse.csr_matrix): Sparse matrix B. d (cupyx.scipy.sparse.csr_matrix or None): Sparse matrix D. alpha (scalar): Coefficient beta (scalar): Coefficient Returns: cupyx.scipy.sparse.csr_matrix rWzcsrgemm2 is not available.rrrrmismatched shapeNrsz'd != None is supported since ROCm 4.2.0csrgemm2_bufferSizeExtrrrT).rrrrtrrrrrGrxrrrryr}r~rrrrr;rcreateCsrgemm2Infor7rr/rr rrrrrrrrLr rrrrr xcsrgemm2NnzrrdestroyCsrgemm2Info)rrrBrrrrr_rinfor beta_datad_descrd_nnzd_datad_indptr d_indicesrrc_nnzrrrrrDrrrrWs    (        c Ctdstd|j}|jdvsJ|dur tj|j|dd}n|jjs&Jt }t d|j||jd|jd|j j |jjj|jjj|jjj|jj|jd |S) a!Converts CSR-matrix to a dense matrix. Args: x (cupyx.scipy.sparse.csr_matrix): A sparse matrix to convert. out (cupy.ndarray or None): A dense matrix to store the result. It must be F-contiguous. Returns: cupy.ndarray: Converted result. r_zcsr2dense is not available.fdFDNrCr/orderrr)rrr/charrrrrrrrrLrr rrrrr2outr/rrrrr_  c Cr) a!Converts CSC-matrix to a dense matrix. Args: x (cupyx.scipy.sparse.csc_matrix): A sparse matrix to convert. out (cupy.ndarray or None): A dense matrix to store the result. It must be F-contiguous. Returns: cupy.ndarray: Converted result. r`zcsc2dense is not available.rNrCrrr)rrr/rrrrrrrrrLrr rrrrrrrrr` rc Ctdstd|j}|dkrdSt}|j\}}t|||||jj j |j j j }t |d}t |d}|j }t|||j j t|||||jj|jj j |j j j |j j |j j tdrwtd|j|||j j |j j j |j j tjdSt||j } t|} t|| j| jdS)zzSorts indices of CSR-matrix in place. Args: x (cupyx.scipy.sparse.csr_matrix): A sparse matrix to sort. razcsrsort is not available.rNrrrY)rrrrrrrxcsrsort_bufferSizeExtrrrrrrcopycreateIdentityPermutationxcsrsortrr rLr/CUSPARSE_INDEX_BASE_ZEROSpVecDescriptorrDnVecDescriptorgatherdesc r2rrrr buffer_sizerP data_origdesc_xdesc_yrrrra@8      c Cr)zzSorts indices of CSC-matrix in place. Args: x (cupyx.scipy.sparse.csc_matrix): A sparse matrix to sort. rbzcscsort is not available.rNrrrY)rrrrrrrxcscsort_bufferSizeExtrrrrrrrrxcscsortrr rLr/rrrrrrrrrrrbfrrc Csztdstd|j}|dkrdSt}|j\}}t|||||jj j |j j j }t |d}t |d}|j } t|||j j |dkr_t|||||jj j |j j j |j j |j j n|dkrzt|||||jj j |j j j |j j |j j ntd |jjd krtd rtd |j||| j j |j j j |j j tjnt||j } t| } t|| j| j|dkrd |_dSdS) zSorts indices of COO-matrix in place. Args: x (cupyx.scipy.sparse.coo_matrix): A sparse matrix to sort. sort_by (str): Sort the indices by row ('r', default) or column ('c'). rczcoosort is not available.rNrrrrDz!sort_by must be either 'r' or 'c'?rYF)rrrrrrrxcoosort_bufferSizeExtrowrrcolrrrr xcoosortByRowxcoosortByColumnryr/rrLrrrrrrr) r2sort_byrrrrrrrrrrrrrrcsL          cCst}|jd}|j}|dkrt|dd}nt|dd}t||j j j |||j j tj t jjj|j |j|f|jdS)Nrrrr)rrrrrrrrxcoo2csrrrrrrrrrr)r2rrrrrrrrd  cCst}|jd}|j}|dkrt|dd}nt|dd}t||j j j |||j j tj t jjj|j |j|f|jdS)Nrrrr)rrrrrrrrr rrrrrrr csc_matrixr)r2rrrrrrrcoo2cscr rcCsptdstdt}|jd}|j}t|d}t ||j j j |||j j tj tjjj|||ff|jdS)aDConverts a CSR-matrix to COO format. Args: x (cupyx.scipy.sparse.csr_matrix): A matrix to be converted. data (cupy.ndarray): A data array for converted data. indices (cupy.ndarray): An index array for converted data. Returns: cupyx.scipy.sparse.coo_matrix: A converted matrix. rezcsr2coo is not available.rrr)rrrrrrrrrxcsr2coorrrrrrr coo_matrix)r2rrrrrrrrrres   recCstdstdt}|j\}}|j}t||j}t|d}|dkr.t |dd}n+t|dd}t d|j|||||j j j |j j j |jj j |j j |j j |j j tjtjtjjj|||f|jdS)Nrfcsr2csc is not available.rrrr)rrrrrrrrr/rrLrrrrrCUSPARSE_ACTION_NUMERICrrrrrr2rrrrrrrrrrrf*   rfcCs<tdstdt}|j\}}|j}t||j}t|d}|dkr.t |dd}nct|dd}t |j}t j } t j} t j} t |||||jjj|jjj|jjj|jj|jj|jj|| | | } t| tj} t |||||jjj|jjj|jjj|jj|jj|jj|| | | | jjtjjj|||f|jdS)Nrhzcsr2cscEx2 is not available.rrrr)rrrrrrrrr/rrrrrrCUSPARSE_CSR2CSC_ALG1csr2cscEx2_bufferSizerrrrr7rrhrrrrr2rrrrrrrx_dtypeactionibasealgorbufferrrrrh8    rhcCs`t}|jd}|j}t|d}t||jj j |||j j tj t j jj|||ff|jdS)aDConverts a CSC-matrix to COO format. Args: x (cupyx.scipy.sparse.csc_matrix): A matrix to be converted. data (cupy.ndarray): A data array for converted data. indices (cupy.ndarray): An index array for converted data. Returns: cupyx.scipy.sparse.coo_matrix: A converted matrix. rrr)rrrrrrrrrrrrrrrr)r2rrrrrrrrrcsc2coo,s  rcCstdstdt}|j\}}|j}t||j}t|d}|dkr.t |dd}n+t|dd}t d|j|||||j j j |j j j |jj j |j j |j j |j j tjtjtjjj|||f|jdS)Nrgrrrrrfr)rrrrrrrrr/rrLrrrrrrrrrrrrrrrrgDrrgcCs<tdstdt}|j\}}|j}t||j}t|d}|dkr.t |dd}nct|dd}t |j}t j } t j} t j} t |||||jjj|jjj|jjj|jj|jj|jj|| | | } t| tj} t |||||jjj|jjj|jjj|jj|jj|jj|| | | | jjtjjj|||f|jdS)Nrizcsc2csrEx2 is not available.rrrr)rrrrrrrrr/rrrrrrrrrrrrr7rrhrrrrrrrrri\rric Cstdstd|jdksJt|}tjddd}t}|j \}}t }t|d}t d|j |tj|||j|jj||jj|jj t|}t||j }t|dd}t|d} t d|j ||||j|jj||jj|jj| jj|jj tjjj|| |f|j d } d | _| S) zConverts a dense matrix in CSC format. Args: x (cupy.ndarray): A matrix to be converted. Returns: cupyx.scipy.sparse.csc_matrix: A converted matrix. rjzdense2csc is not available.rrrr.rrrT)rrrrasfortranarrayr7rrrrr rrLr/rCUSPARSE_DIRECTION_COLUMNr rrrrrrrrr) r2rrrrr nnz_per_colrrrcscrrrrjzs6      rjc Cs,tdstd|jdksJt|}tjddd}t}|j \}}t }t|d}t d|j |tj|||j|jj||jj|jj t|}tjrT|dkrTtd t||j }t|d d}t|d} t d|j ||||j|jj||jj|jj|jj| jj tjjj|| |f|j d } d | _| S) zConverts a dense matrix in CSR format. Args: x (cupy.ndarray): A matrix to be converted. Returns: cupyx.scipy.sparse.csr_matrix: A converted matrix. rkzdense2csr is not available.rrrr.rr@hipSPARSE currently cannot handle sparse matrices with null ptrsrrT)rrrrr r7rrrrr rrLr/rCUSPARSE_DIRECTION_ROWr rrrrr}r~ryrrrrr) r2rrrrr nnz_per_rowrrrcsrrrrrks<      rkc Cstdstd|jjdvsJt}|j\}}t|d}t d|j|||j j |j j j |jj j |j j | }t||j}t|dd}t|d} t d|j||||j j |j j j |jj j |jj j |j|j j |j j | j j |j j |tjjj|| |f|jdS)Nrlz"csr2csr_compress is not available.rr nnz_compressrr)rrr/rrrrrrrLrr rrrrrrrrrr) r2tolrrrr&rrrrrrrrls0      rlcCs.|dkrtjS|dkrtjS|dkrtjSt)Nuint16int32int64)rCUSPARSE_INDEX_16UCUSPARSE_INDEX_32ICUSPARSE_INDEX_64IrGr.rrr_dtype_to_IndexTypesr0c@s,eZdZdddZejfddZddZdS) BaseDescriptorNcCs||_||_||_dSr )rgetdestroy)r r r2 destroyerrrrrs zBaseDescriptor.__init__cCsB|rdS|jdurd|_dS|jdur||jd|_dSdSr )r3rrrrrrs     zBaseDescriptor.__del__cCs |jdurt||j|Str )r2rHrAttributeError)r rIrrr __getattr__ s zBaseDescriptor.__getattr__)NN)r*r+r,rr rrr6rrrrr1s  r1c@s eZdZeddZddZdS)SpMatDescriptorc Cs:tjj|s J|j\}}tj}t|j }|j dkr@t |||j |j jj|jjj|jjjt|j j t|jj || }tj}nT|j dkrdt|||j |jjj|jjj|jjjt|jj || }tj}n0|j dkrt|||j |j jj|jjj|jjjt|j j t|jj || }d}ntd |j tj}t|||S)Nr'coor#z3csr, csc and coo format are supported (actual: {}).)rrrissparserrrrrr/rx createCsrrrrrrr0csrGet createCoorrcooGet createCscry destroySpMatr7) rrrowscolsidx_base cuda_dtyperr2r3rrrrs<      zSpMatDescriptor.createcCst|j||dSr )rspMatSetAttributer)r attributerrrr set_attribute/szSpMatDescriptor.set_attributeN)r*r+r,r-rrFrrrrr7s  r7c@eZdZeddZdS)rc CsP|j}t|j}t|||jj|jjt|jtj |}tj }tj }t |||Sr ) sizerrr/r createSpVecrrr0rspVecGet destroySpVecr)ridxr2rrCrr2r3rrrr5s  zSpVecDescriptor.createNr*r+r,r-rrrrrr3rc@rG)rcCs8t|j}t|j|jj|}tj}tj }t |||Sr ) rrr/r createDnVecrHrrdnVecGet destroyDnVecr)rr2rCrr2r3rrrrDs  zDnVecDescriptor.createNrMrrrrrBrNrc@rG)DnMatDescriptorc Csf|jdksJ|jjs J|j\}}|}t|j}t||||j j |tj }tj }tj }t|||S)Nr)rrrrrrr/r createDnMatrrCUSPARSE_ORDER_COLdnMatGet destroyDnMatrR) rrr@rAldrCrr2r3rrrrOs    zDnMatDescriptor.createNrMrrrrrRMrNrRc Cstdstdt|tjjjr+|j}t|tjjjs&d t |}t ||}| }t|tjjjtjjj fsAt d t ||sF|j n|j ddd}|dt|krZtd|js_J|\} } t|||\}}}|durxt| |j}n t|| krtd|jd kr|d |St|} t|} t|} t}t|}t||jj}t||jj}t !|j}t"j#}t"$|||j%| j&| j&|j%| j&|| }t'|tj(}t")|||j%| j&| j&|j%| j&|||j%j* |S) aMultiplication of sparse matrix and dense vector. .. math:: y = \alpha * op(A) x + \beta * y Args: a (cupyx.scipy.sparse.csr_matrix, csc_matrix or coo_matrix): Sparse matrix A x (cupy.ndarray): Dense vector x y (cupy.ndarray or None): Dense vector y alpha (scalar): Coefficient beta (scalar): Coefficient transa (bool): If ``True``, op(A) = transpose of A. Returns: cupy.ndarray rZzspmv is not available."aT must be csr_matrix (actual: {})rNrrrr)+rrrtrrrrTrrxrrGrrrryrr;rrr/rfillr7rrrrr?r7rrrrrCUSPARSE_MV_ALG_DEFAULTspMV_bufferSizerrrrspMVr)rr2rrrraTr{rrrdesc_arrrrrCalgrrrrrrZ]sX           c Cs`tdstd|j|jkrdksJJ|jjsJ|dus(|jjs(Jt|tjjj rK|j }t|tjjj sFd t |}t||}| }t|tjjj tjjjfsatd t ||sf|jn|jddd} |ss|jn|jddd} | d| d krtd |jsJ| \} } | \} }t|||\}}}|durt| |f|jd }n|jd | ks|jd|krtd |jd kr|d |St|}t|}t|}t}t|}t|}t ||jj!}t ||jj!}t"#|j}t$j%}t$&||||j'|j(|j(|j'|j(|| }t)|tj*}t$+||||j'|j(|j(|j'|j(|||j'j, }|S) aMultiplication of sparse matrix and dense matrix. .. math:: C = \alpha * op(A) op(B) + \beta * C Args: a (cupyx.scipy.sparse.csr_matrix, csc_matrix or coo_matrix): Sparse matrix A b (cupy.ndarray): Dense matrix B c (cupy.ndarray or None): Dense matrix C alpha (scalar): Coefficient beta (scalar): Coefficient transa (bool): If ``True``, op(A) = transpose of A. transb (bool): If ``True``, op(B) = transpose of B. Returns: cupy.ndarray r^zspmm is not available.rNrXrrrrrrC)-rrrrrrtrrrrrYrrxrrGrrryrr;rrr/rrZr7rrRrrr?r7rrrrrCUSPARSE_MM_ALG_DEFAULTspMM_bufferSizerrrrspMMr)rrrDrrrrr^r{rrrrrrr_desc_bdesc_crrrrCr`rrrrrr^sf             cCs6tdstdtjj|stjj|stdt|t j s$td|j dvr-td|j d|j dkrC|j dksHtd td |j |j krRtd |d urZtj}n|d urbtj}ntd ||d urqtj} n|d urytj} ntd||d urd} n|d urd} ntd||d urtj} n|d urtj} ntd||j } | jdkrd} n| jdkrd} n| jdkrd} n| jdkrd} ntd| tt| d}tt| d}tt| d}|d us|dkrtj}n(|d us|dkrtj}n|dkr| jdvrtj}n tj}ntd |tjj|rF|tjkr1td!|j}tjj|s>Jd|}d|}|j d}|j dkrSdn|j d}|jrbtj}|}n|jrltj}|}ntd"t }t!j"|| d#}t#$}|%tj&|'tj(|)||*| t+}||| |||||j,|j-j.|j/|j.j.j0|j1j.j0|j2j.j0|j.j0||| }t j3|ft!j4d#}||| |||||j,|j-j.|j/|j.j.j0|j1j.j0|j2j.j0|j.j0||| |j.j0||| |||||j,|j-j.|j/|j.j.j0|j1j.j0|j2j.j0|j.j0||| |j.j0t567t8|d$S)%aVSolves a sparse triangular linear system op(a) * x = alpha * b. Args: a (cupyx.scipy.sparse.csr_matrix or cupyx.scipy.sparse.csc_matrix): Sparse matrix with dimension ``(M, M)``. b (cupy.ndarray): Dense vector or matrix with dimension ``(M)`` or ``(M, K)``. alpha (float or complex): Coefficient. lower (bool): True: ``a`` is lower triangle matrix. False: ``a`` is upper triangle matrix. unit_diag (bool): True: diagonal part of ``a`` has unit elements. False: diagonal part of ``a`` has non-unit elements. transa (bool or str): True, False, 'N', 'T' or 'H'. 'N' or False: op(a) == ``a``. 'T' or True: op(a) == ``a.T``. 'H': op(a) == ``a.conj().T``. blocking (bool): True: blocking algorithm is used. False: non-blocking algorithm is used. level_info (bool): True: solves it with level information. False: solves it without level information. Note: ``b`` will be overwritten. rmzcsrsm2 is not available.z"a must be CSR or CSC sparse matrixzb must be cupy.ndarray)rrb.ndim must be 1 or 2rrz invalid shapedtype mismatchTFUnknown lower (actual: {})Unknown unit_diag (actual: {})zUnknown blocking (actual: {})Unknown level_info (actual: {})r@rArBrCrDrErFInvalid dtype (actual: {})csrsm2_bufferSizeExtcsrsm2_analysis csrsm2_solveNrYHfdzUnknown transa (actual: {})z@If matrix is CSC format and complex dtype,transa must not be 'H''b must be F-contiguous or C-contiguous.r.N)9rrrrrisspmatrix_csrisspmatrix_cscryrtrndarrayrrr/rGrCUSPARSE_FILL_MODE_LOWERCUSPARSE_FILL_MODE_UPPERrxCUSPARSE_DIAG_TYPE_NON_UNITCUSPARSE_DIAG_TYPE_UNITCUSPARSE_SOLVE_POLICY_NO_LEVELCUSPARSE_SOLVE_POLICY_USE_LEVELrrHr=r<&CUSPARSE_OPERATION_CONJUGATE_TRANSPOSErY _f_contiguous _c_contiguousrrr7rr rrCUSPARSE_MATRIX_TYPE_GENERALr#rr&r)createCsrsm2Inforrrr rrrrr_streamget_current_stream synchronizedestroyCsrsm2Info)rrrlower unit_diagrblocking level_infor%r(rpolicyr/thelperanalysissolvernrhsrrra_descrws_sizewsrrrrms   $              rmc Cs`tdstdtjj|std|jd|jdkr%td |j|dur-t j }n|dur5t j }ntd ||j }|jd krGd }n|jd krOd }n|jd krWd}n|jdkr_d}ntd |tt |d}tt |d}tt |d}tt d}t} |jd} |j} t} | t j| t jt } || | | | j|jjj|jjj|jjj| }tj |ft!j"d}t!j dt!j#d}|| | | | j|jjj|jjj|jjj| ||jj z || | |j$jWnt%ytd |dw|| | | | j|jjj|jjj|jjj| ||jj z || | |j$jWdSt%y/td |dw)aComputes incomplete LU decomposition for a sparse square matrix. Args: a (cupyx.scipy.sparse.csr_matrix): Sparse matrix with dimension ``(M, M)``. level_info (bool): True: solves it with level information. False: solves it without level information. Note: ``a`` will be overwritten. This function does not support fill-in (only ILU(0) is supported) nor pivoting. rnzcsrilu02 is not available.za must be CSR sparse matrixrrzinvalid shape (a.shape: {})FTrjr@rArBrCrDrErFrkcsrilu02_bufferSizecsrilu02_analysisxcsrilu02_zeroPivotr.rza({0},{0}) is missingzu({0},{0}) is zeroN)&rrrrrrsrGrryrxrrzr{r/rrHrrrr rrrr#rcreateCsrilu02Infor rrrrrrr7rr+r Exception)rrrr/rrrrcheckrrrrrrrpositionrrrrn~sj          rnr'cCsltdstd|jdksJ|jjdvsJt|}t|}|dkr1t j j j |j |jd}n'|dkrAt j j j|j |jd}n|dkrQt j j j|j |jd}ntd |t|}tj}t}t||j|j|}t|tj}t||j|j||jjtjd d d} tjd d d} tjd d d} t |j| j!j| j!j| j!jt"| } t#j$r| d krt%d |dkr|j&} t| d } t| |j}t j j j || | f|j d}nH|dkr|j&} t| d } t| |j}t j j j|| | f|j d}n&|dkrt'| d }t'| d }t| |j}t j j j|||ff|j d}t|}t(||j|j||jjd|_)|S)a0Converts a dense matrix into a CSR, CSC or COO format. Args: x (cupy.ndarray): A matrix to be converted. format (str): Format of converted matrix. It must be either 'csr', 'csc' or 'coo'. Returns: cupyx.scipy.sparse.spmatrix: A converted sparse matrix. rozdenseToSparse is not available.rrr'r.r#r8zunsupported format (actual: {})rr,r$rrT)*rrrr/rrr rRrrrrrrrrrGrxr7r"CUSPARSE_DENSETOSPARSE_ALG_DEFAULTrrdenseToSparse_bufferSizerrrdenseToSparse_analysisrrr7r spMatGetSizerrr}r~ryrrdenseToSparse_convertr)r2rxrrrrrrr num_rows_tmp num_cols_tmprrrrrrrrrrost                roc Cstdstd|j}|jdvsJ|dur tj|j|dd}n |jjs&J|j|ks-Jt |}t |}t j }t}t ||j|j|}t|tj}tjr[|jdkr[tdt ||j|j||jj|S) a*Converts sparse matrix to a dense matrix. Args: x (cupyx.scipy.sparse.spmatrix): A sparse matrix to convert. out (cupy.ndarray or None): A dense matrix to store the result. It must be F-contiguous. Returns: cupy.ndarray: A converted dense matrix. rpzsparseToDense is not available.rNrCrrr$)rrr/rrrrrrr7rrRr"CUSPARSE_SPARSETODENSE_ALG_DEFAULTrrsparseToDense_bufferSizerrrr}r~rryrprr) r2rr/rdesc_outrrrrrrrrp s.        rpc Cstdstd|durd}n|durd}n |dvr"td|d tjj|r*n6tjj|rT|dkr;|j}d}n|dkrE|j}d}n |d krP| j}d}| }n tjj |r\ntd |j seJ|j d krsd}| d d }n |j dkr{d}ntd|jd|jd kr|jdkstdtd|j}|jdvrtd|||jkrtd|durtj}n|durtj}ntd||durtj} n|durtj} ntd||dkrtj} n|dkrtj} n |jdvrtj} ntj} |jrtj} n|jrtdkrtd|j}tj} ntd|j\} } | tjkr/|j\} }n|j\}} | |f}tj||jdd}t !}t"#|}t$#|}t$#|}t%}t&j'||jdj(}t)*|j}tj+}zb|,tj-||,tj.| t/|| | |j0|j1|j1|j1||| }tj2|tj3d}t4|| | |j0|j1|j1|j1||||j0j5 t6|| | |j0|j1|j1|j1||||j0j5 |r| d }|Wt7|St7|w)aSolves a sparse triangular linear system op(a) * x = alpha * op(b). Args: a (cupyx.scipy.sparse.csr_matrix or cupyx.scipy.sparse.coo_matrix): Sparse matrix with dimension ``(M, M)``. b (cupy.ndarray): Dense matrix with dimension ``(M, K)``. alpha (float or complex): Coefficient. lower (bool): True: ``a`` is lower triangle matrix. False: ``a`` is upper triangle matrix. unit_diag (bool): True: diagonal part of ``a`` has unit elements. False: diagonal part of ``a`` has non-unit elements. transa (bool or str): True, False, 'N', 'T' or 'H'. 'N' or False: op(a) == ``a``. 'T' or True: op(a) == ``a.T``. 'H': op(a) == ``a.conj().T``. rrzspsm is not available.FroTrYNTHzUnknown transa (actual: )rpz'a must be CSR, CSC or COO sparse matrixrrrrfrrrrkrgrhrirqi-zb must be F-contiguous.rrr@rr.)8rrryrrrrsrtrYconjisspmatrix_coorrreshaperr/rrGrxrrvrwrxryr=r<r|r}r~rrrrrr7rrRspSM_createDescrr7rrrrCUSPARSE_SPSM_ALG_DEFAULTrFCUSPARSE_SPMAT_FILL_MODECUSPARSE_SPMAT_DIAG_TYPEspSM_bufferSizerrrr spSM_analysisr spSM_solvespSM_destroyDescr)rrrrrr is_b_vectorr/r%r(rrrrrc_shaperDrmat_amat_bmat_c spsm_descrrCrrrrrrrr5s    $             rrcCstdstd|j|jkrdksJJt|tjjjs)td t |t|tjjjs:td t ||j s?J|j sDJ|j d|j dkrRt d|j \}}|j \}}t||\}}||f}tjjj||jd}t} t|} t|} t|} t} tj}tj}tj||jdj}tjd|jdj}t|j}tj}d}z7t| |||j| j| j|j| j||| d| }t !|t j"}t| |||j| j| j|j| j||| ||jj# WnStj$y5}zEt%d kr|tj&}t| |||j| j| j|j| j||| d| }t !|t j"}t| |||j| j| j|j| j||| ||jj# WYd }~nd }~wwt'| |||j| j| j|j| j||| d| }t !|t j"}t'| |||j| j| j|j| j||| ||jj# tjdd d}tjdd d}tjdd d}t(| j|jj|jj|jj|dt)|ksJ|dt)|ksJt)|}|j*}t !|d }t !||j}t+| j|jj#|jj#|jj#t,| |||j| j| j|j| j||| tjjj|||f|d }t-| |S)a+Matrix-matrix product for CSR-matrix. math:: C = alpha * A * B Args: a (cupyx.scipy.sparse.csr_matrix): Sparse matrix A. b (cupyx.scipy.sparse.csr_matrix): Sparse matrix B. alpha (scalar): Coefficient Returns: cupyx.scipy.sparse.csr_matrix rqzspgemm is not available.rrrrrr.rXNr,rr).rrrrtrrrrrGrxrrrryr;r/rrr7rrspGEMM_createDescrr=r7rrrrCUSPARSE_SPGEMM_DEFAULTspGEMM_workEstimationrrrrrr CuSparseErrorrCUSPARSE_SPGEMM_ALG2spGEMM_computerrrcsrSetPointers spGEMM_copyspGEMM_destroyDescr)rrrrrrrrrDrrrr spgemm_descrrrrrCrr buff1_sizebuff1cse buff2_sizebuff2 c_num_rows c_num_colsrrrrrrrrqs                rq)rr)NrrFr )NrrT)NrrFF)rr)FF)Nrr)r)NrrFF)rTFFTF)F)r')rTFFr)N __future__r functoolsr5numpyr7platformrvcupyrcupy_backends.cuda.apirrrr}cupy_backends.cuda.libsrr cupy._corer cupy.cudarrrrr cupyx.scipy.sparserr r;r?rLrinfrr|memoizerrrMrrPrQrRrSrTrVrWr_r`rarbrcrdrrerfrhrrgrirjrkrlr0r1r7rrrRrZr^rmrnrorprrrqrrrrsH        !  '      %    2  D 8 ? @ E E i  & &1*. $  CL   G H (#