o
    toitS                     @  s6  d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlmZ d dlmZ d dlmZmZ d dlmZ d dlZd dlZd dlZd dlmZmZmZmZmZmZ d dlmZ d d	lm Z  erd d
l!m"Z"m#Z#m$Z$ d dl%m&Z& d dl'm(Z( G dd de
j)Z*dddZ+G dd dZ,dS )    )annotationsN)defaultdict)Pattern)TYPE_CHECKINGAny)urlretrieve)ONNX_ML
ModelProto	NodeProto	TypeProtoValueInfoProtonumpy_helper)load_model_tests)TestItem)CallableIterableSequence)Backend)TestCasec                   @  s   e Zd ZdS )!BackendIsNotSupposedToImplementItN)__name__
__module____qualname__ r   r   U/home/ubuntu/.local/lib/python3.10/site-packages/onnx/backend/test/runner/__init__.pyr   $   s    r   timesintreturn2Callable[[Callable[..., Any]], Callable[..., Any]]c                   s    dksJ d fdd}|S )N   funcCallable[..., Any]r   c                   s   t  d fdd}|S )Nargsr   kwargsr   c               	     sd   t dd D ](}z
 | i |W   S  ty/   t| d |kr& td|  Y qw d S )Nr   z times tried   )range	Exceptionprinttimesleep)r"   r#   i)r    r   r   r   wrapped,   s   z/retry_execute.<locals>.wrapper.<locals>.wrapped)r"   r   r#   r   r   r   )	functoolswraps)r    r+   r   )r    r   wrapper+   s   zretry_execute.<locals>.wrapper)r    r!   r   r!   r   )r   r/   r   r.   r   retry_execute(   s   r0   c                   @  s   e Zd Z		dYdZd
dZd[ddZd\ddZd\ddZd\ddZd]ddZe	d^ddZ
e	d_ddZe	d`d!d"Ze	dad#d$Ze	dbdcd,d-Zeed.ddd2d3Zeded4d5Z	6dfdgdAdBZe	EdhdidMdNZdjdPdQZdkdWdXZdS )lRunnerNbackendtype[Backend]parent_module
str | Nonetest_kwargsdict | Noner   Nonec                 C  s   || _ || _t | _t | _t | _|pi | _tt| _	t
ddD ]}| |d q!t
ddD ]}| |d q/t
ddD ]}| |d q=t
ddD ]}| |d	 qKt
d
dD ]}| |d qYd S )Nnode)kindNoderealRealsimpleSimplezpytorch-convertedPyTorchConvertedzpytorch-operatorPyTorchOperator)r2   _parent_moduleset_include_patterns_exclude_patterns_xfail_patterns_test_kwargsr   dict_test_itemsr   _add_model_test)selfr2   r4   r6   rtctotr   r   r   __init__>   s$   

zRunner.__init__namestrtype[unittest.TestCase]c                 C  s&   t t|tjfi }| jr| j|_|S N)typerQ   unittestr   rB   r   )rK   rP   	test_caser   r   r   _get_test_case`   s   zRunner._get_test_casepatternc                 C     | j t| | S rS   )rD   addrecompilerK   rX   r   r   r   includef      zRunner.includec                 C  rY   rS   )rE   rZ   r[   r\   r]   r   r   r   excludej   r_   zRunner.excludec                 C  rY   rS   )rF   rZ   r[   r\   r]   r   r   r   xfailn   r_   zRunner.xfailc                 C  sF   dd l }| j D ]\}}| D ]}|j|j||j|_qq	| S )Nr   )pytestrI   itemsvaluesmarkonnx_coverageprotor    )rK   rb   category	items_mapitemr   r   r   enable_reportr   s   zRunner.enable_reportdict[str, dict[str, TestItem]]c                   s   i }| j  D ]Z\}}i ||< | D ]M\ }| jr/t fdd| jD s/td|j|_| jD ]}| rGtd|j	 d|j|_q2| j
D ]}| rYt|j|_qK|||  < qq|S )Nc                 3  s    | ]}|  V  qd S rS   )search).0r^   rP   r   r   	<genexpr>   s    z.Runner._filtered_test_items.<locals>.<genexpr>zno matched include patternzmatched exclude pattern "")rI   rc   rD   anyrU   skipr    rE   rm   rX   rF   expectedFailure)rK   filteredrh   ri   rj   r`   ra   r   ro   r   _filtered_test_itemsz   s.   



zRunner._filtered_test_items"dict[str, type[unittest.TestCase]]c                 C  s^   i }| j  D ]%\}}d| d}| |}t| D ]\}}t|||j q|||< q|S )zList of test cases to be applied on the parent scope
        Example usage:
            globals().update(BackendTest(backend).test_cases)
        OnnxBackendTest)rv   rc   rW   sortedsetattrr    )rK   
test_casesrh   ri   test_case_namerV   rP   rj   r   r   r   r|      s   

zRunner.test_casesunittest.TestSuitec                 C  s:   t  }t| j dd dD ]}|t j| q|S )zTestSuite that can be run by TestRunner
        Example usage:
            unittest.TextTestRunner().run(BackendTest(backend).test_suite)
        c                 S     | j jS rS   	__class__r   clr   r   r   <lambda>       z#Runner.test_suite.<locals>.<lambda>key)rU   	TestSuiterz   r|   rd   addTestsdefaultTestLoaderloadTestsFromTestCase)rK   suitecaser   r   r   
test_suite   s   
zRunner.test_suitec                 C  sN   |  d}t| j dd dD ]}t| D ]\}}t|||j qq|S )zOne single unittest.TestCase that hosts all the test functions
        Example usage:
            onnx_backend_tests = BackendTest(backend).tests
        OnnxBackendTestc                 S  r   rS   r   r   r   r   r   r      r   zRunner.tests.<locals>.<lambda>r   )rW   rz   rv   rd   rc   r{   r    )rK   testsri   rP   rj   r   r   r   r      s   

zRunner.testsref_outputsSequence[Any]outputsrtolfloatatol	model_dirc           	      C  s  zt jt|t| W n! ty. } ztdt| dt| d|p$dd|d }~ww tt|D ]}t|| tt	frt|| tt	fsdt
dt||  d| dt||  d|p_dd		tt|| D ]}| j|| | || | |||d
 qlq5t jj|| j|| jd| dd || jtkrt j|| ||  q5t j|| j|| j t jj|| || ||d q5d S )Nz Unable to compare expected type z and runtime type z (known test=?)zUnexpected type z for outputs[z]. Expected type is z).)r   zOutput z has incorrect shape)err_msg)r   r   )nptestingassert_equallen	TypeErrorrT   r%   
isinstancelisttupleAssertionErrorassert_similar_outputsassert_array_equalshapedtypeobjectassert_allclose)	clsr   r   r   r   r   er*   jr   r   r   r      sZ   	


	
zRunner.assert_similar_outputs   
model_testr   
models_dirc                 C  s   t  M}z+|jsJ td|j d|j  tj|d}t|j| td t	j
|| W n tyH } ztd|j d|   d }~ww W d    d S 1 sTw   Y  d S )NzStart downloading model z from fileDonez!Failed to prepare data for model z: )tempfileTemporaryDirectoryurlr'   
model_nameospathjoinr   onnxutils_extract_model_safer&   )r   r   r   tmpdirfilenamer   r   r   r   download_model   s"   

"zRunner.download_modelc              	   C  s   t jt dt jdd}t dt j|d}t j||j}t jt j|ds[t j|rOd}	 | d	| }t j|rH|d
7 }q5t|| 	 t 	| | j
||d |S )N	ONNX_HOME~.onnxONNX_MODELSmodels
model.onnxr   Tz.old.r   )r   r   )r   r   
expandusergetenvr   r   existsshutilmovemakedirsr   )r   r   	onnx_homer   r   bidestr   r   r   prepare_model_data   s$   
zRunner.prepare_model_dataCPUCUDArh   	test_name	test_funcr!   report_item#list[ModelProto | NodeProto | None]devicesIterable[str]r#   r   c           	        sF    dstd d	 fdd}|D ]}|| qd S )
Ntest_z!Test name must start with test_: devicerQ   r   r8   c                   s    d    j v rtd d dtj  d  td fd
d}t	|j < d S )N_zDuplicated test name "z" in category "rq   zBackend doesn't support device r"   r   device_test_kwargr   c               
     s   zi |}g |  R i |W S  t yA } z"dtjv s%dtjv r6td d|  W Y d }~d S W Y d }~d S d }~ww )Nz-vz	--verbosezTest z is effectively skipped: )r   sysargvr'   )r"   r   merged_kwargsr   )r   device_test_namer#   r   r   r   device_test_func'  s   "zCRunner._add_test.<locals>.add_device_test.<locals>.device_test_func)r"   r   r   r   r   r   )
lowerrI   
ValueErrorrU   skipIfr2   supports_devicer,   r-   r   )r   r   rh   r#   r   rK   r   r   )r   r   r   add_device_test   s   	z)Runner._add_test.<locals>.add_device_test)r   rQ   r   r8   )
startswithr   )	rK   rh   r   r   r   r   r#   r   r   r   r   r   	_add_test  s   


zRunner._add_testr    Fxr   seedr   randombool
np.ndarrayc                 C  s   | j jstd|d|  d| j jjdkr!td|d|  dtdd | j jjjD }|r>tjj	|d}||tj
S t|}t||| tj
S )	z8Generates a random tensor based on the input definition.zMInput expected to have tensor type. Unable to generate random data for model z and input .r   zMCurrently limited to float tensors. Unable to generate random data for model c                 s  s$    | ]}| d r|jndV  qdS )	dim_valuer   N)HasFieldr   )rn   dr   r   r   rp   K  s
    
z-Runner.generate_dummy_data.<locals>.<genexpr>)r   )rT   tensor_typeNotImplementedError	elem_typer   r   dimr   r   default_rngfloat32prodarangereshapeastype)r   r   rP   r   r   gennr   r   r   generate_dummy_data<  s0   

zRunner.generate_dummy_datar:   c                   sh   d g d
 fdd}j jv r'j|d	 j | fi jj   d S |d	 j |  d S )N	test_selfr   r   rQ   r   r8   c           !   
     sJ  j d ur^j dr^tjtjtjtddddj }tj|s-t	d|dtj
tdtjdd}tdtj|d	d
}tj|j}tj|s[t| d}njd u ri}nj}tj|d}d}ts}d|v r}d S t|}|d< tjdrtjjrj|stdjj||fi |}	|	d usJ |rt|d}
t|
}W d    n1 sw   Y  tj|d}tj|st| i }dd |jjD }d}g }tt|jj D ]W}|jj | j!|v rqtj|d| d}|"| |d7 }|jj |  j# djdd}|| j!< t|d}
|
$tj%&|'  W d    n	1 sFw   Y  qtj(|d }g }tt|jj)D ]}| d| d}tj|rv|"| q^d } |d u rtj*+|}|,d |}t-|D ]1\}}tj|d| d}t|d}
|
$tj%&|'  W d    n	1 sw   Y  qn_t-|D ]\}}tj|d| d}t./|| qnCt00tj|dD ]8}t1j|dd}t2|d  }t2|	,|}t3 fd!d"|d# D }j4|||5d$j6|5d%j7|d& qt00tj|d'D ]x}g }tt00tj|d(}t|D ]}tj|d| d}8|||jj | j9 q>g }tt00tj|d)}t|D ]}tj|d| d} 8| ||jj)| j9 qkt2|	,|}j4|||5d$j6|5d%j7|d& q*d S )*Nzonnx/backend/test/data/light/z..zUnable to find model r   r   r   r   r   r   lightTr   F
ai_onnx_mlr   is_compatiblezNot compatible with backendrbtest_data_set_0c                 S  s   h | ]}|j qS r   ro   )rn   r*   r   r   r   	<setcomp>  s    z6Runner._add_model_test.<locals>.run.<locals>.<setcomp>input_z.pbr   )r   rP   r   wb_output_output_ztest_data_*.npzbytes)encodinginputsc                 3  s,    | ]}t  ttfst n V  qd S rS   )r   r   rH   r   array)rn   fr   r   r   rp     s
    
z6Runner._add_model_test.<locals>.run.<locals>.<genexpr>r   r   r   )r   r   r   ztest_data_set*z
input_*.pbzoutput_*.pb):r   r   r   r   normpathr   dirname__file__r   FileNotFoundErrorr   r   r   r   r   r   r   r   loadhasattrr2   callabler  rU   SkipTestprepareopenmkdirgraphinitializerr%   r   inputrP   appendr   writer   
from_arraySerializeToStringsplitextoutput	referenceReferenceEvaluatorrun	enumerater   copyglobr   r   r   r   getr   r   _load_protorT   )!r   r   r#   model_pb_pathr   r   r   	use_dummymodelprepared_modelr  onxtest_data_setfeedsinitsn_inputr  r*   rP   valueprefixexpected_outputsrefr   otest_data_npz	test_datar   test_data_dir
inputs_num
input_fileref_outputs_numoutput_filemodel_markerr   rK   r  r   r'  Z  s  
















z#Runner._add_model_test.<locals>.runModel)r   r   r   rQ   r   r8   )rP   rG   r   )rK   r   r:   r'  r   rB  r   rJ   U  s    

zRunner._add_model_testproto_filenametarget_listlist[np.ndarray | list[Any]]model_type_protor   c           
      C  s  t |dx}| }|dr!t }|| |t| nD|drBt	 }|| t
|}t|tjs<J || n+|drYt }	|	| |t|	 ntd W d    d S W d    d S W d    d S W d    d S 1 sw   Y  d S )Nr  sequence_typer   optional_typezRLoading proto of that specific type (Map/Sparse Tensor) is currently not supported)r  readr   r   SequenceProtoParseFromStringr  r   to_listTensorPrototo_arrayr   r   ndarrayOptionalPrototo_optionalr'   )
rK   rE  rF  rH  r  protobuf_contentsequencetensortoptionalr   r   r   r,    s4   






"zRunner._load_proto)NN)r2   r3   r4   r5   r6   r7   r   r8   )rP   rQ   r   rR   )rX   rQ   r   r1   )r   r1   )r   rl   )r   rw   )r   r~   )r   rR   rS   )r   r   r   r   r   r   r   r   r   r5   r   r8   )r   r   r   rQ   r   r8   )r   r   r   rQ   )r   )rh   rQ   r   rQ   r   r!   r   r   r   r   r#   r   r   r8   )r   r   F)
r   r   r   r   rP   rQ   r   r   r   r   )r   r   r:   rQ   r   r8   )rE  rQ   rF  rG  rH  r   r   r8   )r   r   r   rO   rW   r^   r`   ra   rk   propertyrv   r|   r   r   classmethodr   r0   r   r   r   staticmethodr   rJ   r,  r   r   r   r   r1   =   sB    
"



-)
 $r1   )r   r   r   r   )-
__future__r   r,   r*  r   r[   r   r   r   r(   rU   collectionsr   r   typingr   r   urllib.requestr   numpyr   r   onnx.referencer   r	   r
   r   r   r   onnx.backend.test.loaderr   onnx.backend.test.runner.itemr   collections.abcr   r   r   onnx.backend.baser    onnx.backend.test.case.test_caser   r  r   r0   r1   r   r   r   r   <module>   s6    
