o
    toi                    @  sn  d Z ddlmZ ddlZddlZddlZddlZddlmZ ddl	m
Z
 ddlmZ ddlmZ ddlmZ dd	lmZ ddlZddlZddlZddlZdd
lmZ ddlZddlmZmZmZmZmZm Z  ddl!m"Z" ddl#m$Z$ ddl%m&Z& ddl'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3 ddl4m5Z5 ddl6m7Z7 ddl8m9Z9m:Z: ddl;m<Z< ddl=m>Z>m?Z? ddl@mAZAmBZB ddlCmDZD ddlEmFZF ddlGmHZH ddlImJZJmKZK ddlLmMZMmNZN ddlOmMZP ddlQmRZR erddlSmTZT eUeddZVeUed d!ZWd"d# ZXd$d% ZYd&d' ZZd(d) Z[d*d+ Z\d,d- Z]d>d7d8Z^G d9d: d:ej_Z`ead;kr5ejbd<d= dS dS )?zYou can run a specific test by using the following syntax.

::

    python onnx/test/reference_evaluator_test.py TestReferenceEvaluator.test_function_attribute_nested_graph
    )annotationsN)redirect_stdoutwraps)StringIO)getenv)dedent)TYPE_CHECKING)assert_allclose)AttributeProtoFunctionProto
ModelProtoTensorProtocheckerparser)get_roi_align_input_values)check_model)onnx_opset_version)make_function
make_graph
make_modelmake_model_gen_version	make_nodemake_operatorsetidmake_opsetidmake_sequence_type_protomake_tensormake_tensor_sequence_value_infomake_tensor_value_infomake_value_info)
from_array)ReferenceEvaluator)OpRunOpRunExpand)load_op)_get_indices_is_out)Cast_19Celu)Adam)_apply_causal)_vcelu1)_col2im_naive_implementation_2dcol2im_naive_implementation)Conv_conv_implementation)r.   )_conv_implementation_im2col)SequenceORT_MAX_IR_SUPPORTED_VERSION8$ORT_MAX_ONNX_OPSET_SUPPORTED_VERSION18c                      t   fdd}|S )Nc                    ,   t jdd u rtd | i | d S )Nonnxruntimezonnxruntime not installed	importlibutil	find_specunittestSkipTestargskwargsfn V/home/ubuntu/.local/lib/python3.10/site-packages/onnx/test/reference_evaluator_test.pywrapperX      
z'skip_if_no_onnxruntime.<locals>.wrapperr   rC   rF   rD   rB   rE   skip_if_no_onnxruntimeW      rI   c                   r6   )Nc                    r7   )Ntorchztorch not installedr9   r?   rB   rD   rE   rF   b   rG   z!skip_if_no_torch.<locals>.wrapperr   rH   rD   rB   rE   skip_if_no_torcha   rJ   rL   c                   r6   )Nc                    r7   )Ntorchvisionztorchvision not installedr9   r?   rB   rD   rE   rF   l   rG   z'skip_if_no_torchvision.<locals>.wrapperr   rH   rD   rB   rE   skip_if_no_torchvisionk   rJ   rN   c                 C  s*   t |trt| ||S t|}t| ||S N)
isinstanceintr   r   r   )name	elem_typeshapes_typerD   rD   rE   make_sequence_value_infou   s   
rV   c                 C  sV   dd l }t}| jD ]}|jdv r|j} nq	| jtks|tkr!d S |j|  dgdS )Nr   ) zai.onnxCPUExecutionProvider)	providers)	r8   r4   opset_importdomainversion
ir_versionr2   InferenceSessionSerializeToString)
onnx_modelortonnx_domain_opsetopsetrD   rD   rE   run_ort_inference|   s   



rd   c                   s  t |tstdt|dt| jt|kr$td| jd|dtd  t fddt	 D }t
| j| }t	 D ]0}|| || d	 || d	   }t|| | || d
  || d	  ||  d	 }	|	||< qDt|}
tj|
| jd}t|}t|jd   }t	|D ]L}t||jd   }t|}t	|D ]6}t||}t|}|| |ddd
f  ||  }t|}t|| jrd
||| < q| t| ||| < qq|S )zNaive implementation for `im2col`.

    Args:
        data: image (float)
        kernel_shape: kernel shape
        dilations: dilations
        pads: pads
        strides: strides

    Returns:
        result
    zUnexpected type z for kernel_shape.zShape mismatch z and .   c                   s    g | ]}| |   fqS rD   rD   .0in_dimspadsrD   rE   
<listcomp>   s     z/im2col_naive_implementation.<locals>.<listcomp>   r   dtypeN)rP   tuple	TypeErrortypelenrT   
ValueErrornparrayrangelistrQ   zerosrp   prodr%   r&   )datakernel_shape	dilationsrl   stridesnew_padslist_output_shapedkdndoutput_shapereskernel_sizeres_sizeri   i_rest_resji_kernelt_kerneli_imgt_imgrD   rj   rE   im2col_naive_implementation   s@   
 *


 
r   img
np.ndarrayr}   tuple[int, ...]r~   Sequence[int]rl   r   returnc           
      C  s   d }t | jd D ]6}t | jd D ],}t| ||df ||||}|d u r7| jd d |j }	tj|	| jd}||||df< qq	g |jd t|  dR }	||	S )Nr   rn   .rf   ro   )rx   rT   r   rv   emptyrp   rt   reshape)
r   r}   r~   rl   r   r   ncout	new_shaperD   rD   rE   im2col   s   
r   c                   @  s  e Zd ZdZedddZedddZdd Zdd Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd Zd d! Zd"d# Zd$d% Zd&d' Zd(d) Zd*d+ Zd,d- Zd.d/ Zd0d1 Zd2d3 Zd4d5 Zd6d7 Zd8d9 Zd:d; Zd<d= Zd>d? Zd@dA Z dBdC Z!dDdE Z"dFdG Z#dHdI Z$dJdK Z%dLdM Z&dNdO Z'dPdQ Z(dRdS Z)dTdU Z*dVdW Z+dXdY Z,dZd[ Z-d\d] Z.d^d_ Z/e0d`da Z1e0dbdc Z2ddde Z3dfdg Z4dhdi Z5djdk Z6dldm Z7dndo Z8dpdq Z9drds Z:e;dtdu Z<dvdw Z=e;dxdy Z>e;dzd{ Z?d|d} Z@d~d ZAdd ZBdd ZCdd ZDdd ZEdd ZFdd ZGdd ZHdd ZIdd ZJdd ZKdd ZLe0dd ZMdd ZNe;eOdd ZPdd ZQdd ZRdd ZSdd ZTdd ZUdd ZVdd ZWdd ZXedddZYeZjZ[e\]de^j_e^j`e^j`gddgge^jade^j_ddgddgge^jbdgfde^j_ddgddgge^jade^j_ddgddgge^jbdgfde^j_ddgddgge^jade^j_ddgddgge^jbdgfde^j_ddgddgge^jade^j_ddgddgge^jbdgfde^j_ddgddgge^jade^j_ddgddgge^jbdgfde^j_ddgddgge^jade^j_ddgddgge^jbdgfde^j_ddgddgge^jade^j_ddgddgge^jbdgfde^j_ddgddgge^jade^j_ddgddgge^jbdgfde^j_e^j`e^j`gddgge^jade^j_ddgddgge^jbdgfg	ddgdddZceZjZ[g ddѐdddZddd Zedd Zfdd Zgdd Zhdd Zidd Zjdd Zkelmenodddd Zpdd Zqdd Zrdd Zsdd  Ztdd Zudd Zvdd Zwdd Zxd	d
 Zydd ZzeZjZ[de^{d|dd e^{dd|dde^}dddg dg dg dgfde^{d|dd e^{dd|dde^~dddg dg dg dgfe^{d|ddg dg dg dge^}dddg dg dg dgfe^{ddd gd!d"gd#dg d$fe^~d%e^~d&e^}d&d#dd	fe^~d'e^~d&e^}d&ddd	fgd(d) ZeZjZ[e^{d|dde^{dd|dde^}dddg d*g d+g d,gfe^{d|dde^{dd|dde^~dddg d-g d.g d/gfe^e^{d|ddgd e^e^_ddgddgge^_dd0gddggge^}d1ddg d2g d3gg d4g d5ggfe^{d|ddg d6g d6g d6ge^}dddg d7g d8g d9gfe^{dddgddgd#dg d:fe^~d%e^~d&e^}d&d#dd	fe^~d'e^~d&e^}d&ddd	fgd;d< Zd=d> Zd?d@ ZdAdB ZdCdD ZdEdF ZdGdH ZdIdJ ZdKdL ZdMdN ZeZjZ[g dOdPdQ ZeZjZ[g dRdSdT ZdUdV ZdWdX ZdYdZ ZeZjZ[d[d\gd]d^gd_d`gdafg dbdcgg dddefdfg dgg dgg dhdefgdidj ZeZjZ[dkdlgdmd	g dng dogddgfg dpdmd	dqdrgdsdtgdrdrggg dufg dvdmddqdrgdsdtgdsdwggg dxfg dyg dygdmd	g dzg d{g dogg dzg d{g doggg d|g d|gfg d}d	d	g d~g d~g d~gg dfg d}drd	g d~g d~g d~gg dfddgdd	g dg dgddgfg dde^_g |dg fgdd Zdd ZeZjZ[g ddg ddefg dg dgdg dg dgdfg ddg ddfgdd Zdd ZeZjZ[ejg dg dfejg dg dfejd#gd#gfejg dg dfejg dg dfejd#gd#gfgdd ZeZjZ[e\]ejejfejejfdd ZeZjZ[e\]ejejfejejfdӐddZeZjZ[ejg dg dfejg dg dfejd#gd#gfejg dg dfejg dg dfejd#gd#gfgdd ZeZjZ[e\]ejejfejejfdd ZeZjZ[e\]ejejfejejfdӐddZdd Zdd Zdd ZeZjZ[g ddԐddZeZjZ[g d¢dÐdĄ ZdŐdƄ ZdǐdȄ Zdɐdʄ Zdːd̄ Zd	S (  TestReferenceEvaluatora6  
        <
            ir_version: 7,
            opset_import: [ "": 10, "com.microsoft": 1]
        >
        agraph (float[N, M] B01, float[N, M] B11, float[N, M] B21) => (float[N, M] D0)
        {
            C0 = Add(B01, B11)
            C1 = Sub(B11, B21)
            D0 = Mul(C0, C1)
        }
        m_defstrr   r   c                 C  s   t | }t| |S )zjParses a model from a string representation, including checking
        the model for correctness
        )r   parse_modelr   r   )r   mrD   rD   rE   _load_model   s   

z"TestReferenceEvaluator._load_modelFN            ?c              
     sB  t dtjd d g}t dtjd d g}t dtjd d g}t dtjd g}tdddgdg}| rtdddgdg}	|d ur~|d	k r~rZ rOtd
dgdg d}
n td
dgdgd}
n rgtd
dgdg d}
ntd
dgdg}
t||	|
gd|||g|g}nOrttjgtjdddnd } rttj gtjdddnd }d|rdnd|rdndg}td
|dg}
dd ||fD }t||	|
gd|||g|g|d} fdd}ntdddgdg}	t||	gd|||g|g}dd }|d u rt	|}n
t	|t
d|gd}z	t| W ||fS  ty  } ztd| |d }~ww )NXABYMatMulXAAddY_clip   Clip)minmax)r   )r   lrro   mirR   marW   c                 S  s   g | ]}|r|qS rD   rD   )rh   _rD   rD   rE   rm     s    z=TestReferenceEvaluator._linear_regression.<locals>.<listcomp>initializerc                   s   t || |  S rO   )rv   clipxab	max_value	min_valuerD   rE   f  s   z4TestReferenceEvaluator._linear_regression.<locals>.fc                 S  s   || | S rO   rD   r   rD   rD   rE   <lambda>  s    z;TestReferenceEvaluator._linear_regression.<locals>.<lambda>opset_importszchecker fails for
)r   r   FLOATr   r   r    rv   rw   float32r   r   r   	ExceptionAssertionError)r   rc   r   r   r   r   r   r   node1node2node3graphr   r   inputsr   r   r`   erD   r   rE   _linear_regression   s\   

z)TestReferenceEvaluator._linear_regressionc                 C  sJ   t dtjd d g}| t t| W d    d S 1 sw   Y  d S )Nr   )r   r   r   assertRaisesrr   r!   )selfr   rD   rD   rE   #test_reference_evaluator_exceptions%  s   
"z:TestReferenceEvaluator.test_reference_evaluator_exceptionsc                 C  s   t t j}t| t|}| |jg d | |jdg | |j	ddd t
jddgddggt
jd	}t
jd
dgddggt
jd	}t
jddgddggt
jd	}|d |||dd }|| ||  }t|| d S NB01B11B21D0
   rn   rW   zcom.microsoftr   rf      ro               )r   r   m2_defr   r   r!   assertEqualinput_namesoutput_namesopsetsrv   rw   r   runr
   r   r   sessr   yzr   expectedrD   rD   rE   %test_reference_evaluator_no_attribute*  s   
z<TestReferenceEvaluator.test_reference_evaluator_no_attributec                 C  s   t t j}t| t|}| |jg d | |jdg | |j	ddd t
jddgddggt
jd	}t
jd
dgddggt
jd	}t
jddgddggt
jd	}|jd |||ddd}| |t || ||  }t|d | d S )Nr   r   r   rn   r   r   rf   r   ro   r   r   r   r   r   r   r   r   T)intermediate)r   r   r   r   r   r!   r   r   r   r   rv   rw   r   r   assertIsInstancedictr
   r   rD   rD   rE   2test_reference_evaluator_no_attribute_intermediate8  s   
zITestReferenceEvaluator.test_reference_evaluator_no_attribute_intermediatec                 C  s   t t j}t| t| }| |jg d | |j	dg | |j
ddd tjddgddggtjd	}tjd
dgddggtjd	}tjddgddggtjd	}|d |||dd }|| ||  }t|| d S r   )r   r   r   r   r   r!   r_   r   r   r   r   rv   rw   r   r   r
   r   rD   rD   rE   +test_reference_evaluator_no_attribute_bytesG  s   
zBTestReferenceEvaluator.test_reference_evaluator_no_attribute_bytesc           	   	   C  s  t t j}tjddgddggtjd}tjddgdd	ggtjd}tjd
dgddggtjd}| jdd; t|dd}t }t	| |
d |||d W d    n1 sYw   Y  | }d}| || W d    n1 stw   Y  | jdd@ t|dd}t }t	| |
d |||d W d    n1 sw   Y  | }tdd}| || W d    n1 sw   Y  | jdd@ t|dd}t }t	| |
d |||d W d    n1 sw   Y  | }tdd}| || W d    n	1 sw   Y  | jddB t|dd}t }t	| |
d |||d W d    n	1 s@w   Y  | }tdd}| || W d    d S 1 sbw   Y  d S )Nr   rn   rf   r   ro   r   r   r   r   r   r   r   r   )levelverboser   z:Add(B01, B11) -> C0
Sub(B11, B21) -> C1
Mul(C0, C1) -> D0
a  
                 +I B01: float32:(2, 2) in [0.0, 3.0]
                 +I B11: float32:(2, 2) in [4.0, 7.0]
                 +I B21: float32:(2, 2) in [-7.0, -4.0]
                Add(B01, B11) -> C0
                 + C0: float32:(2, 2) in [4.0, 10.0]
                Sub(B11, B21) -> C1
                 + C1: float32:(2, 2) in [8.0, 14.0]
                Mul(C0, C1) -> D0
                 + D0: float32:(2, 2) in [32.0, 140.0]
                
a  
                 +I B01: float32:(2, 2):[0.0, 1.0, 2.0, 3.0]
                 +I B11: float32:(2, 2):[4.0, 5.0, 6.0, 7.0]
                 +I B21: float32:(2, 2):[-4.0, -5.0, -6.0, -7.0]
                Add(B01, B11) -> C0
                 + C0: float32:(2, 2):[4.0, 6.0, 8.0, 10.0]
                Sub(B11, B21) -> C1
                 + C1: float32:(2, 2):[8.0, 10.0, 12.0, 14.0]
                Mul(C0, C1) -> D0
                 + D0: float32:(2, 2):[32.0, 60.0, 96.0, 140.0]
                   a  
                 +I B01: float32:(2, 2):[0.0, 1.0, 2.0, 3.0]
                 +I B11: float32:(2, 2):[4.0, 5.0, 6.0, 7.0]
                 +I B21: float32:(2, 2):[-4.0, -5.0, -6.0, -7.0]
                Add(B01, B11) -> C0
                -- begin Add.run(2 inputs)
                -- done Add.run -> 1 outputs
                 + C0: float32:(2, 2):[4.0, 6.0, 8.0, 10.0]
                Sub(B11, B21) -> C1
                -- begin Sub.run(2 inputs)
                -- done Sub.run -> 1 outputs
                 + C1: float32:(2, 2):[8.0, 10.0, 12.0, 14.0]
                Mul(C0, C1) -> D0
                -- begin Mul.run(2 inputs)
                -- done Mul.run -> 1 outputs
                 + D0: float32:(2, 2):[32.0, 60.0, 96.0, 140.0]
                )r   r   r   rv   rw   r   subTestr!   r   r   r   getvaluer   r   lstrip)	r   r   r   r   r   r   stdoutr   logrD   rD   rE   -test_reference_evaluator_no_attribute_verboseU  sp   
	


$zDTestReferenceEvaluator.test_reference_evaluator_no_attribute_verbosec           	      C  s   t  \}}tjddgddggtjd}tjddgtjd}tjdgtjd}||||}t|}|d |||dd }t|| d S )Nr   rn   rf   r   ro   r   r   r   r   )r   r   rv   rw   r   r!   r   r
   )	r   r   r   r   r   r   r   r   gotrD   rD   rE   test_reference_evaluator_lr  s   z2TestReferenceEvaluator.test_reference_evaluator_lrc           
      C  sv  | j ddY tjdd\}}tjddgddggtjd	}tjddgtjd	}tjd
gtjd	}||||}t|}|jd }| |j	j
d |d |||dd }	t|	| W d    n1 sbw   Y  | j ddZ tjdd d\}}tjddgddggtjd	}tjddgtjd	}tjd
gtjd	}||||}t|}|jd }| |j	j
d |d |||dd }	t|	| W d    n1 sw   Y  | j dd[ tjdd d\}}tjddgddggtjd	}tjddgtjd	}tjd
gtjd	}||||}t|}|jd }| |j	j
d |d |||dd }	t|	| W d    d S 1 s4w   Y  d S )Nmin+maxoptT)r   r   rn   rf   r   ro   r   r   Clip_11r   r   )r   r   r   )r   r   )r   r   r   rv   rw   r   r!   	rt_nodes_r   	__class____name__r   r
   
r   r   r   r   r   r   r   r   	last_noder  rD   rD   rE    test_reference_evaluator_lr_clip  sH   


$z7TestReferenceEvaluator.test_reference_evaluator_lr_clipc           
      C  s  | j ddh tjddd\}}tjddgdd	ggtjd
}tjddgtjd
}tjdgtjd
}||||}t|}|jd }| |j	j
d | |jd | |jd |d |||dd }	t|	| W d    n1 sqw   Y  | j ddi tjddd d\}}tjddgdd	ggtjd
}tjddgtjd
}tjdgtjd
}||||}t|}|jd }| |j	j
d | |jd | |jd |d |||dd }	t|	| W d    n1 sw   Y  | j ddj tjddd d\}}tjddgdd	ggtjd
}tjddgtjd
}tjdgtjd
}||||}t|}|jd }| |j	j
d | |jd | |jd |d |||dd }	t|	| W d    d S 1 saw   Y  d S )Nr  r  Tr   )r   rc   r   rn   rf   r   ro   r   r   Clip_6r   r   )r   rc   r   g   r   )r   rc   r   g   G)r   r   r   rv   rw   r   r!   r  r   r  r	  r   r   r   r
   r
  rD   rD   rE   "test_reference_evaluator_lr_clip_6  s\   




$z9TestReferenceEvaluator.test_reference_evaluator_lr_clip_6c                 C  sR   t d}t|}tjg dtjdd}|d d|id }|}t|| d S )Naf  
            <
              ir_version: 8,
              opset_import: [ "" : 14, "local" : 1],
              producer_name: "test",
              producer_version: "1.0",
              model_version: 1,
              doc_string: "Test preprocessing model"
            >
            agraph (uint8[H, W, C] x) => (uint8[H, W, C] x_processed)
            {
                x_processed = local.func(x)
            }

            <
              opset_import: [ "" : 14 ],
              domain: "local",
              doc_string: "function 1"
            >
            f1 (x) => (y) {
                y = Identity(x)
            }

            <
              opset_import: [ "" : 14 ],
              domain: "local",
              doc_string: "function 2"
            >
            f2 (x) => (y) {
                y = Identity(x)
            }

            <
              opset_import: [ "" : 14, "local" : 1 ],
              domain: "local",
              doc_string: "Preprocessing function."
            >
            func (x) => (y) {
                x1 = local.f1(x)
                y = local.f2(x1)
            }
        )r   rn   r   ro   rn   rn   r   r   r   )	r   r   r!   rv   rw   uint8r   r   r
   )r   r   r   r   resultr   rD   rD   rE   test_nested_local_functions  s   -z2TestReferenceEvaluator.test_nested_local_functionsc           
      C  s   t dtjd d g}t dtjd g}tddgdgdgdd}t|gd|g|g}t|tddgd	}t| t	d

dtj}|jddd}t|}|d d|id }	t|	| d S )Nr   r   	ReduceSumrn   axeskeepdimsrsrW   r   r   <   r   r   r   axisr  r   r   r   r   r   r   r   r   r   rv   aranger   astyper   sumr!   r   r
   
r   r   r   r   r   r`   r   r   r   r  rD   rD   rE   test_reduce_sum_11;  s   z)TestReferenceEvaluator.test_reduce_sum_11c           
      C  s   t dtjd d g}t dtjd g}tddgdgdgdd}t|gd|g|g}t|tddgd	}t| t	d

dtj}|| jddd}t|}|d d|id }	t|	| d S )Nr   r   ReduceSumSquarern   r  r  rW   r   r   r  r  r  r   r  r   rD   rD   rE   test_reduce_sum_square_11H  s   z0TestReferenceEvaluator.test_reduce_sum_square_11c                 C  s   t dtjd d g}t dtjd d g}t dtjd g}tdddgdgdd}t|gd||g|g}t|tdd	gd
}t| t	
ddt	j}t	jdgt	jd}|jddd}	t|}
|
d ||dd }t||	 d S )Nr   r   r   r  rn   r  r  rW      r   r  r  ro   r  r   r   r   r   r   r   INT64r   r   r   r   r   rv   r  r   r  r   rw   int64r  r!   r   r
   r   r   r   r   r   r   r`   r   r   r   r   r  rD   rD   rE   test_reduce_sum_13U  s   z)TestReferenceEvaluator.test_reduce_sum_13c                 C  sD  t  }d}td|t|dg}tdddgdg}t }d|_d|_tj|_|j	| t
|d	ddgdg|g|dg}tdtjd d g}tdtjd g}tdtjd g}	ttd	ddgdg|dd
gd||g|	g}
t|
||gd}t|}tddtj}tjdgtjd}|d ||dd }|jddd}t|| d S )NcustomrW   rn   r  r   r  r   r  MyReduceSum)r[   r  exampler   	functionsr   r   rf   r   ro   )r   r  r   r  )r   r   r   r   rR   ref_attr_nameINTrs   	attributeappendr   r   r   r   r(  r   r   r!   rv   r  r   r  r   rw   r)  r   r  r
   )r   rc   
new_domainr   nodeattmy_reduce_sumr   r  r   r   r`   r   r   r   r  r   rD   rD   rE   test_reduce_sum_attributed  sV   
	z0TestReferenceEvaluator.test_reduce_sum_attributec                 C  s   t dtjd d g}t dtjd d g}t dtjd g}tdddgdgdd}t|gd||g|g}t|tdd	gd
}t| t	
ddt	j}t	jdgt	jd}|| jddd}	t|}
|
d ||dd }t||	 d S )Nr   r   r   r"  rn   r$  r  rW      r   r  r  ro   r  r&  r   r'  r*  rD   rD   rE   test_reduce_sum_square_18  s   z0TestReferenceEvaluator.test_reduce_sum_square_18c                 C  s   t dtjd d g}t dtjd d g}t dtjd g}tdddgdgdd}t|gd||g|g}t|tdd	gd
}t| t	
ddt	j}t	jg t	jd}|jdd}	t|}
|
d ||dd }t||	 d S )Nr   r   r   r  rn   r$  r  rW   r%  r   r  r  ro   r&  r   r'  r*  rD   rD   rE   test_reduce_sum_13_empty_axes  s   z4TestReferenceEvaluator.test_reduce_sum_13_empty_axesc                 C  s   t dtjd d g}t dtjd d g}t dtjd g}tdddgdgdd}t|gd||g|g}t|tdd	gd
}t| t	
ddt	j}t	jg t	jd}|| jdd}	t|}
|
d ||dd }t||	 d S )Nr   r   r   r"  rn   r$  r  rW   r;  r   r  r  ro   r&  r   r'  r*  rD   rD   rE   $test_reduce_sum_square_18_empty_axes  s   z;TestReferenceEvaluator.test_reduce_sum_square_18_empty_axesc           	      C  s   t dtjd d g}t dtjd g}tddgdgddd}t|gd|g|g}t|tddgd	}t| t	d

dtj}t|}|d d|id }t|| d S )Nr   r   r  rn   r  noop_with_empty_axesr  rW   r%  r   r  r  r   r   r   r   r   r   r   r   r   rv   r  r   r  r   r!   r   r
   	r   r   r   r   r   r`   r   r   r  rD   rD   rE   "test_reduce_sum_13_empty_axes_noop  s   z9TestReferenceEvaluator.test_reduce_sum_13_empty_axes_noopc           	      C  s   t dtjd d g}t dtjd g}tddgdgddd}t|gd|g|g}t|tddgd	}t| t	d

dtj}t|}|d d|id }t|| | d S )Nr   r   r"  rn   r?  r  rW   r;  r   r  r  r   rA  rB  rD   rD   rE   )test_reduce_sum_square_18_empty_axes_noop  s   z@TestReferenceEvaluator.test_reduce_sum_square_18_empty_axes_noopc                 C  s   t dtjd d g}t dtjd g}t dtjd g}tdddgdg}t|gd||g|g}t|tddgd}t| t	d	
d
tj}tjdgtjd}||k}	t|}
|
d ||dd }t||	 d S )Nr   r   ZGreatergrW   r%  r   r   rf   rf   rf   ro   r   r   r   r   r   r   r   r   r   r   r   rv   r  r   r  r   rw   r!   r   r
   r   r   r   rE  r   r   r`   r   r   r   r   r  rD   rD   rE   test_greater     z#TestReferenceEvaluator.test_greaterc                 C  sp   t dddgdg}tddtj}tjdgtjd}||k}t|}|d ||d	d
 }t	|| d S )NrF  r   r   rE  r   rH  rf   ro   rI  r   )
r   rv   r  r   r  r   rw   r!   r   r
   )r   r   r   r   r   r   r  rD   rD   rE   test_node_proto  s   z&TestReferenceEvaluator.test_node_protoc                 C  s   t dtjd d g}t dtjd g}t dtjd g}tdddgdg}t|gd||g|g}t|tddgd}t| t	d	
d
tj}tjdgtjd}||k}	t|}
|
d ||dd }t||	 d S )Nr   r   rE  GreaterOrEqualrG  rW   r%  r   r   rH  rf   ro   rI  r   rJ  rK  rD   rD   rE   test_greater_or_equal  rM  z,TestReferenceEvaluator.test_greater_or_equalc              	   C  s  t dtjd g}tdg dgttjdgtjdd}t|gdg |g}t dtjd g}tdg dgttjdgtjdd}t|gdg |g}ttjdgtjdd	d
}tddd	gdg}tddgdg||d}t dtjd d g}	t dtjd g}
t||gd|	g|
g|gd}t	|}t
|}| t|d tjdgtjd}|d d|id }ttjdgtjd| tjdgtjd}|d d|id }ttjdgtjd| d S )NCConstantrn   ro   )value_floatsgthenr   gelsezeror   rF  r   GIfrE  )then_branchelse_branchrG  r   zReferenceEvaluator(X) -> Zr   )r   r   r   r   r    rv   rw   r   r   r   r!   r   r   r   r
   )r   rQ  bthen
bthen_bodybelse
belse_bodyrV  greaternode_ifr   rE  r   	model_defr   r   r  rD   rD   rE   test_if  sJ   zTestReferenceEvaluator.test_ifc                 C  s  t dtjdg}t dtjdg}tg dtj}tg dtj}tdg dgt|d}tdg dgt|d}t	|gdg |g}t	|gd	g |g}td
dgdg||d}	t
 }
d|
_d|
_|
jdg |
jdg |
j|	g t }|
jtd|g t	tdddgdgdgdt dtjg gt dtjdggd}t|dtd|tddgd}|j|
g t|}|d dtdi}tjg dtjd}t|d | d S )Nthen_outr   else_outrn   rf   r   r   r   )r   r   r   rf   rn   rR  r   outputsvalue	then_body	else_bodyrX  f_condf_resr   rg  rY  rZ  r,  rC   rW   condr   )r[   r   rg  r   )nodesrR   r   rg  testrn   )producer_namer   Tro   r   )r   r   r   rv   rw   r  r   r   r    r   r   r[   rR   inputextendoutputr7  r   rZ   r   BOOLr   r0  r!   r   r
   )r   rc  rd  r   r   then_const_nodeelse_const_noderi  rj  if_noder   rc   r   r   r   r  r   rD   rD   rE   test_if_function1  sV   z'TestReferenceEvaluator.test_if_functionc                 C  s  t  }d}td|t|dg}tdg dg}t }d|_d|_tj|_|j	| tdd	d
gdg}tdddgdg}t
|dd	d
gdg|||g|dg}td	tjd d g}	td
tjd d g}
tdtjd g}ttdd	d
gdg|tdtjdgdgdtddgdggd|	|
g|g}t|||gd}t|}tddtj}tjddgtjd}|d ||dd }t|| d }t|| d S )Nr,  rW   rn   rR  r   rh  biasr   r   r   r   r   r   LinearRegressionY1former_Bq=
ףp?)r[   rz  Absr.  r/  r   r1  r   ro   r&  r   )r   r   r   r   rR   r2  TENSORrs   r4  r5  r   r   r   r   r   r   r   r!   rv   r  r   r  r   rw   r   absr
   )r   rc   r6  r   cstr8  r   r   linear_regressionr   r   r   r   r`   r   r   r   r  r   rD   rD   rE   test_function_attributed  s\   

z.TestReferenceEvaluator.test_function_attributec                 C  sH  t  }d}td|t|dg}tdg dg}t }d|_d|_tj|_|j	| tdg dg}t }d|_d	|_tj|_|j	| t
dtjd g}t
dtjd g}t|gd
g |g}	t|gdg |g}
tdg dgttjdgtjdd}tddgdg}tdddgdg}tddgdg|	|
d}tdddgdg}tdddgdg}t|dddgdg||||||g|dd	g}t
dtjd d g}t
dtjd d g}t
dtjd g}ttdddgdg|tdtjdgd gtd!tjdgd"gd#td$dgdggd%||g|g}t|||gd&}t| t|}| |jd jjd' | |jd jjjd( |jd j}d)}|jD ]}|jjdrM|jsKtd*t| d+d,}q3|sVtd-td. d/!tj}tjdd0gtjd}|"d |d |d1d }t#|| d  }t$|| |"d |d2 |d1d }t#|| d" }t$|| d S )3Nr,  rW   rn   rR  B1rh  bias1B2bias2ri  rj  rV  r   ro   rf  	ReduceMinr   XminrF  rk  rX  r   rm  r   r   r   r   r   r{  r|  	former_B1r~  	former_B2	  )r[   r  r  r  r.  r/  
OpFunctionr!   FzNested node z2 declares no linked attribute but a subgraph does.TzBNo node 'If' was found, has_linked_attribute could not be checked.r   r1  r   r&  r   )%r   r   r   r   rR   r2  r  rs   r4  r5  r   r   r   r   r    rv   rw   r   r   r   r   r   r!   r   r  r  r	  impl_
startswithhas_linked_attributer   r  r   r  r   r  r
   )r   rc   r6  r   cst1r8  cst2rc  rd  ri  rj  rV  minirk  rx  r   r   r  r   r   r   r   r`   r   fctcheckedr7  r   r   r  r   rD   rD   rE   $test_function_attribute_nested_graph  s   


z;TestReferenceEvaluator.test_function_attribute_nested_graphc                  C  s  t  }d}td|t|dg}tdg dg}t }d|_d|_tj|_|j	| tdg dg}t }d|_d	|_tj|_|j	| t
dtjd g}t
dtjd g}t|gd
g |g}	t|gdg |g}
tdg dgttjdgtjdd}tdddgdg}tddgdg|	|
d}tdg dg}t }d|_d|_tj|_|j	| t
dtjd g}t|gd
g |g}t
dtjd g}t|||gdg |g}tdg dgttjdgtjdd}tddgdg}tdddgdg}tddgdg||d}tddd gd!g}td"d!dgd#g}t|d$dd gd#g||||||g|g d%}t
dtjd d g}t
d tjd d g}t
d#tjd g}ttd$dd gd&g|td'tjdgd(gtd)tjdgd*gtd+tjdgd,gd-td.d&gd#ggd/||g|g}t|||gd0}t| t|}td1d2tj}tjdd3gtjd}|d |d |d4d }t|| d* }t|| |d |d5 |d4d }t|| d6 }t|| |d |d7 |d4d }t|| d( }t|| d S )8Nr,  rW   rn   rR  r  rh  r  r  r  ri  rj  c100d   ro   rf  rF  r  
f_cond_100rX  B4rm  B3bias3rV  r   r  r   Lessf_cond_zeror   r   r   r   r   r   r{  )r  r  r  r|  r  r~  r  r  	former_B3i)r[   r  r  r  r  r.  r/  r   r1  r   r&  r   ix  i  )r   r   r   r   rR   r2  r  rs   r4  r5  r   r   r   r   r    rv   rw   r   r   r   r   r   r!   r  r   r  r   r  r
   ) r   rc   r6  r   r  r8  r  rc  rd  
then_body1
else_body1r  rk  rx  cst3
then_body2
else_body2rV  r  r   r   r  r   r   r   r   r`   r   r   r   r  r   rD   rD   rE   +test_function_attribute_nested_nested_graph	  s   	
	

zBTestReferenceEvaluator.test_function_attribute_nested_nested_graphc                 C  sD  G dd d}G dd dt }G dd dt }tdtjd d g}tdtjd g}tddgdgd	d
d}t|gd|g|g}t|td
dgd}t	d
dtjd }	| t t| W d    n1 siw   Y  tddgdgd	d
d}t|gd|g|g}t|td
dgd}| t t||gd W d    n1 sw   Y  tddgdgd	d
d}t|gd|g|g}t|td
dgd}| t t||gd W d    n1 sw   Y  tddgdgd	d
d}t|gd|g|g}t|td
dgd}t|||gd}
|
d d|	id }d|	d	  }t|| d S )Nc                   @  s    e Zd ZdZdd Zdd ZdS )z:TestReferenceEvaluator.test_custom_node.<locals>._InvAlphar,  c                 S  s   || _ || _d S rO   )	onnx_node
run_params)r   r  r  rD   rD   rE   __init__  s   
zCTestReferenceEvaluator.test_custom_node.<locals>._InvAlpha.__init__c                 S     d|| j   fS Nrn   alphar   r   rD   rD   rE   _run     z?TestReferenceEvaluator.test_custom_node.<locals>._InvAlpha._runN)r	  
__module____qualname__	op_domainr  r  rD   rD   rD   rE   	_InvAlpha  s    r  c                   @  s   e Zd Zdd ZdS )z:TestReferenceEvaluator.test_custom_node.<locals>.InvAlpha2c                 S  r  r  r  r  rD   rD   rE   r    r  z?TestReferenceEvaluator.test_custom_node.<locals>.InvAlpha2._runN)r	  r  r  r  rD   rD   rD   rE   	InvAlpha2  s    r  c                   @     e Zd ZdZdddZdS )z9TestReferenceEvaluator.test_custom_node.<locals>.InvAlphar,  Nc                 S  s   |p| j }d||  fS r  r  r   r   r  rD   rD   rE   r    s   
z>TestReferenceEvaluator.test_custom_node.<locals>.InvAlpha._runrO   r	  r  r  r  r  rD   rD   rD   rE   InvAlpha      r  r   r         ?r,  r  r[   r  rn   r   r  r  new_opsr   )r"   r   r   r   r   r   r   r   rv   r  r   r  r   r   NotImplementedErrorr!   rr   r   r
   )r   r  r  r  r   r   r   r   r`   r   r   r  r   rD   rD   rE   test_custom_node  s>   

z'TestReferenceEvaluator.test_custom_nodec           	      C     G dd dt }tdtjd d g}tdtjd g}tddgdgddd}t|gd|g|g}t|tdd	gd
}t	d
dtjd	 }t||gd}| t |d d|i W d    d S 1 sfw   Y  d S )Nc                   @  r  )zDTestReferenceEvaluator.test_custom_no_output_tuple.<locals>.InvAlphar,  Nc                 S  s   |p| j }d||  S r  r  r  rD   rD   rE   r    s   
zITestReferenceEvaluator.test_custom_no_output_tuple.<locals>.InvAlpha._runrO   r  rD   rD   rD   rE   r    r  r  r   r   r  r,  r  r  rn   r   r  r  r  r"   r   r   r   r   r   r   r   rv   r  r   r  r   r!   r   rr   r   	r   r  r   r   r   r   r`   r   refrD   rD   rE   test_custom_no_output_tuple     "z2TestReferenceEvaluator.test_custom_no_output_tuplec           	      C  r  )Nc                   @  r  )zATestReferenceEvaluator.test_custom_empty_output.<locals>.InvAlphar,  Nc                 S  s   ~~dS )NrD   rD   r  rD   rD   rE   r    s   zFTestReferenceEvaluator.test_custom_empty_output.<locals>.InvAlpha._runrO   r  rD   rD   rD   rE   r    r  r  r   r   r  r,  r  r  rn   r   r  r  r  )r"   r   r   r   r   r   r   r   rv   r  r   r  r   r!   r   ru   r   r  rD   rD   rE   test_custom_empty_output  r  z/TestReferenceEvaluator.test_custom_empty_outputc           	      C  r  )Nc                   @  r  )z@TestReferenceEvaluator.test_custom_tuple_tuple.<locals>.InvAlphar,  Nc                 S  sF   |p| j }ttd||  gg}t|tsJ t|d ts!J |S )Nrn   r   )r  rq   rP   r   r   r  r   rD   rD   rE   r    s
   
zETestReferenceEvaluator.test_custom_tuple_tuple.<locals>.InvAlpha._runrO   r  rD   rD   rD   rE   r    r  r  r   r   r  r,  r  r  rn   r   r  r  r  r  r  rD   rD   rE   test_custom_tuple_tuple  s   
"z.TestReferenceEvaluator.test_custom_tuple_tuplec           	        s   G dd d G  fdddt }tdtjd d g}tdtjd g}tddgdgddd	}t|gd
|g|g}t|tddgd}t	d
dtjd }t||gd}| t |d d|i W d    d S 1 sow   Y  d S )Nc                   @     e Zd ZdS )zLTestReferenceEvaluator.test_custom_tuple_unexpected_type.<locals>.CustomTypeNr	  r  r  rD   rD   rD   rE   
CustomType      r  c                      s   e Zd ZdZd fdd	ZdS )zJTestReferenceEvaluator.test_custom_tuple_unexpected_type.<locals>.InvAlphar,  Nc                   s0   ~~  f}t |tsJ t |d  sJ |S )Nr   )rP   rq   r  r  rD   rE   r    s
   zOTestReferenceEvaluator.test_custom_tuple_unexpected_type.<locals>.InvAlpha._runrO   r  rD   r  rD   rE   r    s    r  r   r   r  r,  r  r  rn   r   r  r  r  r  r  rD   r  rE   !test_custom_tuple_unexpected_type  s   
"z8TestReferenceEvaluator.test_custom_tuple_unexpected_typec                 C  sb  t dtjg }t dtjg }t dtjg }tdtjd }tdtjd }tg dtj	}t
dg dgtd	tj|j| td
d}t
dg dgtdtjddgd
d}t
dg dgtdtjddgd
d}	t
dg dgtdtjddgd
d}
t
dddgdgd}t
dddgdgd}t
dg ddgd}t
dddgdgd}t
ddgdgd}t||||	||
|||g	d |||g||g}t
d!g d"d#g|d$}t
d%d#gd&gddd'}td(tj}g }tdtj}ttd)t d*tj|jt d+tj|jtd,tjg gt d&tjd g||gd-d.}tjg d/tj	d0}t|}|||d"}|d |}t|d | d S )1Ncond_incond_out
iter_countseq_inseq_outre  rR  r   const_tensor_x)rR   	data_typedimsvalsrf  oneconst_tensor_onerD   rn   slice_startconst_tensor_zerorn   r   r  const_tensor_axesr   endr   rg  	Unsqueeze	slice_endSlice)r   r  r  	slice_outSequenceInsertIdentity	loop_bodyLoop)
trip_countrn  	seq_emptyseq_res)r   rg  bodyConcatFromSequencer   )r   rg  r  new_axisr   	loop_testr  rn  r  )rR   r   rg  ro  )r   )r   r          @r   r        @r   r  r        @r   r  r  r        @ro   )r   r   ru  r(  r   r   rv   rw   r  r   r   r   rT   flattenfloatr   r)  bool_r   rV   r!   r   r
   )r   r  r  r  r  r  r   x_const_nodeone_const_nodezero_const_node	axes_nodeadd_nodeend_unsqueeze_node
slice_nodeinsert_nodeidentity_noder  r7  node_concatr  r  rn  ra  r   oinfr   r  rD   rD   rE   	test_loop
  s   
z TestReferenceEvaluator.test_loopc                 C  s   t dtjd g}t dtjd g}tddgdgdd}t|gd|g|g}t|}t| t|}|d dt	j
dt	jdd	 id
 }| |jd | |jt	j | | d | | d d S )Nr   r   	Bernoulli        seedrG  rf   r   ro   r  r   gh㈵grZ|
 ?r   r   r   r   r   r   r   r!   r   rv   rz   r   r   rT   rp   assertGreaterr   
assertLessr   r   r   r   r   r   r`   r   r  rD   rD   rE   test_onnxt_runtime_bernoulli  s   $z3TestReferenceEvaluator.test_onnxt_runtime_bernoullic                 C  s   t dtjd g}tdg dgdddgd}t|gdg |g}t|}t| t|}|d i d }| 	|j
d	 | 	|jtj | | d | | d
 d S )Nr   RandomUniformr  rf   r   r  rT   rG  r   r  rn   )r   r   r   r   r   r   r   r!   r   r   rT   rp   rv   r   r	  r   r
  r   r   r   r   r   r`   r   r  rD   rD   rE   !test_onnxt_runtime_random_uniform  s   z8TestReferenceEvaluator.test_onnxt_runtime_random_uniformc                 C  s   t dtjd g}t dtjd g}tddgdgdd}t|gd|g|g}t|}t| t|}|d dt	j
dt	jdid	 }| |jd | |jt	j | | d	 | | d
 d S )Nr   r   RandomUniformLiker  r  rG  r  ro   r   rn   r  r  rD   rD   rE   &test_onnxt_runtime_random_uniform_like  s    z=TestReferenceEvaluator.test_onnxt_runtime_random_uniform_likec                 C  s   t dtjd g}tdg dgdddgd}t|gdg |g}t|}t| t|}|d i d }| 	|j
d	 | 	|jtj d S )
Nr   RandomNormalr  rf   r   r  rG  r   r  )r   r   r   r   r   r   r   r!   r   r   rT   rp   rv   r   r  rD   rD   rE    test_onnxt_runtime_random_normal  s   z7TestReferenceEvaluator.test_onnxt_runtime_random_normalc                 C  s   t dtjd g}t dtjd g}tddgdgdd}t|gd|g|g}t|}t| t|}|d dt	j
dt	jdid	 }| |jd | |jt	j d S )
Nr   r   RandomNormalLiker  r  rG  r  ro   r   )r   r   r   r   r   r   r   r!   r   rv   rz   r   r   rT   rp   r  rD   rD   rE   %test_onnxt_runtime_random_normal_like  s    z<TestReferenceEvaluator.test_onnxt_runtime_random_normal_likec                 C  s^   t jdd}| |jd tjddgddggtjd}t j|dd}t|dd}t	|| d S )Nr  r  r   rn   r   rf   ro   )
r(   creater   r  rv   rw   r   evalr+   r
   )r   instr   r   r   rD   rD   rE   test_eval_celu  s   z%TestReferenceEvaluator.test_eval_celuc                 C  sN   t jddgddggt jd}tj|tjd}tj|tjd}|}t|| d S )Nr   rn   r   rf   ro   to)	rv   rw   r   r'   r  r   FLOAT8E4M3FNr   r
   )r   r   r   dyr   rD   rD   rE   test_eval_cast  s
   z%TestReferenceEvaluator.test_eval_castc                 C  sv   t dd}| |jd |jdd}| |jd tjddgddggtjd	}|j|dd}t	|dd}t
|| d S )
NrW   r(   r  r  r   rn   r   rf   ro   )r$   r   r  r  r  rv   rw   r   r  r+   r
   )r   celur  r   r   r   rD   rD   rE   test_eval_celu_load_op  s   
z-TestReferenceEvaluator.test_eval_celu_load_opc                 C  s   t jdd}| |jd d S )Nr  r  )r)   r  r   r  )r   r  rD   rD   rE   test_create_adam  s   z'TestReferenceEvaluator.test_create_adamc              	   C  s2  t dtjg d}t dtjg d}t dtjg d}t dtjg d}tdg ddgg dd	d	gd
d
gd}t|gd|||g|g}t|tddgd}t|}|d u rWd S t|dd}	| 	|	j
d t t|tgdd}
| 	|
j
d t t|dd}| 	|j
d t d\}}t|D ]}t|D ]}tjd	d	||ftjd}d|dd||f< tjdtjd}td
tdd d|ddd d d d f< tjdggggtjd}|d |||dd }|	d |||dd }t|| |
d |||dd }t|| |d |||dd }t|| qqd S )Nr   NNNNr   r   Wr.   r   r$  r   rn   rn   rn   rn   rn   rf   )rl   r~   r   rG  rW      r   F)	optimizedr   )r  r(  T)r   r   ro   r   rn   rn   r   r   	   r   r      )r   r   r   r   r   r   r   rd   r!   r   r  r.   ConvOptimizedrx   rv   rz   r   minimumr  r   rw   r   r
   )r   r   r   r   r$  r7  r   r`   sess1sess2sess3sess4sHsWri   r   r   r  got3got4rD   rD   rE   	test_conv  sP   0

z TestReferenceEvaluator.test_convc                 C  s*  t dtjg d}t dtjg d}t dtjg d}t dtjd g}t dtjd g}t dtjd g}t dtjd g}t d	tjd g}t d
tjd g}	tdg ddg}
t|
gd||||||||	g|g}t|tddgd}t|}|d u ryd S t	|}d\}}t
|D ]}t
|D ]}tjdd||ftjd}d|dd||f< | jd||de tjdtjd}d|ddd d d d f< |tjdgtjdtjdgtjd|tjdgtjdtjdgtjdtjdgtjdtdgtjd}|d |d }|d |d }t|| W d    n	1 sw   Y  | jd||dq tjdtjd}tdtdd d|ddd d d d f< |tjdgtjdtjdgtjd|tjdgtjdtjdgtjdtjdgtjdtdgtjd}|d |d }|d |d }t|| W d    n	1 sw   Y  | jd||de tjdtjd}d|ddd d d d f< |tjdgtjdtjd gtjd|tjd!gtjdtjd"gtjdtjd#gtjdtd gtjd}|d |d }|d |d }t|| W d    n	1 sw   Y  qqd S )$Nr   r#  wr   x_scalew_scaley_scalex_zero_pointw_zero_pointy_zero_pointQLinearConvr   r9  r<  r8  r:  r=  r;  r>  rG  rW   r'  r   r   r   rn   ro   r   r   1x1)r8  ri   r   r&  3x3r)  rf   r*  r+     칟>n?   gO Y@   {XZ?)r   r   UINT8r   r   r   r   r   rd   r!   rx   rv   rz   r  r   rw   r   r   r
   r.  r  r   )r   r   r8  r   r9  r:  r;  r<  r=  r>  r7  r   r`   r/  r0  r3  r4  ri   r   feedsr   r  rD   rD   rE   test_qlinearconv  s   

0

z'TestReferenceEvaluator.test_qlinearconvc                 C  s  t dtjg d}t dtjg d}t dtjg d}t dtjd g}t dtjd g}t dtjd g}t dtjd g}t d	tjd g}t d
tjd g}	tdg ddg}
t|
gd||||||||	g|g}t|tddgd}t|}| 	d t
jg dg dg dg dg dg dg dgt
jdd}t
jdgt
jd}t
jdgt
jd}t
jdgt
jdd}t
jdgt
jd}t
jd gt
jd}t
jd!gt
jd}t
jd"gt
jd}	||||||||	d}g d#g d$g d%g d&g d'g d(g d)ggg}|d |d }t|| W d    n	1 sw   Y  | 	d* t
jg dg dg dg dg dg dg dgt
jdd}t
jd+gt
jd}t
jdgt
jd}t
jg d,d-t
jd}d.|d/< d0|d1< t
jd+d2gt
jd}t
jd-d-gt
jd}t
jd!gt
jd}t
jd"gt
jd}	||||||||	d}g d3g d4g d5g d6g d7gg d8g d9g d:g d;g d<ggg}|d |d }t|| W d    n	1 sw   Y  | 	d= t
jdt
jd}t
jd+gt
jd}t
jdgt
jd}t
jg d,d-t
jd}t
jd+d2gd>d>ggt
jd}t
jd-d-gt
jd}t
jd!gt
jd}t
jd"gt
jd}	||||||||	d}| td? |d |d  W d    n	1 sRw   Y  W d    n	1 sbw   Y  | 	d@ t
jdt
jd}t
jd+gt
jd}t
jdgt
jd}t
jg d,d-t
jd}t
jg dAt
jd}t
jd-d-gt
jd}t
jd!gt
jd}t
jd"gt
jd}	||||||||	d}| tdB |d |d  W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S )CNr   r#  r8  r   r9  r:  r;  r<  r=  r>  r?  r@  rG  rW   r'  r   single_channel)rG                 :   )r   ;      _      r   @   )8            rQ        )            rT  rN  rT  )   '   |   M   P   f   +   )         S   )   (      )rG     \      *         ro   rn   rn   r   r   rE  rF  r   r&  gO\?rG  rH  {   )r   Q   ]   rj  4   W      )      r;     ~   rG     )   r%  rg  "   r{     Y   )   re  E   r  r;  ry  r;  )C         r_     rZ     )rD  rO              y   )r   e      r      k      multiple_output_channels      p?)rf   rn   r   r   rD     )r   r   rn   rf   rf   )rn   r   rn   r         `?)rG        rw  r]  )   rG     r     )   rG  rG  r^  rG  )   r           )            r  )r  X   r   r  /   )Z   r   !   r      )r   r;  r%  r  r   )r\  r  2   re  L   )0   r   o   J   c   fails_with_w_scale_2Drn   z(w_scale must be a scalar or a 1-D tensorfails_with_w_scale_wrong_length)r  r  rn   z+w_scale elements must match output channels)r   r   rI  r   r   r   r   r   r!   r   rv   rw   r  r   r   r   r
   fullrz   assertRaisesRegexru   )r   r   r8  r   r9  r:  r;  r<  r=  r>  r7  r   r`   r   rJ  r   r  rD   rD   rE   test_qlinearconv_w_scale_vectorx  sJ  
	0	7

 $z6TestReferenceEvaluator.test_qlinearconv_w_scale_vectorc                 C  s  t dtjg d}t dtjg d}t dtjg d}t dtjg d}tdddgdg|||d}	tddgd	g}
td
dd	gdg|||dd}tddgdg}tdddgdg}t|	|
|||gd||g||g}t|tddtddgd}t|	gd||g|g}t|tddgd}t|}zt	|}|d u rW d S W n t
y   d }Y nw d\}}t|}t|D ]}t|D ]}tjdd||ftjd}d|dd||f< tjddg|R tjd}tdt||d df d|ddd d d d f< |d ||d}|d ur|d ||dd }t|d  |  zt|d  |d   W q tyU } ztd| d| d | d!| d"|d  d#|d  |d }~ww qd S )$Nr   r#  r|  Y2r$  r.   rl   r   r~   ShaperT   Im2Colximexperimental)rl   r   r~   r[   Flattenwflatr   rG  rW   r'  rn   r   )r   r   ro   r   r   rf   r   r,  )r   r$  zDiscrepancies: pads=z, dilations=z
, strides=z, kernel_shape=r   
!=
)r   r   r   r   r   r   r   r   r!   rd   ImportErrorrv   r{   rx   rz   r   r.  r  r   r   r
   ravelr   )r   r}   rl   r   r~   r   r|  r  r$  r7  
node_shapenode_im	node_flatnode_gemr   r`   
graph_convonnx_model_convr   	sess_convr3  r4  nkerri   r   r  ort_resr   rD   rD   rE   common_test_im2col3  s   	

z)TestReferenceEvaluator.common_test_im2colc                 C  "   | j dg dddgddgd d S )Nrn   rn   rn   rn   rn   rf   rn   r  r  r   rD   rD   rE   test_im2col_1x1x     
z&TestReferenceEvaluator.test_im2col_1x1c                 C  r  )NrH  r  rn   r  r  r  rD   rD   rE   test_im2col_2x2}  r  z&TestReferenceEvaluator.test_im2col_2x2c                 C  r  )NrA  r  rn   r  r  r  rD   rD   rE   test_im2col_3x3  r  z&TestReferenceEvaluator.test_im2col_3x3c                 C  r  )NrA  )r   rn   rf   r   rn   r  r  r  rD   rD   rE   test_im2col_3x3_pads  r  z+TestReferenceEvaluator.test_im2col_3x3_padsc                 C  s"   | j dg dddgddgd d S )NrA  )r   rn   rn   rn   rn   rf   r  r  r  rD   rD   rE   test_im2col_3x3_strides  r  z.TestReferenceEvaluator.test_im2col_3x3_stridesc                 C  r  )Nr   r   r  rn   r  r  r  rD   rD   rE   test_im2col_5x5  r  z&TestReferenceEvaluator.test_im2col_5x5c                 C  sP  dd l }tdtjg d}tdtjg d}tdtjd g}tdtjd g}tdg ddgg d	d
d
gd
d
gd}t|gd|||g|g}t|tddgd}t	|}	t
g dg dg dg dg dggt
j}t
ddgt
j}
t
d
dgt
j}|jjt|
|d}|	d |||
d}||| }t||d  d S )Nr   r   NNNr   Ir   Col2Imr   r  r   r   r   r   r   rn   r  rG  rW   r'  r   )r         @      &@      0@      5@)r        @      (@      1@      6@)r         @      *@g      2@      7@)r        "@      ,@      3@      8@)r  r        .@g      4@g      9@r   )output_sizer   r   r   r  )rK   r   r   r   r(  r   r   r   r   r!   rv   rw   r  r   r)  nnFoldrq   r   
from_numpynumpyr
   )r   rK   r   r   ISBSr7  r   r`   r   image_shapeblock_shapefoldr  rt  rD   rD   rE   test_col2im  sB   
z"TestReferenceEvaluator.test_col2imc              	   C  sf  dd l }tdtjg d}tdtjg d}	tdtjd g}
tdtjd g}tdg ddg|||d	}t|gd
||
|g|	g}t|tddgd}t	|}|j
jt|t|t|t|t|d}t|}t|D ]E}t|D ]>}tjd||ftjd}d|d||f< tj|tjd}tj|tjd}||| }|d |||d}t||d  qqqkd S )Nr   r   r  r   r  r   r  r  r  rG  rW   r'  r   )r  r   dilationpaddingstridern   ro   r   r  )rK   r   r   r   r(  r   r   r   r   r!   r  r  rq   r   rv   r{   rx   rz   r   rw   r)  r  r  r   r
   )r   sizer  r  rl   r   r~   rK   r   r   r  r  r7  r   r`   r   r  r  ri   r   i_shapeb_shapert  r  rD   rD   rE   common_test_col2im  sF   
z)TestReferenceEvaluator.common_test_col2imc                 C  &   | j dddg dddgddgd d S )Nr   r   r   rf   r   r  rn   r  r  r  rD   rD   rE   test_col2im_2x3     
z&TestReferenceEvaluator.test_col2im_2x3c                 C  r  )N   r  r  r&  rn   r  r  r  rD   rD   rE   test_col2im_2x3_pads  r	  z+TestReferenceEvaluator.test_col2im_2x3_padsc           	      C  s   t jddgt jd}d|d d< t ddgdt ddgt g d	t ddgf\}}}}}t||||||}t||||||}t|| d S )
Nr   r
  ro   r   r   r   r  rn   r&  )rv   rz   r   rw   r,   r-   r
   )	r   r|   r  r}   r~   rl   r  r1r2rD   rD   rE   test_col2im_2d  s   z%TestReferenceEvaluator.test_col2im_2dc              
   C  s,  t dtjg d}t dtjg d}t dtjdg}t dtjg d}tdg d	dgdd
d
gddgg dd
d
gd}t|gd|||g|g}t|tddgd}td	d
tjtjg dg dg dggg dg dg dggg dg dg dggg dg dg dgggtjdtjg d tjdd	}d!|d d d < |d }|d }|d }tjd|jd}t|jd! D ]C}	tdD ]<}
||	|	d
 |
|
d
 f }||
 }t|d"d
d
gg dd
d
g}t|| }|||
  }|||	|
d d d d f< qqt|}|d |}t||d! d#d$ d S )%Nr   )rf   r   r   r   r$  )r   rn   r   r   r   r   r   r.   r%  rn   r   r&  groupr~   r}   rl   r   rG  rW   r'  r   i   )g    ޚg   ]?g    ~n)g   `cKg   @A?g   ,?)g   $w?g   ʩ?g   @)g   ug    ?g   )g    ?g    ?g   )g   
?g   
g   @V?)g    ?g   Jg     )g   ¿g   i_g   ?)g    c?g   ?g   r)g    sg   g    N{)g   @ʲ?g   m?g    @?)g   X?g    Ug   `Ζro   )g   @[¿g   &g   (g   `䠾r   rA  h㈵>atol)r   r   r   r   r   r   r   rv   r  r   r  r   rw   r   rp   rx   rT   r   matmulr  r!   r   r
   )r   r   r$  r   r   r7  r   r`   rJ  r   rG  r   r8  c2mulref1got1rD   rD   rE   test_conv_im2col_group4  s~   
NQ]z.TestReferenceEvaluator.test_conv_im2col_group4c              
   C  sH  t dtjg d}t dtjg d}t dtjdg}t dtjg d}td	g d
dgdddgddgg dddgd}t|gd|||g|g}t|tddgd}td	d
tjd tjdtjdtjdtjdd
}d|d d< t|}	|	d |}
tjg dg dg dgg dg dg dgggtjd}t|
d | d S )Nr   rn   r   r   r   r$  rf   r   r   r   r   rf   r   r#  r.   r%  rn   r   r&  r  rG  rW   r'  r   l   ro   rf   )r   r   r   rn   )r  r  r  )r  r  r  )r  r  r  r   )r   r   r   r   r   r   r   rv   r  r   r  r   rz   r!   r   rw   r
   r   r   r$  r   r   r7  r   r`   rJ  r  r  r   rD   rD   rE   test_conv_strides  s>   

z(TestReferenceEvaluator.test_conv_stridesc           
      C  s   t dtjg d}t dtjg d}tddgdgddgg dddgd}t|gd	|g|g}t|td
dgd}dtdd d d 	d
tji}tjg dg dg dg dgggtjd}t|}|d |}	t|	d | d S )Nr   r#  r   MaxPoolr   r&  rf   r}   rl   r   rG  rW   r'  r   1   r   rv  )g      H@g     G@g     F@g     E@)g     D@g      D@g      C@g      B@)g      ;@g      :@r  r  )r  r        $@r  ro   r   r   r   r   r   r   r   r   rv   r  r   r  r   rw   r!   r   r
   
r   r   r   r7  r   r`   rJ  r   r  r  rD   rD   rE   test_max_pool_2d_1  s4   &
z)TestReferenceEvaluator.test_max_pool_2d_1c           
   
   C  s  t dtjg d}t dtjg d}tddgdgddgg dddgd}t|gd	|g|g}t|td
dgd}dtjg dg dg dg dg dg dg dgggtj	di}tjg dg dg dg dgggtj	d}t
|}|d |}	t|	d | d S )Nr   r#  r   r   r   r&  rf   r!  rG  rW   r'  r   )i  if  i  iB  i  i  r  )i  i  i  r   r}  rG  iF  )iB  B   i  ix  iT  i  i  )r  i  is  iL  i  iJ  i  )i  rz  i  rx  i'  i;  ih  )  i>  rx  i  i  i  i  )i|  i  K   iA  r(  i  i  ro   )g     X@g     @     @r*  )g     8@g     x@g     @     P@)     `@g     @r+  r+  )r,  g     @r,  g      @r   )r   r   r   r   r   r   r   rv   rw   r   r!   r   r
   r%  rD   rD   rE   test_max_pool_2d_2  sN   
z)TestReferenceEvaluator.test_max_pool_2d_2c                 C  s  t dtjd d g}t dtjd d g}t dtjd d g}t dtjd d g}tdg ddgddd	}t|gd
|||g|g}t|tddgd}tj	g dgtj
dt	ddggtj	ddggtj
dd}t|}	|	d |}
tj	g dgtj
d}t|
d | d S )Nr   r  Ur   ScatterElementsr   r  r.  rn   r   r  	reductionrG  rW   r'  r   )r   r  r  r  r  ro   皙?g @)r   r3  r  r  r  r   r   r   r   r(  r   r   r   r   rv   rw   r   r!   r   r
   r   r   Indr.  r   r7  r   r`   rJ  r  r  r   rD   rD   rE   test_scatter_elements	  s*   z,TestReferenceEvaluator.test_scatter_elementsc                 C  s   t dtjd d g}t dtjd d g}t dtjd g}t dtjd d g}tdg ddg}t|gd|||g|g}t|tdd	gd
}tj	ddggtj
dt	ddggtj	dgtj
dd}t|}	|	d |}
tj	ddggtj
d}t|
d | d S )Nr   r  r.  r   	ScatterNDr0  rG  rW   r'  r   r   r  ro   r   r  r4  r5  rD   rD   rE   test_scatternd 	  s&   z%TestReferenceEvaluator.test_scatterndc              
   C  s  t dtjg d}t dtjg d}t dtjd g}t dtjg d}tdg ddgddgd	d	gd
d
gg dddgd}t|gd|||g|g}t|tddgd}td	d
tjtd	d
tjtjg dtjdd}t|}	|	d |}
tjg dg dg dg dg dgggtjd}t|
d
 | |d  d
9  < d|d d< t|}	|	d |}
tjg dg dg dg dg dgggtjd}t|
d
 | d S )Nr   r#  r$  r   r   ConvTransposer%  rn   r   r   r&  )r~   r}   output_paddingrl   r   rG  rW   r'  r   r  )rn   r   r   r      )r   rn   r   r   r  ro   )i  i  i  iA  )id  i.  id/  i!  )i   i3  i4  i$  )il$  i8  i\:  i(  )i  i+  i,  i#  )r   r   r   r   )r   r  r   r   r$  r  rD   rD   rE   test_conv_transpose_2d8	  sd   
z-TestReferenceEvaluator.test_conv_transpose_2dc              	   C  sV  t dtjg d}t dtjg d}t dtjd g}t dtjg d}tdg ddgdd	d	gd
}t|gd|||g|g}t|tddgd}td	d
tjtd	d
tjtjg dtjdd}tjg dg dg dg dg dg dgg dg dg dg dg dg d gggtjd}	t|}
|
d |}t|d! |	 d S )"Nr   r#  r$  r   r   r:  r%  
SAME_UPPERrf   )auto_padr   rG  rW   r'  r   r*  r)  r;  )rn   rf   r   r   r  ro   )r   r   r   rn   rf   rf   )r   r   r   r   r   r  )r   r   r\  r   r
     )r*  r\  r<  r'  #      )r;  r<  r  rA  r  rh  )r;  r  3   r
  rS      )r   r   r*  r      rB  )r   r   r\  r%  &      )r<     T   rX     R   )$   rc  r  rz  t   A   )r  r  r}  ro  i$  r  )H   N   rQ  [      h   r   r$  )r   r   r$  r   r   r7  r   r`   rJ  r   r  r  rD   rD   rE   test_conv_transpose_2d_upperw	  sN   	z3TestReferenceEvaluator.test_conv_transpose_2d_upperc                 C  s  t dtjg d}t dtjd g}t dtjd g}t dtjg d}tdg ddg}t|gd	|||g|g}t|td
dgd}t	d
dtjtjdtjdtjdtjdd}|d }t|d }t|d }|d d }	|d jd | | d }
tjd|
|	dgtjd}t|
D ].}|| }|| | }tj|d||df }|d|	 }tj|j|jfdd|d|f< qt|}|d |}t|d | d S )Nsignalr  
frame_stepframe_lengthr   r#  STFT)rU  rV  rW   rW  rG  rW      r   rD  rn   rD  rn   r  ro   r'  )rU  rV  rW  rf   rn   r   r  )r   r   r   r(  r   r   r   r   rv   r  r   r  r   rw   r)  rQ   rT   r   rx   fftstackrealimagr!   r   r
   )r   rU  rV  rW  r   r7  r   r`   rJ  onesided_lengthnstftsr   ri   startstopcomplex_outc_outr  r  rD   rD   rE   	test_stft	  s<    z TestReferenceEvaluator.test_stftc                 C  s  t dtjg d}t dtjd g}t dtjd g}t dtjd g}t dtjg d}tdg d	dg}t|gd
||||g|g}t|tddgd}t	d
dtjtjdtjdddtdtj tj	dddtjd d   tjdtjdd	}	|	d }t|	d }|	d }t|	d }|d d }
d|jd |jd  d  }tjd||
dgtjd}t|D ]4}|| }|| | }tj|d||df | d|
 }|d|
 }tj|j|jfdd|d|f< qt|}|d |	}t|d | d S )NrU  r  rV  windowrW  r   r#  rX  )rU  rV  rg  rW  rG  rW   rY  r   rD  rZ  r  ro   r  rf   r   r'  rn   r[  )r   r   r   r(  r   r   r   r   rv   r  r   r  r   rw   r)  cospirQ   rT   r   rx   r\  r]  r^  r_  r!   r   r
   )r   rU  rV  rg  rW  r   r7  r   r`   rJ  r`  ra  r   ri   rb  rc  rd  re  r  r  rD   rD   rE   test_stft_with_window	  sJ   * z,TestReferenceEvaluator.test_stft_with_windowc                 C  s   t dtjg d}t dtjd d g}t dtjg d}t dtjd g}tdg ddgddd	d
d|d	}t|gd|||g|g}t|tddgdS )Nr   r#  roisr   r  RoiAlignr   rk  r  r   rf   r   output_half_pixel)output_heightoutput_widthsampling_ratiospatial_scalecoordinate_transformation_modemoderG  rW   rY  r   )r   r   r   r(  r   r   r   r   )r   rt  r   rk  r   r  r7  r   rD   rD   rE   get_roi_align_model
  s"   z*TestReferenceEvaluator.get_roi_align_modelc                 C  sp   |  |}t \}}}|||d}t|}|d u rd S |d |}t|}	|	d |}
t|
d |d dd d S )Nrm  r   r  r  )ru  r   rd   r   r!   r
   )r   rt  r`   r   batch_indicesrk  rJ  r   r   r  r  rD   rD   rE   common_test_roi_align"
  s   
z,TestReferenceEvaluator.common_test_roi_alignc                 C  sr   | j dd | d W d    n1 sw   Y  | j dd | d W d    d S 1 s2w   Y  d S )Navgrt  r   )r   rw  r  rD   rD   rE   test_roi_align.
  s   "z%TestReferenceEvaluator.test_roi_alignc                 C  s   dd l }ddlm} | |}t|}t \}}}|d |||d}	|dddd}
|
||||g}t|	d |dd	 d S )
Nr   )RoIAlignrm  r  r   rf   )rr  rq  r  r  )	rK   torchvision.opsr{  ru  r!   r   r   r  r
   )r   rt  rK   r{  r`   r   r   rv  rk  r  r   r   rD   rD   rE   common_test_roi_align_torch6
  s   
z2TestReferenceEvaluator.common_test_roi_align_torchc                 C  s<   | j dd | d W d    d S 1 sw   Y  d S )Nrx  ry  )r   r}  r  rD   rD   rE   test_roi_align_torchC
  s   "z+TestReferenceEvaluator.test_roi_align_torchc                 C  s0  t dtjd g}t dtjd g}t dtjd g}t dtjd g}t dtjd g}tddgg ddd	}t|gd
|g||||g}t|tddgd}dtd	tj
i}	tjg dtj
dtjg dtj
dtjg dtj
dtjdgtj
dg}
t|}|d |	}tdD ]}tj|| |
|  qd S Nr   r|  r  Y3Y4Splitr|  r  r  r  r   )num_outputsrG  rW   r;  r   r   r   rn   rf   ro   r  )r   r   r  r*  )r   r   r   r   r   r   r   rv   r  r  r   rw   r!   r   rx   testingassert_equalr   r   r|  r  r  r  r7  r   r`   rJ  r   r  r  ri   rD   rD   rE   
test_splitL
  s&   z!TestReferenceEvaluator.test_splitc                 C  s>  t dtjd g}t dtjd g}t dtjd g}t dtjd g}t dtjd g}tdddgg d}t|gd	|g||||g}t|td
dgd}td	tj
tjg dtjdd}	tjg dtj
dtjg dtj
dtjddgtj
dtjddgtj
dg}
t|}|d |	}tdD ]}t|| |
|  qd S )Nr   r|  r  r  r  r  splitr  rG  rW   r;  r   r   )r   r   rf   rf   ro   )r   r  r  r  r   r   r  r*  r   )r   r   r   r   r   r   r   rv   r  r  r   rw   r)  r!   r   rx   r
   r  rD   rD   rE   test_split_2d
  s*   z#TestReferenceEvaluator.test_split_2c                 C  s  t dtjd g}t dtjd g}t dtjd g}t dtjd g}t dtjd g}tddgg ddd	}t|gd
|g||||g}t|tddgd}dtd	tj
i}	tjg dtj
dtjg dtj
dtjg dtj
dtjdgtj
dg}
t|}|d |	}tdD ]}t|| |
|  qdtd	tj
i}	tjg dtj
dtjg dtj
dtjg dtj
dtjg tj
dg}
t|}|d |	}tdD ]}t|| |
|  qd S r  )r   r   r   r   r   r   r   rv   r  r  r   rw   r!   r   rx   r
   r  rD   rD   rE   test_split_num_outputs_4
  s:   z/TestReferenceEvaluator.test_split_num_outputs_4c           
      C     t dtjd d g}t dtjd g}tddgdgdd}t|gd|g|g}t|tddgd	}dt	d

dtji}t|}|d |}tjg dtjd
d}	| |d  |	  d S )Nr   r   ArgMinrn   r[  rG  rW   r;  r   r\  r   r   )r   r   r   ro   r   rn   r   r   r   r   r(  r   r   r   r   rv   r  r   r  r   r!   r   rw   r)  r   tolist
r   r   r   r7  r   r`   rJ  r  r  r   rD   rD   rE   test_argmin
     z"TestReferenceEvaluator.test_argminc           
      C  r  )Nr   r   ArgMaxrn   r[  rG  rW   r;  r   r\  r  r   r   r   ro   r  r   r  r  rD   rD   rE   test_argmax
  r  z"TestReferenceEvaluator.test_argmaxc                 C  s,  t dtjd d g}t dtjd g}t dtjd g}t dtjd g}t dtjd g}tdg ddgtd	ddgdgg}t|d
||||g|g}t|tddgd}tj	dggtj
dtj	dgtj
dtj	dgtj
dtj	dgtj
dd}	t|}
|
d |	}tj	dgtj
d}| |d  |  d S )Nr   startsendsr  r   r  )r   r  r  r  TSqueezerG  rW   r;  r   r   ro   rn   r   r   r   r(  r   r   r   r   rv   rw   r)  r!   r   r   r  )r   r   r  r  r  r   ro  r   r`   rJ  r  r  r   rD   rD   rE   test_slice_squeeze
  s&   z)TestReferenceEvaluator.test_slice_squeezec           
      C  s   t dtjd d g}t dtjd g}tddgdgdgdgdgdtddgdgdgd	g}t|d
|g|g}t|tddgd}dtj	dggtj
di}t|}|d |}tj	dgtj
d}	| |d  |	  d S )Nr   r   r  r  r   rn   )r  r  r  r  )r  rG  rW   r   r   ro   r  )
r   r   r   ro  r   r`   rJ  r  r  r   rD   rD   rE   test_slice_squeeze_6
  s   z+TestReferenceEvaluator.test_slice_squeeze_6c           	      C  sD  t dtjd d g}t dtjd d g}tddgdg}t|gd|g|g}t|tddgd}t| t|}|j	d }| 
|jjd	 |d dtjd
tjdid }| 
|jd | 
|d d t|tddgd}t| t|}|j	d }| 
|jjd |d dtjd
tjdid }| 
|jd | 
|d d d S )Nr   r   
ReduceMeanrG  rW   rY  r   r   ReduceMean_1r  ro   r  r   r   rn   r;  ReduceMean_18)r   r   r   r   r   r   r   r   r!   r  r   r  r	  r   rv   onesr   rT   )	r   r   r   r   r   r`   r   clsr  rD   rD   rE   test_onnxrt_reduce_mean
  s(   
 
 z.TestReferenceEvaluator.test_onnxrt_reduce_meanr"  c           	      C  s   g }t dtjd dgt dtjd gg}t dtjd d gt dtjd gg}| dkrE|ttjdgtjdd	d
 t|dd	gdgddd}nt|dgdgddgdd}tddgdgdd
tdddgdgdd
|tddgdgdd
g}t	|d|||}g }g d}|ttj|tj
dddd
 |ttjdgtjddd
 t dtjd dgg}t dtjd dgt dtjd dgg}tdddgd d!gd"|dd#td$d!gd%gd&ddgd'td(d%gd)gd*d
td+d)dgddgd,ddd-g}t	|d.|||}t|td/| gd0S )1Nnext_inr   nextnext_outscan_outr;  rn   ro   axis_redr   cdistdf_17_C0cdistdf_17_reduced0cdistdf_17_ReduceSumSquarer   )rR   r  )rR   r  r  r  cdistd_17_IdentitySubcdistdf_17_Subcdistdf_17_IdentityOnnxIdentity)Pg   @:?g   g   g    ?g   `?g   9?g   `Ƕٿg   Jg    Og   3?g   h	ܿg    sg   g   @`6?g   dg    Lg   ?g   @_ÿg   @Eg   g?g   lg   пg   q?g    v?g   C?g   3?g   `~?g   @Q?g   ˄g   ڞg   @׿g   `Kg    g    ?g   g    zӿg   @E?g   Xӿg   <?g   `ؿg    u(@g    ǿg    Eg    m?g   g    ?g   @?g   T?g   ?g   Tg   @Bʿg    M	?g    9?g   `Y@g    -?g   `Q?g   E?g   &7׿g    [?g   ]Lg    Xg    i?g   g   `ۜg   g   ?g   ;˿g   g   kZ?g   Z?g    &?g   `Ah?g    lg   @D?g    ?G?g    p?g   hg   @>?g   Wg   `1Dֿ)rB  r   
Sc_Scancstrf   
To_TopKcstrr  valuesindicesScanUU032UUUU033UUSc_Scan)rR   r  num_scan_inputs	TransposeTr_transposed0Tr_Transpose)rR   permSqrtSq_Y0Sq_SqrtTopKTo_TopK)rR   largestsorteddummyrW   r   )r   r   r   r5  r    rv   rw   r)  r   r   r   r   r(  r   r   )	rc   	reduce_opinitializersr   rg  node_reducero  r   
list_valuerD   rD   rE   _cdist_model
  s   
Rz#TestReferenceEvaluator._cdist_modelr  g]l,@g.s,&2@ro   rf   r   r   r   ReduceL1gQ@gK-J3@g%s}N@gZOc10@r;  r   r%  ReduceL2g&?gq#i?giE@g=@r   ReduceLogSumgd?gUU?g(?gQOIR?ReduceLogSumExpg?g?g9z%4~@g	Y;@	ReduceMaxgS?gsn`{?go@g@r  gύ]`?gJ?g%s}N @gZOc10@gNs	@ghi
@gXE!@gHg`y#@
ReduceProdrY  rc   rQ   c                   st  |\ }t ddt j}i }| | }t|}|d d|i}||d|f<  fdd|jd j	jD }	|	d j
}
t |	d jfd|
i}t||gd	}|d d|i}||d
|f< d}| D ]S\}}tt|t|ddD ]C\}}|j|jkrtd d| d|j d| d|j dt ||  }|dkrtd| d d| d| d| d| qsqdd S )Nr  )r   r   rr  r  c                   s   g | ]}|j j r|qS rD   )r  r	  r  )rh   r   r  rD   rE   rm      s    z9TestReferenceEvaluator.test_op_reduce.<locals>.<listcomp>r   	op_schemar  ref_clconstantTstrictzShape mismatch for z, :z != re   gư>zDiscrepancies (max=z) for r   r  )rv   r  r   r  r   r  r!   r   r  r  _schemars   r  itemszipreversedrT   r   r  r   )r   reduce_op_expectedrc   r   r   resultsmodelr   r  clschemanew_clbaselinekvr   r   diffrD   r  rE   test_op_reduce  s<   `

&&z%TestReferenceEvaluator.test_op_reduce)r%  )rY  )r;  	ref_opsetc                 C  s   t dtjg d}t dtjg d}tddgdgg}t|d|g|g}tjddddtj	}t
|td|gd	}t|}	|	d d|id
 }
t
|td|gd	}t|}|d d|id
 }| |
j|j t|
| d S )Nr   r#  r   MeanVarianceNormalizationrG  r   rn   rW   r   r   )r   r   r   r   r   rv   randomrandr  r   r   r   r!   r   r   rT   r
   )r   rc   r  r   r   ro  r   r   r`   r  r  ref_onnx_modelref_expectedr   rD   rD   rE   test_mvn8  s   zTestReferenceEvaluator.test_mvnc           
      C  s   dd }| }t jddgddggt jd}t jdd	ggt jd}t jd
dggt jd}t |||g}t|}|||d}|d |}	t||	d  d S )Nc                  S  s8  g } g }g }g }t  dd}g }tdddgdgddd	}|| d
d | D }tddddgdg||}|| |tdtjg  |tdtjg  |tdtjg  |tdtjg  tdddgdgddd	}| | tdddgdgddd	}| | dd | D }	t| d||}
t	|
|	|dS )Nrn   )rW   custom_domainConcatzx:0zx:1r__0r   rW   )r  r[   c                 S  &   g | ]\}}t ||d u rdn|qS r  r   rh   r[   r\   rD   rD   rE   rm   _      zZTestReferenceEvaluator.test_concat_in_a_function.<locals>.create_model.<locals>.<listcomp>r  concat_2I__0I__1I__2r__4r__3c                 S  r  r  r  r  rD   rD   rE   rm   z  r  numpyxr/  )
r   r   r5  r  r   r   r   DOUBLEr   r   )ro  r   rg  r0  r   	nodes_fctr7  opset_imports_fctr  r   r   rD   rD   rE   create_modelT  sJ   



zFTestReferenceEvaluator.test_concat_in_a_function.<locals>.create_modelr   r   r   r   ro   rn   rf   r   )r  r  r  r   )rv   rw   float64vstackr!   r   r
   )
r   r  r`   x1x2x3r   r  rJ  r  rD   rD   rE   test_concat_in_a_functionS  s   /z0TestReferenceEvaluator.test_concat_in_a_functionc              	   C  s   t dtjd g}t dtjd g}tttddgdgtjdgd|g|g}t|}t	dddtj
g}|d d|id	 }| |t	dddtj
gtjk  d S )
Nr   r   Castr  rG  Clp?Q}>ʈÿr  r   )r   r   r   STRINGr   r   r   r!   rv   rw   nanr   
assertTruer  str_allr   r   r   r  r  r|   r  rD   rD   rE   test_cast_float_to_string  s"   
"z0TestReferenceEvaluator.test_cast_float_to_stringc              	   C  s   t dtjd g}t dtjd g}tttddgdgtjdtddgdgtjdgd|g|g}t|}t	ddd	tj
g}|d d|id
 }t|t	ddd	tj
g d S )Nr   r   r  rE  r  rG  r  r  r  r   )r   r   r   r   r   r   r  r!   rv   rw   r  r   r
   r  rD   rD   rE   "test_cast_float_to_string_and_back  s    z9TestReferenceEvaluator.test_cast_float_to_string_and_backc                 C  sR  t dtjd }t dtjd }t dtjd }tdddgdgddg}tt|d||g|g}t|}t	
dd	t	j}t	jdt	jd
}|d ||d}	t	jddgddgddgggt	jd
t	jddgddgddgggt	jd
t	jddgddgddgggt	jd
gg}
| t|
d t|	d  t|
d |	d dd D ]	\}}t|| qd S )!Nr   r   rE  SplitToSequencerf   r[  rG  r;  rn   r   r   ro   rI  r  r   r  r  r  r  r  r  r  r  r  r  r  r  r#  r  r  r  r   Tr  r   r   r   r(  	UNDEFINEDr   r   r   r!   rv   r  r   r  r   rw   r)  r   r   rt   r  r
   r   r   r   rE  ro  r  r  r|   r  r  r   r   r   rD   rD   rE   test_split_to_sequence  s$   """z-TestReferenceEvaluator.test_split_to_sequencec                 C  sV  t dtjd }t dtjd }t dtjd }tdddgdgddg}tt|d||g|g}t|}t	
dd	t	j}t	jg d
t	jd}|d ||d}	t	jddgddgddgggt	jdt	jddgddgddgggt	jdt	jddgddgddgggt	jdgg}
| t|
d t|	d  t|
d |	d d d!D ]	\}}t|| qd S )"Nr   r   rE  r  rf   r[  rG  r;  r  rf   rf   rf   ro   rI  r  r   r  r  r  r  r  r  r  r  r  r  r  r  r#  r  r  r  r   Tr  r  r  rD   rD   rE   test_split_to_sequence_1d  s$   """z0TestReferenceEvaluator.test_split_to_sequence_1dc           
        s   t dtjd }t dtjd }tddgdgdddg}tt|d|g|g}t|}t	d
d	tj |d d i} fd
dt jd D g}| t|d t|d  t|d |d ddD ]	\}}	t||	 qed S )Nr   rE  r  rf   r   r  rG  r;  r  c                   s"   g | ]} d d d d |f qS rO   rD   rg   r|   rD   rE   rm     s   " zTTestReferenceEvaluator.test_split_to_sequence_nokeepdims_noinput.<locals>.<listcomp>Tr  )r   r   r   r  r   r   r   r!   rv   r  r   r  r   r   rx   rT   r   rt   r  r
   )
r   r   rE  ro  r  r  r  r   r   r   rD   r  rE   )test_split_to_sequence_nokeepdims_noinput  s   z@TestReferenceEvaluator.test_split_to_sequence_nokeepdims_noinputc                 C  s  t dtjd g}t dtjd g}t dtjd g}t dtjd g}t dtjd g}tttddgdgtjdtddgd	gtjdtd
g dgtdtjdgg ddtd
g dgtdtjdgg ddtddgdgtjdtdd	gdgtjdtddgdgtjdtddgdgtjdgd|g||||g}t	|}t
jg dt
jd}tj|tjt
j}	tj|tjt
j}
|d d|i}t|d |	 t|d |
 t|d |	 t|d |
 d S )Nr   F1F2F3F4r  f81r  f82rR  C1r   r   rn   rf   g?r  rh  C2rG  ro   r   rn   rf   r   )r   r   r   r   r   r   r  
FLOAT8E5M2r   r!   rv   rw   r   onnxnumpy_helpersaturate_cast	ml_dtypesfloat8_e4m3fnr  float8_e5m2r   r
   )r   r   r   r!  r"  r#  r  r  r|   	expected1	expected2r  rD   rD   rE   test_cast_float8  sd   
z'TestReferenceEvaluator.test_cast_float8z2.0z9assert_allclose does not support ml_dtypes in numpy < 2.0c           	   
   C  s   t dtjd g}t dtjd g}tttddgdgtjdtdddgdgdd	tddgdgtjdgd
|g|g}tjddgtj	d}|
tj}t|}|d d|i}t|d | G dd dt}t||gd}|d d|i}t|d | d S )Nr   r   r  f8r  CastLikef32r   )saturaterG  g    cAro   c                   @  s   e Zd ZdZdS )z>TestReferenceEvaluator.test_cast_like_float8.<locals>.CastLikerW   N)r	  r  r  r  rD   rD   rD   rE   r5  :  s    r  )r   r   r   r   r   r   FLOAT8E4M3FNUZrv   rw   r   r  r.  float8_e4m3fnuzr!   r   r
   r#   )	r   r   r   r  r|   r   r  r  r5  rD   rD   rE   test_cast_like_float8  s,   z,TestReferenceEvaluator.test_cast_like_float8c           
   	   C  s   t dtjd g}t dtjd g}t dtjd g}tttddgdgtjdtddgdgtjdgd|g||g}t|}t	j
g dt	jd}tj|tj}tj|tj}|d d|i}	| |	d	  |  | |	d
  |  d S )Nr   r   r!  r  r  rG  r'  ro   r   rn   )r   r   r   r  r*  r   r   r   r!   rv   rw   r   r+  r,  r-  r.  r/  r0  r   r   r  )
r   r   r   r!  r  r  r|   r1  r2  r  rD   rD   rE   test_cast_float8_outputA  s(   z.TestReferenceEvaluator.test_cast_float8_outputc           	      C  s  t jddddddddd	d
dddt jt j t jgt jd}tjt jdddddddddddddddt jgt jdtjt jdddddddddddddddt jgt jdtjt jdddddddddddddddt jgt jdtj	t jddddddddd ddddddt jgt jdi}d!d" }|
 D ]A\}}| j|d#/ ||}t|}|d d$|id% }t|| | |j|j | |j|j W d    n1 sw   Y  qd S )&Ng	?i`  i  iP  i0  i  iig-C6?g{Gz?i       j@ro   g      ?g      v@g      z@g      t@g      p@g      pg      Xr  g      ?g      |@g      |g      n@g      ng      ?g      x@g      ?g      @g      g?c                 S  st   t dtjd g}t dtjd g}tddgdg| d}tddgdgtjd}t||gd|g|g}t|}t| |S )Nr   r   r  r  r  r   )r   r   r   r   r   r   r   )r  r   r   r   r   r   r`   rD   rD   rE   model_cast_cast  s   zCTestReferenceEvaluator.test_float8_4_types.<locals>.model_cast_castr  r   r   )rv   rw   infr  r   r   r  r8  r*  FLOAT8E5M2FNUZr  r   r!   r   r
   r   rT   rp   )	r   r   r   r=  r  expectr`   r  r   rD   rD   rE   test_float8_4_typesX  s   W

z*TestReferenceEvaluator.test_float8_4_typesc                 C  s   t dtjd g}t dtjd g}tttddgdgtjdgd|g|g}t|}tj	g dtj
d}|tj}|d d|i}tj|d | d S )	Nr   r   r  r  rG  r   rn   rf   r<  r  ro   r   )r   r   r   BFLOAT16r   r   r   r!   rv   rw   r   r  r.  bfloat16r   r  assert_array_equalr   r   r   r  r  r|   r   r  rD   rD   rE   test_cast_bfloat16_output  s    
z0TestReferenceEvaluator.test_cast_bfloat16_outputc                 C     t dtjd g}t dtjd g}tttdg dgtdtjdgdgdtdg dgtdtjdgd	gdtd
g ddgtdddgdgddgd|g|g}t|}t	j
g dt	jd}t	j
g dt	jd}|d d|i}t|d | d S )Nr   r   rR  scalern   r  r(  rV  r  QuantizeLinearr   rI  rV  r  DequantizeLinearr   r[  rG  rB  ro   r   rn   rf   i  r  r   r   r   r   r   r   r   r  r!   rv   rw   r   r   r
   rF  rD   rD   rE   test_quantize_linear_e4m3  :   z0TestReferenceEvaluator.test_quantize_linear_e4m3c                 C  s   t dtjd g}t dtjd g}tttdg ddgtdddgdgdd	gd
|g|gtdtjdgdgtdtjdgdgg}t|}t	j
g dt	jd}t	j
g dt	jd}|d d|i}t|d | d S )Nr   r   rJ  rK  r  rL  rI  r   r[  rG  rn   r  rV  r  rB  ro   rM  rN  rF  rD   rD   rE   %test_quantize_linear_e4m3_initializer  s(   z<TestReferenceEvaluator.test_quantize_linear_e4m3_initializerc                 C  rH  )Nr   r   rR  rI  rn   r  r(  rV  r  rJ  rK  r  rL  r   r[  rG  rB  ro   )r   rn   rf   i  r  )r   r   r   r   r   r   r   r*  r!   rv   rw   r   r   r
   rF  rD   rD   rE   test_quantize_linear_e5m2#  rP  z0TestReferenceEvaluator.test_quantize_linear_e5m2c                 C     t dtjd g}t dtjd g}tttdg ddggd|g|gtdtjdgdgtd	tjdgd
gg}t|}t	j
g dt	jd}t	j
g dt	jd}|d d|i}t|d | d S )Nr   r   rJ  rK  rG  rI  rn   r  rV    )r  g      `r        333333@333333@g      @g         @     ro   )rT  i         i  r\  r]    r   r_  r   r   )r   r   r   UINT16r   r   r   r   r!   rv   rw   r   uint16r   r
   rF  rD   rD   rE   test_quantize_linear_uint16C  s2   z2TestReferenceEvaluator.test_quantize_linear_uint16c                 C  rS  )Nr   r   rJ  rK  rG  rI  rn   r  rV  r,  )r  g     r  rU  rV  rW  rX  rY  g    @g    g    @g    g     @g      rZ  r[  ro   )r,  r        i  rG  rc  rd  rT  irT   rT  re  rT  re  r   )r   r   r   INT16r   r   r   r   r!   rv   rw   r   int16r   r
   rF  rD   rD   rE   test_quantize_linear_int16  s2   z1TestReferenceEvaluator.test_quantize_linear_int16c                 C     t dtjd g}t dtjd g}tttdg ddgddgd|g|gtdtjd	gd
gtdtjd	gdgg}t|}t	j
g dt	jd}t	j
g dt	jd}|d d|i}t|d | d S )Nr   r   rL  rK  r   r[  rG  rI  rn   r  rV  rT  )i0u  iy  r^  i  ro   )g     g     r  g      }@)r   r   r`  r   r   r   r   r   r!   rv   rw   ra  r   r   r
   rF  rD   rD   rE   test_dequantize_linear_uint16  *   z4TestReferenceEvaluator.test_dequantize_linear_uint16c                 C  ri  )Nr   r   rL  rK  r   r[  rG  rI  rn   r  rV  i )iiii  ro   )g     @g     @g       g     @)r   r   rf  r   r   r   r   r   r!   rv   rw   rg  r   r   r
   rF  rD   rD   rE   test_dequantize_linear_int16  rk  z3TestReferenceEvaluator.test_dequantize_linear_int16r\  r   rn   r   r1  )r   r   r   r   )r   r   r   r   )r   r   r   r   )rn   r   r   r   )r   r  r   r  )r   r  r  r  r  r  )      ?r  rn   rA  )r   r   r  r   r  r   r   r   )rD  rL  rn     r  rG     r   )rW  D   rO  rF  rn  rs  rn  r  rm  r  r   r  r   )r   r   r   r   r   r  )r*  r\  r  )r   r   r   r   c                 C  s   t dtjd g}t dtjd g}tj|tjd}	tj|tjd}
tt	t
dg ddg||dgd|g|gtdtj|	j|	td	tj|	j|
g}t|}tj|tjd}|d urptj|tjd}|d d|i}t|d
 | d S | t |d d|i W d    d S 1 sw   Y  d S )Nr   r   ro   rJ  rK  r  
block_sizerG  rI  rV  r   )r   r   r   INT8rv   rw   r   int8r   r   r   r   rT   r!   r   r
   r   ru   r   r   rI  
zero_pointr  rr  r   r   r   
scale_datazp_datar  r  r|   r  rD   rD   rE   test_blocked_quantize_linear  sB   ;	"z3TestReferenceEvaluator.test_blocked_quantize_linear)r   rn   r   r   )r\  r   r  r
  )rn  -   r  r'  )r   r   rf   r   )r*  r\  rB  r  )rA  rn  6   r  r   r  r  )rn   rn   r   r   )r   r   r\  r\  )r*  r*  rk  rk  )rf   rn   r   )r   rf   r   r   r   r   r;  rk  )r'  r;  rB  r   r\  r%  rs  rz  )rD  r  rL  r@  rB  rk  r'  r  )r  r   rf   r   r   r*  c                 C  s   t dtjd g}t dtjd g}tj|tjd}	tj|tjd}
tt	t
dg ddg||dgd|g|gtdtj|	j|	td	tj|	j|
g}t|}tj|tjd}|d urptj|tjd}|d d|i}t|d
 | d S | t |d d|i W d    d S 1 sw   Y  d S )Nr   r   ro   rL  rK  rq  rG  rI  rV  r   )r   r   rs  r   rv   rw   r   rt  r   r   r   r   rT   r!   r   r
   r   ru   ru  rD   rD   rE   test_blocked_dequantize_linearW  sB   E	"z5TestReferenceEvaluator.test_blocked_dequantize_linearc              	   C  s   dd }d}d}d}d}t dtjg d}t d	tjd }td
dgd	g||||dg}tt|d|g|g}	t|	}
tj	
ddddtj}|
d d|i}||||||}| t|d t| d S )Nc           
      S  s   t dt j}t | jD ]4\}}}}	t| |td|t	|d d  t
d|t|d d  d ||	f d |||||	f< q| ||| |  |  S )N)r   r   r   r   r   rn   rf   r   )rv   rz   r  r   ndindexrT   r  r   mathfloorr   ceil)
r   r  betarz  r   
square_sumr   r   hr8  rD   rD   rE   	_expected  s    z2TestReferenceEvaluator.test_lrn.<locals>._expectedg-C6*?r  r  r   r   )r   r   r  r  rE  LRN)r  r  rz  r   rG  r   r   )r   r   r   r  r   r   r   r!   rv   r  r  r  r   r   r   rt   )r   r  r  r  rz  r   r   rE  ro  r  r  r|   r  r   rD   rD   rE   test_lrn  s   zTestReferenceEvaluator.test_lrnc                 C  s   t ddt jd t ddt jt jdt jdd}tddgdgddgdgd	d
}tdi ||}tdi ||}t	|| d S )Nr   )rn   rn   r   rn   r   r  r  ro   r%  NOTSETr  r~   r}   rl   r   r?  rD   
rv   r  r   r  r   rz   r   r/   r0   r
   r   rJ  rA   r   r  rD   rD   rE   test_conv_im2col_1d     z*TestReferenceEvaluator.test_conv_im2col_1dc                 C  s   t ddt jd t ddt jt jdt jdd}tddgdgd	d	gdgd
d}tdi ||}tdi ||}t	|| d S )Nr  )rf   r   r   rn   )r   r   r   r  ro   r%  r   r   r  r  rD   r  r  rD   rD   rE   test_conv_im2col_1d_pad0  r  z/TestReferenceEvaluator.test_conv_im2col_1d_pad0c                 C  s   t ddt jd t ddt jt jdt jdd}tdddgd	d	gg d
ddgdd}tdi ||}tdi ||}t	|| d S )N   )rn   rn   r   r  rn   r*  r)  r  ro   r%  r   r&  r  r  rD   r  r  rD   rD   rE   test_conv_im2col_2d  s"   z*TestReferenceEvaluator.test_conv_im2col_2dc                 C  s   t ddt jd dt ddt j t jdt jdd	}tdddgddgg d
ddgdd}tdi ||}tdi ||}t	|| d S )Nr  )rf   r   r   r   rn   rf   r;  )r   r   rn   rf   r  ro   r%  r  r  r  rD   r  r  rD   rD   rE   test_conv_im2col_2d_pad0  s"   z/TestReferenceEvaluator.test_conv_im2col_2d_pad0c                 C  s   t ddt jd dt ddt j t jdt jdd	}tdddgd
d
gddgd dd}tdi ||}tdi ||}t	|| d S )NrO  )rn   rn   r   r   rn   rf   r*  r)  r  ro   r%  r   
SAME_LOWER)r  r~   r}   r   rl   r?  rD   r  r  rD   rD   rE   test_conv_im2col_2d_autopad.  s   z2TestReferenceEvaluator.test_conv_im2col_2d_autopadc                 C  s   t ddt jd t ddt jt jdt jdd}tdg d	g d
g dg d	dd}tdi ||}tdi ||}t	|| d S )Ni  )rn   rn   r   r   r%  rn   r<  )rn   rn   r   r   r   r  ro   r%  rn   rn   rn   r  )rn   rn   rn   rn   rn   rn   r  r  rD   r  r  rD   rD   rE   test_conv_im2col_3d@  s&   z*TestReferenceEvaluator.test_conv_im2col_3dc                 C  s   t ddt jd t ddt jt jdt jdd}tdddgd	d	gg d
ddgdd}tdi ||}tdi ||}t	|| d S )Nr  r  rn   r{  r  r  ro   r%  r   r&  rf   r  r  rD   r  r  rD   rD   rE   test_conv_im2col_2d_stridesU     z2TestReferenceEvaluator.test_conv_im2col_2d_stridesc                 C  s   t ddt jd t ddt jt jdt jdd}tdd	dgd
d
gg dd	d	gdd}tdi ||}tdi ||}t	|| d S )Nr  r  rn   r{  r  r  ro   r%  rf   r   r&  r  r  rD   r  r  rD   rD   rE   test_conv_im2col_2d_dilationsg  r  z4TestReferenceEvaluator.test_conv_im2col_2d_dilations))r  )r  )r  )r  )r  )r  r"  c           
      C  s   t dtjd }t dtjd }tddtj}t|dgdgddg}t	t
|d|g|g}t|}|d d|i}|d }	| |	tj | |	jd d S )	Nr   r   r   )rn   r   rf   r   r$  rG  rD   )r   r   r   rv   r  r   r  r   r   r   r   r!   r   r   ndarrayr   rT   )
r   opr   r   r|   ro  r  r  r  rrD   rD   rE   test_reduce_op_no_axisy  s   z-TestReferenceEvaluator.test_reduce_op_no_axis)r  r  r   r   )r   )r   c                 C  sH  t dtjd }t dtjd }t dtjd }t dtjd }tjdgtjd}tdg ddgd	d
}tt	|gd|||g|g}t
|}	tjdgtjdd| }
tjddg| tjd}|	d |
||dd }| |jd|  | |jtj tddg|tj}|	d |
||dd }| |jd|  | |jtj d S )Nr   PVr   r   ro   Pad)r   r  r  r  )r   rg  rt  rG  rn   r  r   r  r   r   )r   )r   r   r   r(  rv   rw   r   r   r   r   r!   r   r)  r   r   rT   rp   repeatr  )r   dimr   r  r  r   rh  r7  r  r  r   pr  rD   rD   rE   test_pad  s"   zTestReferenceEvaluator.test_padc              
   C  s   t dtjd }t dtjd }tddgdgdtddgdgtdtjdgdgd	g}tt|d
|g|g}t|}t	j
dt	jd}|d d|id }| |jd | |jt	j tt	j
dt	jd| d S )Nr   r   r  rT   r  ConstantOfShaperh  rn   rf  rG  ro   r   rD   )r   r   r   r   r   r`  r   r   r!   rv   rw   r   r   r   rT   rp   ra  r
   r   r   r   ro  r  r  r   r  rD   rD   rE   test_constant_of_shape  s"   	z-TestReferenceEvaluator.test_constant_of_shapec                 C  s   t dtjd }t dtjd }tdg dgtdtjdgdgdtd	dgd
gdtdd
gdgtdtjdgdgdtdddgdgg}tt|d|g|g}t	|}t
jdt
jd}|d d|id }| |jd | |jt
j tt
jdt
jd| d S )Nr   r   rR  liker   rn   rf   r(  r  rT   r  r  r  rh  rf  r5  rG  ro   r   rD   )r   r   r   r   r   r`  r(  r   r   r!   rv   rw   r   r   r   rT   rp   ra  r
   r  rD   rD   rE   test_constant_of_shape_castlike  s0   z6TestReferenceEvaluator.test_constant_of_shape_castlikec                 C  sB  dt jg dg dg dg dg dgt jdi}t jg dg d	g d
g dg dgt jdt jdt jdt jdt jdg}tdtjd }tdtjd }tdtjd }tdtjd }tddgg dg}t	t
|d|g|||gtdt d gd}t|}	|	d |}
| t|
d tdddD ]}t|
| ||  qd S )Nr   )dg6`gQs[ؿgq2 d!?ggc%^?gQF
?g2o?g?gܝ>G?gG@b?gpA?g/ʿg`Y`?g??g@@?gߥ.gt?>P?g<0?ge~% 9g"?gf<߉xgs nY?gA?gOd* ǼgɀqgZ9?g} ixg`?g«kgY Ȧgd?JKg>Y?g LQ?gu@gc0@"?gZֈɾ?g.ykj?g}Ͽh?g2_ѿgg``m?gVHϿgO?g ?g2 vE?g~ۺg?Ͽg:h T?g(ۿ㣿gp}L?g& ׿gӿg֟?gj@@[?g(SHcÿgR,^Rgͼӟ(ɷ?g'U?gם՟P?g2ÿgk{?g; ȿgW^U0?gn ޒgGgĿg3ſ2ѿgT) 8?g^ҿgSQg=vpp?gS_ſgxпgK/Zeg)4$w?g_!@r?g8z?g0(Ngグg|p=fugoҿgN ?gPL2 ?gɗ֓?gtY?g؁?g"cٿF?ge] d`?gut~g/!?g:'?gS*gH?g=T@Au1?gH{?g_@?ggi- BI?gU?q?gHT`EϿgi4-ٿg" ?gʃ)dg9Ͽg3ֿg3d?gP>g@Ӫg- ,Ggk?g d;mgKT?g8?giz_?gဉDοg0+?g ¬?gO?gy_?g%D?g&j'?g+[gu?gv `˿g	?m?g{A>?gggCv?g4.&?g#?g>ġ@?gTg xg"Ug@F?gZ 7?g$s4`?g?u~ؿg~`ӽ?g?sZgs#|?gZ&!?g|Zzɿg(Ϳ?g^stS$ſg+q9gL>?gUo?gR^Igɿg?V??g蒷?g8-?g_ֿg@}\οgcg)?gĿgOɿg9Z?gWK v]?g>@{?gsVg14 @[n?gEc@ѿgVAg;?r?gj &;g#% ÿg:`i?gB@oƿgf_Xjg=?gHCſgk @]ſgG5?g	. ?gr{?g*8⮤g3Z  ,Zgä?g{`b?gjxvԿg^?g1?g?g 7gzPl߅h?g|?gdy1?g$ 9¿gtՉ?g0_?ge߿gx?X?gI??gG]t?gߛgfϾ"Z?g&@?g̅vʿgҿ@ſg?	?gyW?)dg?FgEFI@3Կg?gZa}gѠ 
:?g5 g)?g~ ?g69q?g?g6XCg|8ҿg?Ԫοg  [0?g.,?g_gԻ?g՟E:g@Ŀg@T?g|iͿgf k̿gE& ?g'@gu^ eŮgP{?gSh?5og=6% ?gg]~:gv!g MgB	 |?g6`r?g]ſgB7?g%Q?g7ف?gg g))L?g*- Jӿg9;ґ?gCr?g>Yͱ?g(?KĿgu gSV?gޟNZ?gE?g?_Sݿg|k`HѿgRS?gۨe?g#@ſg4׶gդϟ?g!5n?g]y ?g-?g}a ?g7_1ԿgrXƿg(;?gQ㿠Kg[#g9*X_4V?go@¿g<~?	ѿgVH?g`2J¿g}3?gڒ ?g.K?ÓgGd?gӕ`?g#K@+g1 ߷?g!?gב ǿg\!>?gmCl7\?gCn~W?g9hgW?2L?gM?gXp?go, 
ǿgs̠?gFė 1?g?ݿg?gTԄ?g	 %ag"d?gg=!p?g?qgAEȿg
?g9c"Vպ?g{wXk?)dgƣg-@ɧӿgˬ?0)?gcſgc fgR˷䤿g	 ?gu9e?glû?g!H?g1?j?gDXԿg7Z??g߄ gs?g%`g%(hg f?g)1ƿg0c@?gU
g`?֠9g=?Nn?gle lgᩱg0@}gK>?gw4򟌆?gWkByEgt??gCU?g2 v?g邀?g; ۿgW\md?g^?ս?gmg,@?gT }ĿgN?gGb Dտgu?gg4\m?gVKן?g7$@g?¿gE?gfFS?g~P?gBMӿgTzտg(?g?Ԏ?g2~>͌?g}8_g$9 ?g?gﲿf?gÑ!`zg@\ F?g~@eǿg8EU?gsC Y?gg ĿgP?go	 
geʿg_ ?g\u8hg Cɿgz]ڈgu% ?gyd*?gf* xq?gZ߿ǥ?g1_9?g^?gQÄɰͿg!`Tg?gԓͧgs)@ig?grJ?gB?gIzgq"?gt:꾿gJd`k?gu=>R?g gr?gu]?gc?gTyf?gH*?gf?g1g&o}пgG`?gxKu?)dgu@?gnO~ҿgJ_!g9 kg +g-nt?g[?g _ÿg2Z?n?g_S`?gA 0cu?gwsRֿgܣgaS˭?gI?g-a`bпg(???g]["=gLֿ#~ѿgZG`?gvϿVڿge?ꖿg|?g2MԿg^;?gX_?g3o-߱?gFV߼?g9ǿgqM&gj,??g:Y+?g?g&@.g(䟑w?gl_{gΧr ?g-f??g g] (?g]v	 Oݿgj`PKӿg$ߏ?gyB@BX?g*gS>ϿgY?geƿg4@?g,P &ݿgz`ҿg	~?gf?	?g?1gq^{пg?g)`$?gHo}?gqyF?g> G?g87@Eɿgɷg	_Ng^E`g)˿gkd`?gjF_g}пgzη?gL@ĿgSؿg>&`e<?gB`h٤?g> gWmN?gb_gtߏvgA gb9?տg7B篿g?g '-\?g>+Ʃ?gzi_`?gbP?gHWo?g!/gw?gEf@
?ge$5g@?gW ?gh<g; ?g g?gbj@ÿg#@`5g%`|?gR`Rro   )drV  rO  rQ  rD     rZ  r  r  r_     rQ  i   rM           r  r        w   r     z   r        r^  r  r  rV        r         r  r  ry     a   r  r  rG  rd  r  r|  rJ     r  U         s   rd  r     r  r  r  r        q   rq  r~  r        r  ^      r  r  rt  r  r  r  r  r  r  r     r     r  r     r  r;     r  r  r  r  ra  r  F   r  rr  )db   r  r]  x   r  r  r  r        r  r     rZ     r     rZ  rd  r  g   rt  r  ri     r  r  ra  rJ  }   r     r  rO  r  r     r  j   r  p   rw  r  r  r  r  r`  r  r  re  r  ro  rM  r  r  r  r  r[  r  r  rQ  r  r  rF  r     n   ri  r  r  r  rM  r  rM  rJ  rD  r  rM  rI  r  rZ  r  r  rQ  r  r  u   r|  r  rC  r  r  r  rF  r     rS  r  r  rQ  )dr  rl  r  r  r  r  r`  r  r  r  r  r  r  r  r  ro  r  r  r  r  r  rg  r  rw  rF     rr  r  r  ri  rr        r  r  r  r  r  rJ  r  r{  r        r     r  r     rR  ry  r  r  r  r  r  rt  r  r  r  rI  r  r  r  r  r  r  r  r  rM  r  r     r  r  r  r  r  r  rp  r  rN  r  r  rR  r  r  r  r  rS  r  r  rr  r  r  r  r  r  rN  r  )drJ  V   r  r  r  r  r  r  r]  r  r  r  rQ  rD  r  r  rr  r  r  r  r  r  r~  r  r  r  r     rd     r  r  r  >   rp  r  r  r  r  r  rx  r_  r  r  r  rM  r  rt  r|  r  rf  r  r  r  ri  r     r  r  r  m   rZ  r  r  r     r  rS  r  v   r  rr  r  rj  r  r  r  r  r  rD  r  ro  r  rN  r  r  r  r  rM  rH  r  r  r  r  r  r  r  rU  r  rt  )dr  r  r  r  ro  r  r_  r  r  r~  r  re  rJ  rp  r  `   r  ro  rq  r  r  r  r  rK  r  r_  r  r  r  r  r  r     rL  r  r  r_  r  r  r  rR  r{  r  r  rd  r  r  r  r}  9   r  r  r  ri  rU  r`  r  r  r  r  r  r  rr  r  r  r  r  r  r  r  G   r  r  r  r  r  r  rJ  rf  r     r  r  r_  r}  r  r  r  rZ  r"  r  r  r  r  rJ  r  r  r  r  ro  g    v?r  r   rI  zpDynamicQuantizeLinear)r   rI  r  rG  rW   rn   r   r   rf   r   )rv   rw   r   r  r   r   r   rI  r   r   r   r   r   r!   r   r   rt   rx   r
   )r   rJ  r   r   r   ScaleZpro  r  r  r  ri   rD   rD   rE   test_dynamic_quantize_linear  s   ffff               ffff                       z3TestReferenceEvaluator.test_dynamic_quantize_linearabcdefz.comz.netzabc.comzdef.netr  )catdogsnakes)catsdogssnakesr  )r  r  r  rD   )r      ßr   )aau   ßßyyc                 C  s   t dtjd }t dtjd }t dtjd }tdddgdgd}tt|gd||g|g}	t|	}
|
d t	|t	|d^}}tj
|| | |jjdd	h | |j| d S )
Nr   r   r   StringConcatr  rG  )r   r   Or.  )r   r   r  r   r   r   r!   r   rv   rw   r  rE  assertInrp   kindr   rT   )r   r   r   r   expected_shaper   r   r   r7  r  r  r  r   rD   rD   rE   test_string_concat  s   	"z)TestReferenceEvaluator.test_string_concatz1,2,34,5,6,)123)456)1,4,6rW   r  rW   r  r  )rf   rf   rn   )r  r  r  z5,6)rn   rf   rf   )r  r  r  )r  rW   rW   )r  r  rW   )rf   rf   r   )zhello world !z  hello   world !z hello world   ! )helloworld!r  z	o-n-n--x-z	o-n----nx-)or   r   rW   r   rW   )r  r   rW   rW   rW   nx r  c                 C  s   t dtjd }t dtjd }t dtjd }tddgddg||d}	tt|	gd|g||g}
t|
}tj	|t
d}|d d|i^}}}tj|tj	|t
d tj|tj	|tjd d S )Nr   Splits	MaxSplitsStringSplit)r   rg  	delimitermaxsplitrG  ro   )r   r   r  INT32r   r   r   r!   rv   rw   objectr   r  rE  r)  )r   r   r"  r#  expected_splitexpected_num_splitsr   r  r   r7  r  r  r  
num_splitsr   rD   rD   rE   test_string_split  s$   Oz(TestReferenceEvaluator.test_string_splitc                 C  s  t dg ddgd}t|gdtdtjd d gtdtjdgtd	tjdgtd
tjd d gtdtjdgtdtjdgtdtjdgtdtjdggtdtjd d gg}t|tddgdd}t|}t	
g dg dg}|d8 }|t	j}t	j
dgt	jd}t	j
dgt	jd}t	
g dg dg dg dg}|d8 }|t	j}t	j
dgt	jd}	t	j
dgt	jd}
t	j
dgt	jd}t	j
d gt	jd}|d t|||||	|
||d}t	jt	j
g d!g d"gt	jd|d#  d S )$NQLinearMatMul)r   a_scalea_zero_pointr   b_scaleb_zero_pointr;  r>  r   r  rG  r   r+  rn   r,  r   r-  r.  r;  r>  rW   rB  r*  r   r]   )r  r  r      )r   r  rG  rE  ri  gF%u{?ro   )r  rC     )r  rG  rG  )r   ri     )ri  rd  ru  g!u|?ig'?)rm  ir4  )rn   irB  r   )r   r   r   r   r   rs  r   r   r!   rv   rw   r  rt  r   r   r   r  rE  )r   r7  r   r`   r   r   r+  r,  r   r-  r.  r;  r>  r  rD   rD   rE   test_qlinearconv_int8s  sf   

" z,TestReferenceEvaluator.test_qlinearconv_int8)zwww.google.comzwww.facebook.comzwww.bbc.co.ukzwww\.[\w.-]+\.\bcom\b)TTF)Onnx
tensorflowNumpy)PytorchCythonnumbaz^[A-Z][a-z]*$)TFTr  )zaccount@gmail.comzaccount@hotmail.comz	not emailzaccount2@yahoo.comz,(\W|^)[\w.\-]{0,25}@(yahoo|gmail)\.com(\W|$))TFFTr  c                 C  s   t dtjd }t dtjd }tddgdg|d}tt|gd|g|g}t|}	|	d dt	
|i^}
}t	j|
| | |
jjd | |
j| d S )Nr   r   RegexFullMatchr   rg  patternrG  r   )r   r   r  ru  r   r   r   r!   r   rv   rw   r  rE  r   rp   r  rT   )r   r   r>  r   r  r   r   r7  r  r  r  r   rD   rD   rE   test_regex_full_match  s   z,TestReferenceEvaluator.test_regex_full_matchc                 C  s   t dtjd }t dtjd }tddgdgdd}tt|gd|g|g}t|}| t	 |
d dtdgi W d    d S 1 sEw   Y  d S )Nr   r   r<  zx)r=  rG  r   )r   r   r  ru  r   r   r   r!   r   ru   r   rv   rw   )r   r   r   r7  r  r  rD   rD   rE   test_regex_invalid_pattern  s   "z1TestReferenceEvaluator.test_regex_invalid_pattern)r   r         ?rf   ffffff
@r   rB  rn  )r   r   rf   rf   r   r   r   r   )r   r   rA  rf   rB  r   rn  )r   r   rf   rf   r   r   r   )      -r   rA  rf   rB  r   rB  )r   r   r   rf   rf   rf   rf   rf   )rC  rD  r   rA  rf   rB  r   )r   r   r   rf   rf   rf   rf   c           	      C     t dtjd g}t dtjd g}tttdg dgtdtjdgdgdtdg dgtd|dgd	gdtd
g ddgtdddgdgd	dgd|g|g}t|}|d dt	
|i}t|d	 | d S Nr   r   rR  rI  rn   r  r(  rV  r   rJ  rK  r  rL  r[  rG  r   r   r   r   r   r   r   r!   r   rv   asarrayr
   	r   qtyper|   r   r   r   r  r  r  rD   rD   rE   test_quantize_linear_int2  6   z0TestReferenceEvaluator.test_quantize_linear_int2c           
      C     t d|d g}t d|d g}tttddgdg|dgd|g|g}t|}tjg dtj	|d}|
tj	|}|d d|i}	| |	d  |  d S 	Nr   r   r  r  rG  )r   rn   g333333@g@r   r   ro   r   r   r   r   r   r!   rv   rw   r+  helpertensor_dtype_to_np_dtyper  r   r   r  
r   	cast_fromcast_tor   r   r  r  r|   r   r  rD   rD   rE   test_cast_int2_output  &   

z,TestReferenceEvaluator.test_cast_int2_outputrS  TensorProto.DataTyperT  c           
      C     t d|d g}t d|d g}tttddgdgtjdgd|g|g}t|}tjt	dtj
d}|tj|}|d d|i}	tj|	d | d S )	Nr   r   r  r  rG  rf   ro   r   r   r   r   r   r   r   r!   rv   rw   rx   r   r  r+  rP  rQ  r   r  rE  rR  rD   rD   rE   test_cast_int2_input8      	
z+TestReferenceEvaluator.test_cast_int2_input)r   r   rf   rf   r   r   rB  rH  )r   r   rf   rf   r   r   rH  )r1  r   rf   rf   r   r   ro  )r\  r1  r   rf   rf   r   r   c           	      C  rE  rF  rG  rI  rD   rD   rE   test_quantize_linear_int4S  rL  z0TestReferenceEvaluator.test_quantize_linear_int4c           
      C  rM  rN  rO  rR  rD   rD   rE   test_cast_int4_output  rV  z,TestReferenceEvaluator.test_cast_int4_outputc           
      C  rX  )	Nr   r   r  r  rG  r   ro   r   rY  rR  rD   rD   rE   test_cast_int4_input  r[  z+TestReferenceEvaluator.test_cast_int4_inputc           
      C  s  t dtjdg}t dtjdg}t dtjdg}tdddgdgtdg dgd	gd
dtdddgdgddgtddgd}tdddgdgtdg dgd	gddtdddgdgtddgdgttddgdgddgdg |gttdddgdgddgdg |gddgtddtddgd}tttddgdgddgd|g|gdtddtddg||gd }dtj	d!gtj
d"i}t|}|d |}	tjtj	d#gtj
d"|	d$  d S )%Nr   Nrt  thisfctaddinput2rR  r  r   CC0rS  rR   r   A1r   rW   r   r   r  rr  CCrF  rn  rX  r[   rT  rU  :IFrY  rZ  rR   rn   rp  r   r]   r   r0  r   ro   r   r   r   r   r   r   r   r   r   r   rv   rw   r   r!   r   r  r  
r   r   rt  rE  func_def_addfunc_defra  rJ  r  r   rD   rD   rE   'test_a_function_calling_a_function_once  sz   
!$z>TestReferenceEvaluator.test_a_function_calling_a_function_oncec           
      C  s  t dtjdg}t dtjdg}t dtjdg}tdddgdgtdg dgd	gd
dtdddgdgddgtddgd}tdddgdgtdg dgd	gddtdddgdgtddgdgttddgdgddgdg |gttdddgdgddgdg |gddgtddtddgd}tttddgdgddtddgdgddgd|g|gd tddtddg||gd!}dtj	d"gtj
d#i}t|}|d |}	tjtj	d$gtj
d#|	d%  d S )&Nr   r`  rt  ra  rb  rc  rR  r  r   rd  re  r   rf  r   rW   r   r   r  rr  rg  rF  rn  rX  rh  rT  rU  ri  rj  rn   ztmprp  r   rk  r   ro   r   rl  rm  rD   rD   rE   )test_a_function_calling_a_function_double  s|   
!	$z@TestReferenceEvaluator.test_a_function_calling_a_function_doublec           
      C  s  t dtjdg}t dtjdg}t dtjdg}tdddgdgtdg dgd	gd
dtdddgdgddgtddgd}tdddgdgtdg dgd	gddtdddgdgtddgdgttddgdgddgdg |gttdddgdgddgdg |gddgtddtddgd}tttddgdgddtddgdgddgd|g|gd tddtddg||gd!}G d"d# d#t}||}|j	
 D ]}	| |	| qd S )$Nr   r`  rt  ra  rb  rc  rR  r  r   rd  re  r   rf  r   rW   r   r   r  rr  rg  rF  rn  rX  rh  rT  rU  ri  rj  rn   rq  rp  r   rk  c                   @  r  )z[TestReferenceEvaluator.test_overload_reference_implementation.<locals>.MyReferenceEvaluatorNr  rD   rD   rD   rE   MyReferenceEvaluator  r  rt  )r   r   r   r   r   r   r   r   r!   
functions_r  r   )
r   r   rt  rE  rn  ro  ra  rt  r  r  rD   rD   rE   &test_overload_reference_implementation[  s~   
!	z=TestReferenceEvaluator.test_overload_reference_implementation))r  q=
ףp?)r8  rw  )r*  333333?)r?  rx  )r   r   )r   r   )FLOAT16gMb`?)rC  g{Gz?styper  r  c                 C  s  t t|}tttddgdg|dtddgdg|dtdddgdgtddgd	gtjdgd
tdtjg dtdtjg dgtd	tjg dgtddgdd}t|dd}t	
dd dt	j}t	
dd dt	j}t||d}|| }	|d |d }
t|
|	|d d S )Nr  r   Xcr  r   Ycr   ZcrE  r   r  rW   r;  r*  r/  r   r   r   rf   r   r   rI  r  )getattrr   r   r   r   r   r   r   r!   rv   r  r   r  r   r   r   r
   )r   rz  r  ityper  r  r   r   rJ  r   r  rD   rD   rE   test_add_custom_dtype  s0   

z,TestReferenceEvaluator.test_add_custom_dtype))r   )r   )ry  )rC  )r  )r8  )r*  )r?  )INT4)UINT4)INT2)UINT2c           
      C  s  t t|}tttddgdg|dtddgdg|dtdddgdgtddgd	gtjdgd
tdtjg dtdtjg dgtd	tjg dgtddgdd}t	|}t
dd dt
j}t
dd d dt
j}t||d}||k}|d |d }	t
j|	| d S )Nr  r   r{  r  r   r|  rO  r}  rE  r   r  rW   r;  r*  r/  r~  rI  r   )r  r   r   r   r   ru  r   r   r   r!   rv   r  r   r  r   r   r   r  r  )
r   rz  r  r  r  r   r   rJ  r   r  rD   rD   rE   test_cmp_custom_dtype  s0   

 z,TestReferenceEvaluator.test_cmp_custom_dtypec              
   C  s   t ttdg ddgdddgdtdtjd td	tjd td
tjd gtdtjd gtddgd}tj	dtj
dd}tjdggggtjd}tjdggggtj
d}tjg dtj
dd}t|}|d |||d}t|d | d S )Nr/  )r|   r  updatesrE  r   addr1  rR   r|   r  r  rW   r;  r   r'  ro   )rf   rf   rf   rf   r   rn   )rn   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   )r   r   r   r   r   r   r(  r   rv   rz   r   r   rw   r)  r!   r   r
   )r   r  r|   r  r  r   r  r  rD   rD   rE   test_scatter_elements_4d  s<   	

z/TestReferenceEvaluator.test_scatter_elements_4dc                 C  sP   |  d}t|}tjdtdd}|dgd|gid }| d|j d S )	Na  
        <
            ir_version: 8,
            opset_import: [ "" : 21 ]
        >
        preprocess (seq(float[X, Y]) images) => (float[N, 5, 5] preprocessed)
        {
            seq = SequenceMap<
                body=preprocess_single(float[X, Y] image) => (float[5, 5] resized)
                {
                    size = Constant<value=int64[2] {5, 5}>()

                    resized = Resize<
                        mode="linear",
                        axes=[0, 1]
                    >(image, , , size)
                }
            >(images)
            preprocessed = ConcatFromSequence<axis=0, new_axis=1>(seq)
        }
        )r   r   r   ro   preprocessedimagesr   )rn   r   r   )r   r!   rv   rz   rp   r   r   rT   )r   r  	evaluatorimageInrt  rD   rD   rE   test_sequence_axis  s   z)TestReferenceEvaluator.test_sequence_axisc              	   C  s   t ttddgdgddgdtdtjd gtdtjd gtddgd	}tj	d
d
tj}t|}|d d|i}| |j|d j d S )N	LeakyRelur   r   rA  r  rR   rW   r;  r   r   r   r   )r   r   r   r   r   r   r   rv   r  randnr  r  r!   r   r   rp   )r   r  r   r  r  rD   rD   rE   test_convert_ml_dtypes;  s   
	z-TestReferenceEvaluator.test_convert_ml_dtypesc                 C  s   t jdt jd}t|d | |jt j tt jdt j t j gddt j gg dg|jd| t j	dt jd}t|d | |jt j tt jddt j t j gdddt j gg dg|jd| d S )NrA  ro   r   rn   r  r  r  )
rv   r  float16r*   r   rp   r
   rw   r>  rz   )r   r   rD   rD   rE   test_apply_causalJ  s&   
(
(z(TestReferenceEvaluator.test_apply_causal)r   r   r   r   )FNr   r   r  )rc   rQ   r  )rc   rQ   r  rQ   )rS  rW  rT  rW  )rz  r   r  r  )r	  r  r  r   staticmethodr   r   r   r   r   r   r   r  r  r  r  r!  r#  r+  r:  r<  r=  r>  rC  rD  rL  rN  rP  rb  ry  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r!  r"  rI   r7  rK  r  r  r  r  r  r  r  r  rL   r  r  r  r  r  r  r  r&  r-  r7  r9  r=  rT  rf  rj  ru  rw  rz  r}  rN   r~  r  r  r  r  r  r  r  r  r  parameterizedexpand	itertoolsproductrv   rw   r  r   r)  r  r  r
  r  r  r  r  r  r3  r=   skipIfversion_utilsnumpy_older_thanr:  r;  rA  rG  rO  rQ  rR  rb  rh  rj  rl  r  r   rz   r  ry  dstackr|  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r)  r5  r?  r@  r   r  r  rK  r   ry  rU  rZ  r  r  r]  r^  r_  rp  rs  rv  r  r  r  r  r  r  rD   rD   rD   rE   r      s   8T
%/43	,37n}3 


*
\ <E
')

 (!2?215
' :
Z_#:3
 
  <D
	
	



	


	

8*
	
	:
&	




	

B)"        -$

  
GF	







NOK r   __main__rf   )	verbosity)r   r   r}   r   r~   r   rl   r   r   r   r   r   )c__doc__
__future__r   r:   r  r~  r=   
contextlibr   	functoolsr   ior   osr   textwrapr   typingr	   r.  r  rv   r  r  numpy.testingr
   r+  r   r   r   r   r   r   $onnx.backend.test.case.node.roialignr   onnx.checkerr   	onnx.defsr   onnx.helperr   r   r   r   r   r   r   r   r   r   r   r   onnx.numpy_helperr    onnx.referencer!   onnx.reference.op_runr"   r#   onnx.reference.opsr$   %onnx.reference.ops._op_common_indicesr%   r&   onnx.reference.ops._op_listr'   r(   3onnx.reference.ops.aionnx_preview_training._op_listr)   onnx.reference.ops.op_attentionr*   onnx.reference.ops.op_celur+   onnx.reference.ops.op_col2imr,   r-   onnx.reference.ops.op_convr.   r/   onnx.reference.ops_optimizedr-  .onnx.reference.ops_optimized.op_conv_optimizedr0   collections.abcr1   rQ   r2   r4   rI   rL   rN   rV   rd   r   r   TestCaser   r	  mainrD   rD   rD   rE   <module>   s    8



0                                               
;