o gТ╫iгmу@sЦddlZddlZddlmZddlZddlZddlmZddlZej а dбr.ddlZddДZej а dбrєz%test_compare_einops..й┌to_numpy)щщАrщй┌size┌float32· b [s...] czb ... c -> b c┌mean)┌ reductionzb ([s])... czb h w c -> b 1 1 czb (s [ds])... cщ)┌dszb (h h2) (w w2) c -> b h w c┌sum)r┌h2┌w2)rr)rщ z... [c1], [c1] c2 -> ... c2z... c1, c1 c2 -> ... c2)rrщ@zb [s...] c, [s...] s2 -> b s2 czb h w c, h w s2 -> b s2 cйrrrщ)rrrza b *z!a b c d, a b e -> a b ((c d) + e))┌einx┌types┌SimpleNamespace┌np┌random┌uniform┌astyperrr┌reducer┌dot┌einsum┌pack┌id)r!┌setupr ┌w┌yrrr ┌test_compare_einopssh ¤ ¤ ¤ ¤¤¤¤¤¤r0┌torchcCs╠ddl}tjddДdН}tаdddб}tаddб}ttа|d|б|аd ||б|Гttj ||dd Н|аd ||б|Гtаdddб}tаddб}ttа|d|б|аd ||б|Гttj ||dd Н|аd ||б|Гtаdddб}tаddб}ttа|d|б|аd||б|Гttj ||dd Н|аd||б|Гtаdddб}tаddб}ttа |d|б|аd||б|Гtаdddб}tаddб}ttа |d|б|аd||б|Гtаdddб}tаddб}ttа |d|б|аd||б|Гtаdб}tаddб}ttа||б|аd||б|Гtаdб}tаdбtj|j tjdНаtjб}t||d|d|df|аd||б|Гtаdб}tj|gdвd d Н\}}}|jd!|dd"d#Н\}} } t|||Гt|| |Гt|| |ГdS)$NrcSs|абабабSr)┌detach┌cpu┌numpyrrrr rQsz$test_compare_torch..r rrr)щrrz[_] ..., i ... -> i ...)┌dim)rr5rщza [_] ..., a i ... -> a i ...йrrr5rz#a b [_] ..., a b i ... -> a b i ...йr5·[_] ..., i -> i ...·a [_] ..., i -> a i ...·a b [_] ..., i -> a b i ...)r5щrz[_], ... -> ...йrrrйr5rй┌dtypeй.rй.r7й.r·[a...], b... [3] -> b...йrr5)rr щщ ·$a (b1 + b2 + b3) -> a b1, a b2, a b3r й┌b1┌b2)r!r"r#r1┌rand┌randintr┌gather┌get_at┌take_along_dim┌index_select┌take┌tensor┌shaper┌to┌int64┌splitr,)r!r-r ┌coords┌indices┌x1┌x2┌x3┌x1e┌x2e┌x3errr ┌test_compare_torchNsЮ¤¤¤¤¤¤¤¤¤ ¤ $¤ ra┌ tensorflow┌true┌TF_FORCE_GPU_ALLOW_GROWTHcCspddl}tjddДdН}tjаdб}tjjddtjdН}ttj||dd Н|а d ||б|Гtjаdб}tjjddtjdН}ttj||dd Н|а d ||б|Гtjаdб}tjjddtjdН}ttj||dd Н|а d||б|Гtjаdб}tjjddtjdН}ttj||dddН|а d||б|Гtjаdб}tjjddtjdН}ttj||dddН|а d||б|Гtjаdб}tа tjjdtjdНtа tа|бtjбtjб}ttа ||б|а d||б|Гt||d|d|df|а d||б|Гtjаdб}tа tjjdtjdНtа tа|бtjбtjб}ttа ||б|а d||б|Гtjаdб}tа tjjdtjdНtа tа|бtjбtjб}ttа ||б|а d||б|Гtjаdб}tа tjjdtjdНtа tа|бddЕtjбtjб}ttj ||ddН|а d ||б|Гtjаdб}tа tjjd!tjdНtа tа|бddЕtjбtjб}ttj ||ddН|а d"||б|ГdS)#NrcSs|абSr)r4rrrr r╕sz)test_compare_tensorflow..r r>r9r)┌maxvalrAй┌axisr:rr7r;rrr<)rr5)┌ batch_dimsrgza [_] ..., a i -> a i ...r8za b [_] ..., a b i -> a b i ...йrr@rErBrCrDr?)r5r5rйrr)rhza [...], a [2] -> a)rr5rza [...], a b [2] -> a b)r!r"r#┌tfr%r&┌int32rrOrP┌castrrU┌ gather_nd)r!r-r rYrrr ┌test_compare_tensorflow╡sЪ¤¤¤¤¤0 ¤¤0 ¤0 ¤8¤8¤ror4cCsь ddl}tjddДdН}tjjddН}tjjddН}ttа||б|аd||б|Гtjjd dН}tjjd dН}ttа||б|аd ||б|ГtjjddН}tjjddН}ttа||б|аd ||б|ГtjjddН}tjjddН}ttа||б|аd||б|ГtjjddН}tjjddН}ttа||б|аd||б|Гtjjd dН}tjjd dН}ttа||б|аd ||б|ГtjjddН}tjjddН}ttа||б|аd||б|ГtjjddН}tjjddН}ttj ||ddН|аd||б|ГtjjddН}tjjddН}ttj ||dgdgfdН|аd||б|Гttj ||ddН|аd||б|ГtjjddН}tjjddН}ttj ||ddН|аd||б|Гtjjd dН}tjjd dН}ttа ||б|аd||б|ГtjjddН}tjjddН}ttа ||б|аd||б|ГtjjddН}tjjddН}ttа||б|аd||б|Гtjjd dН}tjjddН}ttа||б|аd ||б|ГtjjddН}tjjd!dН}ttа ||б|аd"||б|ГtjjddН}ttj|dd#Н|аd$|б|ГtjjddН}ttj|dd#Н|аd%|б|ГtjjddН}ttа|б|аd%|б|ГtjjddН}ttа|б|аd$|б|ГtjjddН}tjjddН}ttj||gd&d#Н|аd'||б|ГtjjddН}tjjddН}ttj||gd&d#Н|аd(||б|Гtjjd)dН}tjjd*dН}t|аd+||бtj|tа|dddddЕf|jdd,Е|jбgd&d#Н|Гt|аd-||бtj|tа|dddЕddf|jd|jd|jd|jd,fбgdd#Н|Гtjjd dН}tjjddН}ttj||d.d/Н|аd0||б|Гttj||d1d/Н|аd2||б|ГtjjddН}tjjddН}ttаd3||б|аd||б|Гttаd4|б|аd%|б|Гttаd5||б|аd6||б|Гtjjd7dН}ttаd8|б|аd9|б|Гtjjd!dН}tjjd*dНddЕdftа|jddЕбdddЕfаd:б}t||ddЕdf|ddЕdff|аd;||б|Гtаd<б}t||аd=d>б|Гt||jd?d>d.r йrrr)rrza [b], [b] c -> a c)rz[a], [a] ->)r=rr)r=rrz... a [b], ... [b] c -> ... a criz... [b], [b] -> ...)r r rr)r r rrz#x... [b], y... [b] c -> x... y... cr7)┌axesйrrr)r rrGrza b [c], d [c] e -> a b d e)rr7rj)rr7z[a] b, c [a] -> b czx... [a], y... [a] -> x... y...йrrza b, a b -> a bza, b -> a b)rr za..., b... -> (a b)...rfz[a] bza [b]rHz..., ... -> ... (1 + 1)z... a, ... b -> ... (a + b)r)rGzb h w c1, c2 -> b h w (c1 + c2)rzb c1 h w, c2 -> b (c1 + c2) h w┌ij)┌indexingzx, y -> x y, x y┌xyzx, y -> y x, y xz ab,bc->aczab->az ab,bc->abcza b, b c -> a b c)rrzaa->aza a -> arlz[x y], a [2] -> a)rrr z-> 3 4 5gz-> a...)┌a)rr)rrr za c, a b d -> a b c d)rr7rrza b c d -> c b d a)rrrrzb [x], b [x] -> b)rz c -> 3 c 5rFrщ rIr rJ)rrr)┌axis1┌axis2za b b -> a b)&r!r"r#r$r%r&r┌matmulr)┌ tensordot┌inner┌multiply┌outer┌kron┌flip┌fliplr┌flipud┌stackr,┌concatenate┌broadcast_torU┌meshgridr*r┌asarrayr'rP┌zeros┌newaxis┌add┌ transpose┌squeeze┌pytest┌raises┌ ExceptionrX┌diagonal)r!r-r r/┌img┌vecrY┌ vec_as_img┌outr[r\r]r^r_r`rrr ┌test_compare_numpys ¤ ¤ ¤ ¤ ¤ ¤ ¤¤¤¤¤ ¤ ¤ ¤ ¤ ¤ ¤ ¤ ¤ ¤¤¤8¤H¤¤¤¤ ¤¤ ¤@"¤ ¤¤8¤ ¤B¤>¤ ¤ ¤rЦ┌jaxcs*ddl}tjddДdН}|jаtjjб}|jаtjjб}|jаtjj б}tа tjj ddНб}tа tjj ddНб}ttjа||б|d||Г|Гtа tjj ddНб}tа tjj d dНб}ttjа||б|d ||Г|Гttjа||б|d||Г|Гtа tjj ddНб}tа tjj ddНб}|d ||ГjdksШJВ|d|ГjdksвJВ|d|Г\}}|jdks░JВ|jdks╖JВtа tjj ddНбаdб}d-ddД} |jj| ddН} | d|Гj|jks▄JВ| d|Гj|jksчJВ| d|Гj|jksЄJВ| d|ddНj|jks JВtа tjj ddНбаdб}tjаdбЙd ЙЗЗfd!dД}|аd"||бj|jkРs)JВ|аd#||бj|jkРs7JВ|аd$||бj|jkРsEJВtа tjj d%dНбаdб}tа tjj d&dНбаdб}d'd(Д} |jа| б}|d)||Гjd*kРsuJВtа tjj d+dНбаdб}|jаtjjбd|Гjd,kРsУJВdS).NrcSs tа|бSr)r$rИrrrr rZs z"test_compare_jax..r )щ rrrrzi [n n], [n] -> i [n])rШrrzi [n n], i [n m] -> i [n m]zi [n n], i [n] m -> i [n] m)rШrzi [n n], j [n] -> [n] (i j))rщdzi [n n] -> i)rШzi [n n] -> i [n], [n n] i)rrrШ)rr=r=щrчёhуИ╡°ф>cSs,tа|б}tа|б}||tjа||бSr)┌jnpr┌varrЧ┌lax┌rsqrt)r ┌epsilonrrЭrrr ┌ normalizes z#test_compare_jax..normalizez ... -> ...)┌ signaturez... [c]z[...] crzb [s...] (g [c])r)┌ggЪЩЩЩЩЩ╣?cstjаИdИ|бdИS)NgЁ?)rЧr%┌ bernoulliйrUй┌dropout_rate┌keyrr rПsz..., ...zb ... c, b czb ..., brprscSstа|бdtа|бS)Nr)rЬrrБ)r r/rrr ┌myfuncШsz test_compare_jax..myfuncza [b], c [b] -> c [b] a)rrrr)rr )rЫ)r!r"r#rЧ┌adapt_with_vmaprЬ┌linalg┌solve┌det┌eigrИr$r%r&rrUr'┌PRNGKeyr~┌adapt_numpylike_reduce┌norm)r!r-┌einsolve┌eindet┌eineig┌Ar ┌vals┌vecsrб┌einnormalize┌dropout_factorr/rй┌ einmyfuncrrжr ┌test_compare_jaxWsh ¤ ¤ ¤ &r╗┌scipycCsXddl}tjddДdН}tjjddН}tjjddН}ttjа ||б|а d||б|ГdS) NrcSrrrrrrr rйrz$test_compare_scipy..r rpr)r rza c, b c -> (a b) c)r!r"r#r$r%r&rr╝rл┌ khatri_raor~)r!r-r r/rrr ┌test_compare_scipyжs¤r╛┌eindexc Csdddl}tjddДdН}tjjddН}ttа|dб|аd|б|Гttа|d б|аd |б|Гttа |dб|а d|б|Гt аtбПtа|d бWdГn1sSwY|аd|бtjjddН}tjjddНdtа |jddЕбddЕdddfаdб}ttа||dб|аd||б|Г|аd|б}t аtбПtа||dбWdГn1s░wY|аd||бtjjddН}tjjddНdtа |jddЕбddЕdddfаdб}ttа||dб|аd||б|Г|аd||dбt аtбПtа||ddбWdГn 1РswYtjjddН}tjjddНdddЕftа |jdd ЕбddЕdfаdб}ttа||d!б|аd"||б|Гt аtбПtа||d#бWdГn 1РshwYd$}d%}tjjd&dН} tjjd&dНdtа ||gбddЕdddfаdб}ttj| |d'||d(Н|jd)ttj| jd*Н|| ||d(Н|ГdS)+NrcSrrrrrrr r╗rz%test_compare_eindex..r rrza b c d -> [d] c a bza b c [d] -> [1] c a bza b c d -> [c, d] a bza b [c d] -> [2] a bza b c d -> [a, c, d] bz[a] b [c d] -> [3] bza b c d -> c [d] a bza b c [d] -> c [1] a b)rGrr )N.rrlza b c d, [a, b] e c d -> e c dz[a b] c d, [2] e c d -> e c dzi e c d -> e i c dza b c d, e [a, b] c d -> e c dz[a b] c d, e [2] c d -> e c d)r7rr r7za c d, [a] e c d -> e c dz[a] c d, [1] e c d -> e c dz[a] c d, e c d -> e c dza c d, e c d -> e c d)щrrr )r└rzb h w c, [h, w] b -> b czb [h w] c, [2] b -> b czb h w c, b [h, w] -> b cr rGrrza b c, [d, e] a b c -> d e b c)┌d┌ez[d e] b c, [2] a b c, a b cr@)r!r"r#r$r%r&r┌EX┌argmin┌argmaxrОrПrРrИrUr'rOrPr,┌scatter┌add_atrrЙrA) r!r-r rY┌coords2rТ┌idxr┴r┬┌updatesrrr ┌test_compare_eindex╕sА ¤ ¤ ¤ <¤ <¤ @¤ 6 ¤r╦csЁddlЙtjddДdН}tаtjjddНаdбб}tаtjjddНаdбб}tаtjjddНаdбб}dЙЗЗfd d Д}ЗfddД}Зfd dД}Иj а |б}|fЗfddД }t||||Г||||Г|Гt||||Г||||Г|Гtа|j dбtа|j dб} } tаtjjd| j d| j dИfdНаdбб}Иаd| | б}Иаd||tjб} | ddЕtjf| tjddЕfk}tа|tjddЕddЕtjf|tjб}t| ||Гtаtj| j d| j dftdНб}t|||Гt|||ГdS)NrcSrrrrrrr rrz1test_compare_multiheadattention..r )rr=rrrrcs@Иjd||ИdН}Иаd|tа|jdИбб}Иаd||бS)Nz#b q (h [c]), b k (h [c]) -> b q k hй┌hz b q [k] hrHz#b q [k] h, b [k] (h c) -> b q (h c)йr)┌softmaxrЬ┌sqrtrUй┌q┌k┌vr╡йr!┌headsrr ┌attn_einx_fulls z7test_compare_multiheadattention..attn_einx_fullcsxtj|dИdН}tj|dИdН}tj|dИdН}tаd||бtа|jdб}tjj|ddН}tаd||б}tа|d бS) Nzb q (h c) -> b q h cr╠zb k (h c) -> b k h czbqhc,bkhc->bqkhrHщ■ rfzbqkh,bkhc->bqhczb q h c -> b q (h c)) r┌ rearrangerЬr*r╨rUrЧ┌nnr╧)r╥r╙r╘r╡┌outputйr╓rr ┌attn_einopssz4test_compare_multiheadattention..attn_einopscs8Иаd||б}Иаd|tа|jdбб}Иаd||бS)Nz[c], k [c] -> kz[k]rHz[k], [k] c -> cr╬r╤йr!rr ┌attnsz-test_compare_multiheadattention..attncs|d|||ИdНS)Nz8b q (h [c]), b [k] (h [c]), b [k] (h [c]) -> b q (h [c])r╠r)r╥r╙r╘┌einattnr▄rr ┌attn_einx_decomposed$sz=test_compare_multiheadattention..attn_einx_decomposedr7rzq, k -> q kz q k, b q k h,r@)r!r"r#rЬrИr$r%r&r'rЧrкr┌arangerU┌ greater_equal┌where┌infrК┌tril┌ones┌bool)r-r╥r╙r╘r╫r▌r▀rрrс┌qs┌ksr╡┌mask1┌A1┌mask2┌A2┌mask3rr╒r ┌test_compare_multiheadattentions> ¤ ¤".$*$rЁ┌flax)┌linencszddlЙGЗfddДdtjГ}tаdб}|ddН}|аdtjаdбi|б}|dd j d ks0JВ|а ||бj dks;JВdS)Nrcs,eZdZUdZeed<ejЗfddДГZdS)z"test_compare_flax..MyLayerr┌c_outcs$ЗfddД}Иjd||ИjdН}|S)NcsИаdtjаdб|бS)N┌weightg{оGсzД?)┌paramr┌┌initializers┌normalrей┌selfrr rOsz=test_compare_flax..MyLayer.__call__..z%... [c_in], [c_in] c_out -> ... c_outйrє)r)rє)r∙r rЇr▐r°r ┌__call__Msz+test_compare_flax..MyLayer.__call__N) ┌__name__┌ __module__┌__qualname__rє┌int┌__annotations__r┌┌compactr√rr▐rr ┌MyLayerJs r)rrЪrЪrrr·┌paramsщ*rЇ)rr)rrЪrЪr)r!r┌┌ModulerЬrЙ┌initrЧr%rпrU┌apply)rr ┌modelrrr▐r ┌test_compare_flaxGs r )$┌ importlibr"┌ functoolsrr4r$┌conftestrrО┌util┌ find_specrr0r1ra┌os┌environrbrkrorЦ┌ jax.numpyrЬrЧr╗┌scipy.linalgr╝r╛┌eindex.numpyr├r╦rЁrёrЄr┌r rrrr ┌sV?a ]ALH=√