o ig.@s$UddlZddlmZddlmZmZmZmZmZm Z ddl m Z ddl m Z ddlmZmZeeefZeeeeefZeeeeefZdd d d Zeeeeeeffed <d dddededd dZeeeeeeffed<e dedfedededede eededd ddedededededededefddZe dedfededededede eedd d dedededededededefd!d"Ze d#edede eedd d$dededededef d%d&ZGd'd(d(eZ gd)Z!dS)*N) defaultdict)DictListOptionalTupleUnioncast) get_array_ops)registry)FloatsXd GeneratorT?)L2L2_is_weight_decay grad_clip SGD_DEFAULTSgMbP?g?g+?g:0yE>rr) learn_ratebeta1beta2epsrrr ADAM_DEFAULTSzRAdam.v1rrrrr)rrrrrr use_averagesrc Cst||||||||dd S)NT)rrrrrrr use_radam Optimizer)rrrrrrrrrD/home/ubuntu/.local/lib/python3.10/site-packages/thinc/optimizers.pyRAdam rzAdam.v1)rrrrrrrc Cst||||||||dd S)NF)rrrrrrrrr)rrrrrrrrrrrAdam8r r!zSGD.v1)rrrrc Cst||||dd|dS)Nr)rrrrrrr)rrrrrrrrSGDQs r"c@speZdZUdZeeefed<eeefed<eeeefed<ee e fed<eee fed<eee fed<e ed<e ed <e ed <e ed <e ed <e ed <e ed<e ed<eeeeed<gdZed ededed edddddd ed ededed edede de de fddZddZddZddd ee e fd!ed"ed#e fd$d%Zd&d'Zd(d)Zd*S)+rzDo various flavours of stochastic gradient descent, with first and second order momentum. Currently support 'vanilla' SGD, Adam, and RAdam. mom1mom2averages schedules nr_update last_seenrrb1b2rrrr _radam_buffer)r#r$r%r&r'r(rrr)r*rrrrr+rrTF)rrrrrrrrrc Csi|_i|_|r i|_nd|_i|_tt|_tt|_|d||d||d||d||d||d|||_ | |_ dd t d D|_ dS) al Initialize an optimizer. learn_rate (float): The initial learning rate. L2 (float): The L2 regularization term. beta1 (float): First-order momentum. beta2 (float): Second-order momentum. eps (float): Epsilon term for Adam etc. grad_clip (float): Gradient clipping. use_averages (bool): Whether to track moving averages of the parameters. use_radam (bool): Whether to use the RAdam optimizer. L2_is_weight_decay (bool): Whether to interpret the L2 parameter as a weight decay term, in the style of the AdamW optimizer. Nrrr)r*rrcSsg|]}gdqS))NNNr).0_rrr sz&Optimizer.__init__.. ) r#r$r%r&rintr'r(_set_attr_or_schedulerrranger+) selfrrrrrrrrrrrr__init__s"        zOptimizer.__init__c Cst|tttfrt|||dSt|trt|}||j|<z t||t|WdSt t fyH}zd|dt |d|}t |d}~ww)NzInvalid schedule for 'z' (z) ) isinstancefloatboolr0setattrlistiterr&next StopIteration TypeErrortype ValueError)r3namevalueeerrrrrr1s  zOptimizer._set_attr_or_schedulec CsN|jD]\}}zt|}Wntyt||}Ynwt|||qdS)N)r&itemsr;r<getattrr8)r3keyschedulerArrrstep_scheduless  zOptimizer.step_schedulesr)lr_scalerFweightsgradientrIcCsNt|dkr ||fSt|}|j|d7<|j|}|jdkr+|js+||j|7}|jr5|||j}|jrE|||||||\}}n'|j dkr\|j dkr\| ||||||\}}n|j dkrct |||j |8}|d9}|jdkr|jr|||j |j|8}|jdur||jvr|j|jdd|j|<||j|||||fS)zCall the optimizer with weights and a gradient. The key is the identifier for the parameter, usually the node ID and parameter name. r rrNfloat32)dtype)lenr r'rrr clip_gradientr_radamr)r*_adamNotImplementedErrorrr%allocshapeupdate_averages)r3rFrJrKrIopsnr_updrrr__call__s8        zOptimizer.__call__cCs||jvr||j|j|<||jvr||j|j|<|||j}|||j}|j||j||j|d} |j|j|jg|j d|j d} d} | d| d} } | d\}}| |9} | d||d 7} | |9} | d||7} | d d7<| d t | d d }| d |d kr|d|d }}na| d |d <|| d }d d|d}|d | d |d|}||d<|dkrt d||d|d|d |||d d|| d }n| rdd|| d }nd}||d <|dkr+| dd kr|| d | d|7}|j | | d}|| | d| |7}n$|d krO| dd krD|| d | d|7}|| | d| 7}|||j|||jfS)N)stepexp_avg exp_avg_sqr)lrbetasr weight_decaybufferTrZr[r]r rYr_r/rrr^r\r)r#alloc1fsizer$ reshape1fr'rr)r*rr+r0mathsqrtxp reshape_frT)r3rVrJgradrIrFrW weights_1D gradient_1Dstategroupdegenerated_to_sgdrZr[rrbufferedN_sma step_sizebeta2_t N_sma_maxdenomrrrrPs           zOptimizer._radamc Cs|||j}|||j}||jvr||j|j|<||jvr*||j|j|<|j|} |j|} |j} |j} d| |} d| |}|j|d| }|j}| ||| | | | |||\}}} } | |j|<| |j|<| ||j | ||j fS)Nrg?) rfrer#rdr$r)r*rradamrjrT)r3rVrJrKrIrFrWrlrmr#r$r)r*fix1fix2r\rrrrrQHs,          zOptimizer._adamN)__name__ __module__ __qualname____doc__rKeyTr __annotations__rstrr r0r6r7r __slots__r FloatOrSeqr4r1rHrrXrPrQrrrrresv    .   ) Nr)r!rr"rrr)"rg collectionsrtypingrrrrrrbackendsr configr typesr r r0rr~r6rIntOrSeqrr7rr optimizersrr!r"objectr__all__rrrrs