o #i@sddlZddlZddlZddlmZddlmZddlmZ ddl m Z ddl m Z gdZe eZdZdZd Z  ddededededededeefddZddZddZdejfddZdS) N)closing)Optional) get_logger)barrier)create_c10d_store get_free_portget_socket_with_portzAddress already in usezSocket Timeoutz_tcp_store/num_membersXT is_server server_addr server_port world_sizetimeoutwait_for_workers use_libuvc CsB|dkr|dkrtd|d||dkrtd||dkr"|nd} |dkr,|} nt} td|| ||||z/tjtj|| ||tj |d|d } |durT| } n| |d } |rbt | ||d td | WSt y} z*t | t kr||krtd | |||d7}n t d|d| d| WYd} ~ nd} ~ wwq%)Nr r zCserver_port must be specified when world_size > 1, got server_port=z , world_size=z+sever_port: %s, specified, ignoring retriesTziCreating c10d store on %s:%s world_size : %s is_server : %s timeout(sec): %s use_libuv : %s )seconds) host_nameportr is_masterrr)r)rzSuccessfully created c10d storez)port: %s already in use, attempt: [%s/%s]zon z, port: z already in use) ValueErrorloggerinfor functoolspartialdistTCPStoredatetime timedelta_check_full_rank RuntimeErrorstr_ADDRESS_IN_USEwarning) r rrrrrretriesrattemptr store_builderstoreer+i/home/ubuntu/SoloSpeech/.venv/lib/python3.10/site-packages/torch/distributed/elastic/utils/distributed.pyrsb           rc CsPz t||t|dWdSty'}zt|tkr"td|d|d}~ww)N) key_prefixbarrier_timeoutztimed out waiting for all z members to join)r_TCP_STORE_INITr"r#_SOCKET_TIMEOUT TimeoutError)r)rrr*r+r+r,r!hs  r!cCs>t}t||dWdS1swYdS)Nr )rr getsockname)sockr+r+r,rts  $rreturnc Cstjddtjtjd}|D]<}|\}}}}}t|||}z|d|d|WStyI}z|tj d|dWYd}~q d}~wwt d) a Returns a free port on localhost that is "reserved" by binding a temporary socket on it. Close the socket before passing the port to the entity that requires it. Usage example :: sock = _get_socket_with_port() with closing(sock): port = sock.getsockname()[1] sock.close() # there is still a race-condition that some other process # may grab this port before func() runs func(port) localhostN)hostrfamilytype)r5rrzSocket creation attempt failed.)exc_infozFailed to create a socket) socket getaddrinfo AF_UNSPEC SOCK_STREAMbindlistenOSErrorcloserr%r")addrsaddrr7r8proto_sr*r+r+r,rzs     r)r r r Tr N)rrr: contextlibrtypingrtorch.distributed distributedr'torch.distributed.elastic.utils.loggingr%torch.distributed.elastic.utils.storer__all____name__rr$r0r/boolr#intfloatrr!rrr+r+r+r,sJ      K