o i+o@s<UdZddlmZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl m Z m Z ddlmZmZmZddlmZddlZddlmZeejejdd ed Zd ZeejjZd Z ed dZ!gdZ"hdZ#de$d<d#d$ddZ%eGdddZ&GdddZ'd d!Z(e)d"kre(dSdS)%aFleet deployer: provision N GPU instances on Vast.ai, bootstrap, start production workers. Usage: python scripts/deploy_fleet.py --num-gpus 5 --max-price 0.55 python scripts/deploy_fleet.py --num-gpus 5 --dry-run # show what would be rented python scripts/deploy_fleet.py --status # check running fleet python scripts/deploy_fleet.py --destroy-all # tear down everything ) annotationsN)ThreadPoolExecutor as_completed) dataclassfieldasdict)Path) load_dotenvz'%(asctime)s [%(levelname)s] %(message)s)levelformatfleetzhttps://console.vast.ai/api/v0z%bharathkumar192/codecbench-sft:latestresultszfleet_state.json) RTX 4090zRTX 3090z RTX 3090 TiL40SL40zRTX 4080z RTX 4080Sz RTX A6000z RTX A5000> _w<x̦zset[int] BAD_MACHINESvram_gbfloatgpu_namestrreturnintcs6|dkrdStfdddD}|r|dkrdSdS) N(c3s|]}|vVqdSN).0krr"scripts/deploy_fleet.py /sz"pick_batch_size..)40903090A100)any)rrhigh_endr"r%r&pick_batch_size,s  r0c@sveZdZUded<ded<ded<ded<dZded <d Zded <d Zded <dZded<d Zded<d Z ded<dS) InstanceInfor instance_idrr machine_idrprice_per_hourrssh_hostrssh_portcreatedstatus batch_sizer worker_pidN) __name__ __module__ __qualname____annotations__r5r6r8r:rr;r"r"r"r&r15s      r1c@seZdZdLdMddZdNdOddZdPddZdQddZdRdSddZdTdSd d!ZdUdVd$d%Z dWd&d'Z (dXdYd+d,Z dWd-d.Z dWd/d0Z dQd1d2ZdZd5d6Zd[d7d8Zd[d9d:Zd\d;d<Zd]d?d@Zd^d_dEdFZd^d`dGdHZd[dIdJZdKS)a FleetDeployer皙?api_keyr max_pricercCs:||_||_d|dd|_g|_tdd|_dS)NzBearer zapplication/json) Authorizationz Content-Typez.ssh id_ed25519)rBrCheaders instancesrhome identity_file)selfrBrCr"r"r&__init__Ds zFleetDeployer.__init__rlimitrr list[dict]c Cs|dddiddiddid|id|jiddggd }td D]:}tjtd |j|d d }|jdkr;td|dq| | dg}dd|D}t |dddS| gS)N on-demandeqTFr-lte dph_totalasc)rMtyperentablerentednum_gpusrrRorder /bundles/rFjsontimeoutr9offerscSsZg|])}|dtvr+t|dddkr+t|dddkr+t|dddkr|qS)r3 reliabilityrg(\? inet_down2 disk_spacer)getrrr#or"r"r& Zsz/FleetDeployer.search_offers..cSs&|d|dp d |dpd fS)NrRrbrra)rergr"r"r&cs&z-FleetDeployer.search_offers..key) rCrangerequestspostBASE_URLrF status_codetimesleepraise_for_statusr]resorted)rJrrMbodyattemptrr`filteredr"r"r& search_offersKs,     zFleetDeployer.search_offersrWc Cs.i}tD]}|j|dd}|D]}||d<q|||<td|t|qg}t}dD]'}||gD]}t||kr=n|d}||vrGq3||||q3q+g} tD]}|dvr^qW| ||gqW| j ddd | D]}t||kr|n|d}||vrqr||||qr|d |S) zUPick N offers prioritizing high-throughput GPUs (L40S, 4090) then filling with value.rc)rM _gpu_typez %s: %d offers available)rrrr3cSs|dS)NrRr"rir"r"r&rjsz0FleetDeployer.pick_best_offers..rkN) GPU_PREFERENCESrzloggerinfolensetreappendaddextendsort) rJrWper_typegpur`rgpicked used_machinesmid remainingr"r"r&pick_best_offershsD          zFleetDeployer.pick_best_offersofferdictInstanceInfo | Nonec Cstd}i}d}d}|r?|D]*}|}|r>|ds>d|vr>|dd\}}|||<|dkr8|}q|dkr>|}q|dd d } |d d} t| | } d | d } t ddd|d d|| d} |rx|rxd| d<|| d<|| d<d}t dD]"}t j t d|dd|j| d}|jdkrtd|dq~|r|jstd|d|r|jdddSddS|d }|sdS|dd d } t||d d!|d"d |d#d | t| |d dd$}td%||j|j|j|S)&Nz.envr#=r-DOCKER_USERNAME DOCKER_PATgpu_ramrramkdir -p /tmp/pipeline; cd /app && python3 - <<'PY' import os keys = [ 'HF_TOKEN','R2_ENDPOINT_URL','R2_BUCKET_DESTINATION','R2_ACCESS_KEY_ID', 'R2_SECRET_ACCESS_KEY','ACCOUNT_ID','S3_API','DATABASE_URL','VAST_API' ] with open('/app/.env', 'w') as f: for k in keys: f.write(f"{k}={os.environ.get(k, '')}\n") PY python3 -m codecbench.pipeline.cli sft-run --batch-size z6 --offer-id vast_$CONTAINER_ID >/tmp/worker.log 2>&1 &< ssh_directzfleet-r)imagediskruntypelabelenvonstartzhttps://index.docker.io/v1/docker_login_repodocker_login_userdocker_login_passrYz/asks/id/)rFr]r_zCreate failed for offer %s: %sz no response new_contractunknownr3rR)r2rr3r4rr:z*Created instance %d: %s @ $%.3f/hr (BS=%d)) PROJECT_ROOTexists read_text splitlinesstrip startswithsplitrer0 DOCKER_IMAGErmrnputrprFrqrrrsokr}errortextr]r1r~rr4r:)rJrenv_fileenv_vars docker_user docker_patliner$vrrbsr create_bodyrxrwiidr~r"r"r&create_instances          zFleetDeployer.create_instanceXr~r1r^boolc CsVt}t||krtjtd|jd|jdd}|js%tdq|d|}t |t r<|r:|dni}|dpBd }|d pId }|d krg|d rg|d rg|d |_ |d |_ d |_ dSd|vso|dvrtd|j||ddd|_ dStt|}td|j|pd||ddtdt||ks d|_ dS)N /instances/rr[rFr^rGr actual_statusr status_msgrunningr5r6TError>deadexitedofflinezInstance %d failed: %s / %sfailedFz Instance %d: %s [%ds] %spendingPr^)rrrnrerpr2rFrrsr] isinstancelistr5r6r8r}rrr~) rJr~r^t0rxinstactualmsgelapsedr"r"r&wait_instance_readys<     z!FleetDeployer.wait_instance_readyc Cst}t||krKz(tjddddddt|jdt|jd|jdg d d d d }|jd kr3Wd SWn ty=Ynwt d t||ks dS)Nssh-oStrictHostKeyChecking=noConnectTimeout=10-i-proot@zecho OKTrLcapture_outputrr^r F) rr subprocessrunrrIr6r5 returncode Exceptionrs)rJr~r^rrxr"r"r&wait_sshs$      zFleetDeployer.wait_sshcmdc Csddt|jdddddt|jd|j|g }z%tj|dd|d }|jd kr:td |j |j |j d dWd SWdStj yNtd|j Yd Sw)NrrrrzServerAliveInterval=30rrTrrr^rzSSH cmd failed on %d (%s): %s FzSSH cmd timed out on %d)rrIr6r5rrrr}rr2rstderrrTimeoutExpired)rJr~rr^fullrxr"r"r&run_sshs zFleetDeployer.run_sshcCsdttd}gd}ddddd|jd|jg||d |jd }tj|d d d d }|jdkS)Nr) z--exclude=venv/z--exclude=repos/z--exclude=data/z--exclude=results/z--exclude=metafiles/z--exclude=__pycache__/z--exclude=.git/z--exclude=*.pycz--exclude=.cursor/z--exclude=models/rsyncz-avzz --timeout=60z-ezssh -i z -o StrictHostKeyChecking=no -p rz:/app/Txrr)rrrIr6r5rrr)rJr~srcexcludesrrxr"r"r& sync_code!s   zFleetDeployer.sync_coderlog_file poll_intervalcCsd|d|d}ddt|jddddd t|jd |j|g }z tj|d d d d }|jdkr?td|j |j ddWdSWnt y[} ztd|j | WYd} ~ dSd} ~ wwt } t | |krt |z`tjddt|jddddd t|jd |jd|dg d d dd } | jdkrd| jvrdd| jdD} | r| ddd} | dkrWd Std| |j | jddWdSWn t yYnwt | |kshtd|j |dS) zGRun a command via nohup, poll for completion. Survives SSH proxy drops.znohup bash -c 'z; echo __DONE_EXIT_$?' > z 2>&1 & echo $!rrrrrrrTr[rrz'Failed to launch detached cmd on %d: %sNrFzSSH launch error on %d: %sztail -5 z 2>/dev/nullrL __DONE_EXIT_cSsg|]}d|vr|qS)rr")r#lr"r"r&rhMz2FleetDeployer.run_ssh_detached.. 0z-Detached cmd failed (exit=%s) on %d. Tail: %srz&Detached cmd timed out on %d after %ds)rrIr6r5rrrr}rr2rrrrrsstdoutrr)rJr~rrr^rwrapperrrxercheck exit_linecoder"r"r&run_ssh_detached1sX    zFleetDeployer.run_ssh_detachedcCsrtd|j|j|j|dddstd|jdStd|j|j|j|dd ds4td |jdSd |_d S) zLightweight bootstrap: pip install -e . (code from rsync) + verify GPU. Docker image already has all deps and Spark-TTS repo. Models download on first worker.setup(). z [%d %s] pip-editable ...zDcd /app && python3 -m pip install --no-cache-dir -e . 2>&1 | tail -3rr^zpip install -e . failed for %dFz [%d %s] verify-gpu ...zipython3 -c "import torch; assert torch.cuda.is_available(); print('GPU:', torch.cuda.get_device_name(0))"rzGPU verification failed for %d bootstrappedT)r}r~r2rrrr8)rJr~r"r"r&bootstrap_instance[sz FleetDeployer.bootstrap_instancec Csd|jd|jd}ddt|jddddd t|jd |j|g }td D]}z\tj|d d d d}|j rC|j dd nd}|j dkrg| rgt||_d|_td|j|j|j|jWd Std|d|j|j |j dd|j ddWn0tjytd|d|jYnty}ztd|d|j|WYd}~nd}~wwtdq&td|jdS)zStart the production pipeline worker as a background process. Uses bash -c with explicit fd closing so SSH doesn't hang waiting for nohup. zCcd /app && python3 -m codecbench.pipeline.cli sft-run --batch-size z --offer-id vast_z+ /tmp/worker.log 2>&1 & echo $!rrrrzConnectTimeout=30rrrYTrrrrrrworker_runningz&Worker started on %d (%s) PID=%d BS=%dz6Worker start attempt %d on %d: rc=%d out=[%s] err=[%s]r-8Nz'Worker start attempt %d timed out on %dz'Worker start attempt %d error on %d: %sz-Failed to start worker on %d after 3 attemptsF)r:r2rrIr6r5rmrrrrrrisdigitrr;r8r}r~rwarningrrrrrrsr)rJr~ worker_cmdrrwrx last_linerr"r"r& start_workerlsB &  " zFleetDeployer.start_workerc Cs||}|s dSz!|j|dds||jWdStd|j|j|jd|_|WSt yN}zt d|j|t ||jWYd}~dSd}~ww)zPCreate instance and wait for it to start. Worker auto-starts via onstart script.Nrru>Instance %d running at %s:%d — onstart worker auto-launchingrzProvision failed for %d: %s %s) rrdestroy_instancer2r}r~r5r6r8rr traceback format_exc)rJrr~rr"r"r& provision_ones(     zFleetDeployer.provision_oner2NonecCs8tjtd|d|jdd}|jrtd|dSdS)Nrrr[rzDestroyed instance %d)rndeleterprFrr}r~)rJr2rxr"r"r&r szFleetDeployer.destroy_instancecCs:|}|D] }||dqtjddtddS)Nr2T) missing_okzAll fleet instances destroyed) load_stater FLEET_STATE_FILEunlinkr}r~)rJstaterr"r"r& destroy_alls  zFleetDeployer.destroy_allcCs8tjjddddd|jD}ttj|dddS)NT)parentsexist_okcSsg|]}t|qSr")rr#ir"r"r&rhsz,FleetDeployer.save_state..r9)indent)rparentmkdirrG write_textr]dumps)rJdatar"r"r& save_stateszFleetDeployer.save_statecCstr ttSgSr!)rrr]loadsr)rJr"r"r&rszFleetDeployer.load_stateoffer_id dict | Nonec CsdD]@}ddddiddiddid|id d ggd }tjtd |j|d d}|js*q|dgD]}|d|krA|Sq2qdS)zSFetch a specific offer by ID. Searches multiple price tiers to avoid limit cutoffs.)g333333?rAg@g@irOrPTFr-rQrRrS)rMrTrUrVrWrRrXrZr[r\r`rN)rnrorprFrr]re)rJr"rCrvrxrgr"r"r&get_offer_detailss   zFleetDeployer.get_offer_detailsF offer_ids list[int]dry_runcsg}|D]}|}|r||qtd|q|s#tddStddtdt|dtdd}t|D]_\}}|d dd }t |} |d } || 7}|d d} |d d} |dd} t |dddd}td|dd|ddd|dd| d| dd| dd| dd| dd|q?td|dd |d!d"d#tdd|rtd$dSt d%t|t d&d'd(Lfd)d*|D}t |D]5}||}z|}|rj|Wqty}ztd+|d|WYd}~qd}~wwWdn 1swYd,d-jD}t|t|}tddtd.t|d/t|d0td|D]%}td1|jd2|jdd3|jd|jdd4|jd|jd5|jqM|r~td6|d7tdddS)8z)Deploy specific hand-picked offers by ID.z)Offer %d not found or no longer availablezNo valid offers found!NrF======================================================================z MANUAL FLEET DEPLOYMENT ( GPUs)rrrrRrbinet_upra geolocation? r-. r<20 .0fzGB BS= $.3fz/hr DL=z UL= rel= Total fleet cost: $/hr ($.2f/day)" DRY RUN — no instances created*Provisioning %d instances (8 at a time)...r prov max_workersthread_name_prefixci|] }j||qSr"submitr rfpoolrJr"r& z5FleetDeployer.deploy_by_offer_ids..Provision error for %s: %scSg|] }|jdkr|qSrr8rr"r"r&rhz5FleetDeployer.deploy_by_offer_ids.. DEPLOYMENT RESULT: r workers running Instance :  BS= /hr PID=: instance(s) failed to provision)r$rr}rrprintr enumeraterer0rr~rrresultrGr rr2rr:r4r;r5r6)rJr%r'r`oidrg total_costrvramrpricedlulrelgeofuturesfutrr~rr failed_countr"rDr&deploy_by_offer_idss       (        z!FleetDeployer.deploy_by_offer_idscs|}|stddStddtdt|dtdd}t|D]@\}}|ddd }t|}|d } || 7}td |d d |ddd|dd|d| dd|dd|dddq*td|dd|dddtdd|rtddStdt|t d d!d"Kfd#d$|D} t | D]4} | | } z| } | rj | Wqty}ztd%| d|WYd}~qd}~wwWdn1swYd&d'j D}t|t|}tddtd(t|d)t|d*td|D]%} td+| jd,| jdd-| jd| jdd.| jd | jd/| jq|rHtd0|d1tdddS)2z0Deploy N GPU instances across diverse GPU types.zNo suitable offers found!Nrr(z FLEET DEPLOYMENT PLAN (r)rrrrRr.r-r/rr0r1r2z GB VRAM BS=r3r4z /hr machine=r3r5rar6r7r8r9r:r;r<r r=r>crAr"rBrfrDr"r&rF$rGz.FleetDeployer.deploy_fleet..rHcSrIrJrKrr"r"r&rh0rLz.FleetDeployer.deploy_fleet..rMrrNrOrPrQrRrSrTrU)rr}rrVrrWrer0r~rrrXrGrr rr2rr:r4r;r5r6)rJrWr'r`rZrrgr[rr\rarbrr~rrrr"rDr& deploy_fleets~   (         zFleetDeployer.deploy_fleetc Csp|}|s tddStddtdt|dtd|D]}|d}|dd }|d d }|d d }z>tjtd|d|jdd}|jrt|d|}t |t rg|re|d ni}|dpr|dprd} nd|j } Wn t yd} Ynwd} d } |r|r| dkrzFt jddt|jdddddt|d |d!g d"d"dd#}|jd kr|jd} | r| d$nd%} | d%k} d| dd$d&d} Wn t yYnw| rd'n | dkrd(| nd)}td*|d+|d,|d-|d.d td/| d0|d1|| r-| dd2dD]}td3|dd4qq$tddddS)5z$Check health of all fleet instances.z0No fleet state found. Deploy first with --deployNrr(z FLEET STATUS (z instances)r2rr,r5rr6rrrrrrGr cur_stater api_error_api_unreachableFrrrrrzConnectTimeout=5rrzetail -3 /tmp/worker.log 2>/dev/null && ps aux | grep 'codecbench.pipeline.cli' | grep -v grep | wc -lTrrrrOKzVAST: WORKER_DOWNz [z ] Instance z (z) BS=r:z Vast: z SSH: rSz Log: r)rrVrrernrprFrr]rrrqrrrrrIrrrrjoin)rJrrrrssh_hssh_prxd vast_status worker_alive recent_loglines proc_count status_emojirr"r"r& check_status>sh          (zFleetDeployer.check_statusN)rA)rBrrCr)rL)rrrMrrrN)rWrrrN)rrrr)r)r~r1r^rrr)r)r)r~r1rrr^rrr)r~r1rr)rr) r~r1rrrrr^rrrrr)r2rrr)rr)rrN)r"rrr#)F)r%r&r'rrr)rWrr'rrr)r<r=r>rKrzrrrrrrrrr r r rr rr$rdrervr"r"r"r&r@Cs,    + S    *  !       ?6r@cCs4tjdd}|jdddd|jdtddd |jd td d |jd tdd |jddd|jdddd|jdddd|}tj dd}|sTt dt dt ||jd}|jrd|dS|jrm|dS|jss|jr|jr|jdd|jdD|jddS|j|j|jddS|dS)Nz(Fleet deployer for Vast.ai GPU instances) descriptionz--deploy store_truezDeploy new fleet)actionhelpz --offer-idsz9Comma-separated Vast.ai offer IDs to deploy (manual pick))rTdefaultrzz --num-gpusr)rTr{z --max-priceg?z --dry-run)ryz--statuszCheck fleet healthz --destroy-allzDestroy all fleet instancesVAST_KEYrzERROR: VAST_KEY not setr-)rCcSsg|]}t|qSr")rr)r#xr"r"r&rhrzmain..,)r')argparseArgumentParser add_argumentrrr parse_argsosenvironrerVsysexitr@rCrr8rvdeployr'r%rdrrerW print_help)parserargsrBdeployerr"r"r&mainzs8        r__main__)r)rrrrrr)*__doc__ __future__rrr]loggingrrrrrr concurrent.futuresrr dataclassesrrrpathlibrrndotenvr basicConfigINFO getLoggerr}rp__file__resolverrrrr|rr?r0r1r@rr<r"r"r"r&sH        ;#