o pi_Y@sRddlmZddlZddlZddlZddlmZddlmZddl m Z ddl m Z ddl mZmZddlZddlZddlmZddlZddlmZdd lmZdd lmZdd lmZdd lmZm Z m!Z!m"Z"dd l#m$Z$ddl%m&Z&ddl'm(Z(m)Z)ddl*m+Z+ddl,m-Z-ddl.m/Z/GdddZ0eGdddZ1Gdddej2Z3dS)) annotationsN) dataclass)cached_property) import_module)Path)AnyOptional)_load) ModelSummary) __version__)Audio)ProblemSpecificationsTaskUnknownSpecificationsError)track_model_init)check_dependencies) AssetFileNamedownload_from_hf_hub)map_with_specifications) SlidingWindow) DataLoaderc@s eZdZdS) IntrospectionN)__name__ __module__ __qualname__rrM/home/ubuntu/.local/lib/python3.10/site-packages/pyannote/audio/core/model.pyr:src@s&eZdZUded<ded<ded<dS)Outputint num_frames dimensionrframesN)rrr__annotations__rrrrr>s  rcspeZdZdZ   dbdcfd d ZeddddZejdeddZddZedfddZ e jdgddZ e j ddZ dhdiddZ edjddZ e dkd!d"Zd#d$Zdhd%d&Zd'd(Zdld+d,Zdmd/d0Zdnd2d3Zdod5d6Zd7d8Zdod9d:Zd;d<Zd=d>ZdpdqdEdFZdrdGdHZdrdIdJZ K ?dsdtdOdPZ KdudvdRdSZ KdudwdTdUZe  K    dxdyd`daZ!Z"S)zModelaBase model Parameters ---------- sample_rate : int, optional Audio sample rate. Defaults to 16kHz (16000). num_channels : int, optional Number of channels. Defaults to mono (1). task : Task, optional Task addressed by the model. >N sample_rater num_channelstask Task | NonecsBt|dks Jd|dd||_t|jjdd|_dS)Nr&z7Only mono audio is supported for now (num_channels = 1)r'r(downmix)r'mono)super__init__save_hyperparametersr)r hparamsr'audio)selfr'r(r) __class__rrr.Rs   zModel.__init__returnrcCs|jSN)_taskr2rrrr)csz Model.taskcCs|`||_dSr6)specificationsr7)r2r)rrrr)gs cCsdSr6rr8rrrbuildmsz Model.build&Specifications | tuple[Specifications]c CsF|jdurz|j}W|Sty}ztd|d}~ww|jj}|S)NzModel specifications are not available because it has not been assigned a task yet. Use `model.task = ...` to assign a task to the model.)r)_specificationsAttributeErrorrr9)r2r9errrr9rs  zModel.specificationsr9cCslt|ttfs tdtdd|D}t|dkrtdtdd|D}t|dkr1td||_dS)NzEOnly regular specifications or tuple of specifications are supported.cs|]}|jVqdSr6)duration.0srrr z'Model.specifications..r&z1All tasks must share the same (maximum) duration.csr?r6) min_durationrArrrrDrEz/All tasks must share the same minimum duration.) isinstancertuple ValueErrorsetlenr<)r2r9 durations min_durationsrrrr9s   cCst|dr |`dSdS)Nr<)hasattrr<r8rrrr9s r@ float | None torch.TensorcCs6|p tt|jj}tjd|jj|j |f|j dS)Nr&)device) nextiterr9r@torchrandnr0r(r1get_num_samplesrQ)r2r@rrr__example_input_arrays zModel.__example_input_arraycCs|Sr6)_Model__example_input_arrayr8rrrexample_input_arrayszModel.example_input_arrayrcCsZ|jdd}|jdd|}|jdd|dd}t||jj||jj||jjdS)z(Internal) framesr&)r r)frame)startr@step)receptive_field_sizereceptive_field_centerrr0r')r2r^receptive_field_stepreceptive_field_startrrrreceptive_fields    zModel.receptive_fieldcCs|jdSr6)r) prepare_datar8rrrrczModel.prepare_datac Cs2|dkr |j|j_|jr|j|tdd|D}|}|z |j|dd\}}Wn'tyW}zdt |vrKd|j d}t |n|WYd}~nd}~ww|D]\}} |t | f|vrn| |jq\|jr||j_|j|jtd d|D} td d| |D|_ dS) Nfitcs |] \}}|t|fVqdSr6idrBnamemodulerrrrDzModel.setup..F)strictz size mismatchzModel has been trained for a different task. For fine tuning or transfer learning, it is recommended to train task-dependent layers for a few epochs before training the whole model: .csrfr6rgrirrrrDrlcss|]\}}|VqdSr6r)rBrj_rrrrDs)trainerr)setuprJ named_modules state_dictr:load_state_dict RuntimeErrorstrtask_dependentwarningswarnrhtorQmodelsetup_loss_funcsetup_validation_metriclist) r2stagebeforeoriginal_state_dict missing_keysunexpected_keysr>msgrjrkafterrrrrqs@         z Model.setupcCs(dti|jj|jjd|jd|d<dS)Npyannote.audio)rkclass)versions architecturer9)r r4rrr9r2 checkpointrrron_save_checkpointszModel.on_save_checkpointrdict[str, Any]cCs|dd|_|dS)Nrr9)r9rqrrrron_load_checkpoints zModel.on_load_checkpoint waveforms"torch.Tensor | tuple[torch.Tensor]cKs d}t|)NzAClass {self.__class__.__name__} should define a `forward` method.)NotImplementedError)r2rkwargsrrrrforwardsz Model.forwardnn.Module | tuple[nn.Module]cCs dd dd}t|j|S) aTGuess default activation function according to task specification * sigmoid for binary classification * log-softmax for regular multi-class classification * sigmoid for multi-label classification Returns ------- activation : (tuple of) nn.Module Activation. Nr9Specifications | Noner5 nn.ModulecSsL|jtjkr tS|jtjkrtjddS|jtjkr tSd}t|)N)dimz>TODO: implement default activation for other types of problems) problemr BINARY_CLASSIFICATIONnnSigmoidMONO_LABEL_CLASSIFICATION LogSoftmaxMULTI_LABEL_CLASSIFICATIONr)r9rrrr__default_activations    z6Model.default_activation..__default_activationr6)r9rr5r)rr9)r2_Model__default_activationrrrdefault_activations zModel.default_activationrcC |jSr6)r)train_dataloaderr8rrrr0 zModel.train_dataloadercC|j||Sr6)r) training_stepr2batch batch_idxrrrr5rdzModel.training_stepcCrr6)r)val_dataloaderr8rrrr:rzModel.val_dataloadercCrr6)r)validation_steprrrrr?rdzModel.validation_stepcCstjj|ddS)NgMbP?)lr)rToptimAdam parametersr8rrrconfigure_optimizersBszModel.configure_optimizersF module_namerv requires_gradbool list[str]c Cs|d}t}t|ddjD]?\}}|d}tddt||D}|r1t|t|dkr1q|jddD]} || _q7|j|d | |||krNnq||vrZt d ||S) z3Helper function for freeze_up_to and unfreeze_up_tornr) max_depthcss |] \}}||kr|VqdSr6r)rBtoken other_tokenrrrrDMsz Model.__up_to..r&Trecurse)modezCould not find module ) splitr~r rrziprKrrtrainappendrI) r2rrtokensupdated_modulesrjrk name_tokensmatching_tokens parameterrrr__up_toEs&    z Model.__up_tocC|j|ddS)acFreeze model up to specific module Parameters ---------- module_name : str Name of module (included) up to which the model will be frozen. Returns ------- frozen_modules : list of str List of names of frozen modules Raises ------ ValueError when requested module does not exist Note ---- The order of modules is the one reported by self.summary("full"). If your model does not follow a sequential structure, you might want to use freeze_by_name for more control. Fr _Model__up_tor2rrrr freeze_up_togzModel.freeze_up_tocCr)aiUnfreeze model up to specific module Parameters ---------- module_name : str Name of module (included) up to which the model will be unfrozen. Returns ------- unfrozen_modules : list of str List of names of frozen modules Raises ------ ValueError when requested module does not exist Note ---- The order of modules is the one reported by self.summary("full"). If your model does not follow a sequential structure, you might want to use freeze_by_name for more control. Trrrrrrunfreeze_up_torzModel.unfreeze_up_toTmoduleslist[str] | strrc Cst}t|tr |g}|D]}t||}|jddD]}||_q||||q tt|t|}|r?t d|d|S)z7Helper function for freeze_by_name and unfreeze_by_nameTrz&Could not find the following modules: rn) r~rGrvgetattrrrrrrJrI) r2rrrrrjrkrmissingrrr __by_names    zModel.__by_namestr | list[str]cC|j||ddS)a#Freeze modules Parameters ---------- modules : list of str, str Name(s) of modules to freeze recurse : bool, optional If True (default), freezes parameters of these modules and all submodules. Otherwise, only freezes parameters that are direct members of these modules. Returns ------- frozen_modules: list of str Names of frozen modules Raises ------ ValueError if at least one of `modules` does not exist. Frr_Model__by_namer2rrrrrfreeze_by_names zModel.freeze_by_namecCr)a/Unfreeze modules Parameters ---------- modules : list of str, str Name(s) of modules to unfreeze recurse : bool, optional If True (default), unfreezes parameters of these modules and all submodules. Otherwise, only unfreezes parameters that are direct members of these modules. Returns ------- unfrozen_modules: list of str Names of unfrozen modules Raises ------ ValueError if at least one of `modules` does not exist. Trrrrrrunfreeze_by_nameszModel.unfreeze_by_namePath | str | io.BytesIOrm subfolder str | Nonerevisionrstr | bool | None cache_dirPath | str | NoneOptional['Model']c Kst|tjs tj|r|durtd|} d} nNtj|r?|dur'td|r4t||t j j } nt|t j j } d} n(t |}d|vrKtdt |t j ||||d} | dur\dS|dre|nd} |durqd d } | }t| |d d } | d d} td | d id| d dd}t|}| d dd}t||}zt| tjr| d|j| f||d d|}Wn;ty}z/dt |vrt| tjr| dd}t||j| f|d d d|}|WYd}~S|d}~ww| |_t||S)uLoad pretrained model Parameters ---------- checkpoint : Path, str, or byte buffer Model checkpoint, provided as one of the following: * path to a local `pytorch_model.bin` model checkpoint * path to a local directory containing such a file * identifier of a model on Huggingface hub * pre-loaded model checkpoint as a byte buffer map_location: optional Same role as in torch.load(). Defaults to `lambda storage, loc: storage`. strict : bool, optional Whether to strictly enforce that the keys in checkpoint match the keys returned by this module’s state dict. Defaults to True. subfolder : str, optional Folder inside the hf.co model repo. revision : str, optional Revision when loading from the huggingface.co model hub. token : str or bool, optional Token to be used for the download. cache_dir: Path or str, optional Path to the folder where cached files are stored. kwargs: optional Any extra keyword args needed to init the model. Can also be used to override saved hyperparameter values. Returns ------- model : Model Model See also -------- `huggingface_hub.hf_hub_download` Nz0Revisions cannot be used with local checkpoints.local@z:Revisions must be passed with `revision` keyword argument.)rrrr)z pyannote/z pyannoteai/ huggingfacecSs|Sr6r)storagelocrrrdefault_map_locationTsz3Model.from_pretrained..default_map_locationF) map_location weights_onlyrrr$rrkrr)rrmr loss_funczModel has been trained with a task-dependent loss function. Set 'strict' to False to load the model without its loss function and prevent this warning from appearing. )rGioBytesIOospathisfilerIisdirrrr$valuervrlower startswithpl_loadrrrseekload_from_checkpointrurxry _otel_originr)clsrrrmrrrrrpath_to_model_checkpoint otel_originrloaded_checkpointrrrk class_nameKlassr{r>rrrrfrom_pretraineds2           zModel.from_pretrained)r%r&N)r'rr(rr)r*)r5r)r)r)r5r;)r9r;r6)r@rOr5rP)r5rP)r5r)rr)rrPr5r)r5r)r5r)F)rrvrrr5r)rrvr5r)TF)rrrrrrr5r)T)rrrrr5r)rrrrr5r)NTNNNN)rrrmrrrrrrrrrr5r)#rrr__doc__r.propertyr)setterr:r9deleterrXrYrrbrcrqrrrrrrrrrrrrrrr classmethodr __classcell__rrr3rr$Esf         8    !   "  ! " r$)4 __future__rrrrx dataclassesr functoolsr importlibrpathlibrtypingrr lightningrTtorch.nnr torch.optim#lightning.fabric.utilities.cloud_ior r7lightning.pytorch.utilities.model_summary.model_summaryr pyannote.audior pyannote.audio.core.ior pyannote.audio.core.taskr rrrpyannote.audio.telemetryr!pyannote.audio.utils.dependenciesrpyannote.audio.utils.hf_hubrrpyannote.audio.utils.multi_taskr pyannote.corertorch.utils.datarrrLightningModuler$rrrrs8