o }oiw @sddlZddlZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z ddl m Z ddlmZmZmZmZmZmZmZmZddlZddlmmmZddlmZddlmZ ddl!m"Z#dd l!m$Z$dd l%m&Z&dd l'm(Z(dd l)m*Z*dd l+m,Z,ddl-m.Z.ddl/m0Z1ddl2m3Z3ddl4m5Z5ede.dZ6edZ7e1e8Z9dede:fddZ;de#jfeffddZ?dJddZ@dd ZAd!d"ZBGd#d$d$ZCGd%d&d&ZDdKd'eee*fd(d)ZEdee>effd*d+ZFdeje fd?d@ZNdAeje dEee7dFee>dGe:de7f dHdIZdS)LN)deepcopy)Path)locate)AnyCallableDictListOptionalTypeTypeVarUnion)dump)loadconfig)partial) serialization)Self)Artifact) IOProtocol)ModelConnector)enable) to_config)loggingConnT)boundCkptTypeargreturncCs |tjkSN) dataclasses_HAS_DEFAULT_FACTORY)rr"K/home/ubuntu/.local/lib/python3.10/site-packages/nemo/lightning/io/mixin.py_is_default_factory5s r$datacCstj|dd}|D]5\}}t|tjrt|}|||<q t|r@t|j r@t |j }|D]}|j |kr?| ||<nq0q d|vrIt dtt|jdt|j|d<t|tjrgd|d<|S)NT)include_defaults __fn_or_cls__z]It is not supported to dump objects of functions/classes that have a __fn_or_cls__ parameter.._target_ _partial_) config_libordered_argumentsitems isinstanceConfig_ordered_arguments_with_defaultr$r is_dataclassr'fieldsnamedefault_factory ValueErrorinspect getmodule get_callable__name__ __qualname__rPartial)r%resultkeyr ordered_argr2fieldr"r"r#r09s.      " r0r/cCst|}||S)z(Returns a YAML representation of `data`.)r0represent_data)dumperr% type_namevaluer"r"r#!_config_representer_with_defaultsUs rDcCst||ddS)Nr;)rB)rD)rAr%r"r"r#"_partial_representer_with_defaults[srEcCsnz|}t|jd|j}d}Wnty,|j}t|jd|j}d}Ynw||d}||S)a Represent a given object as YAML using the specified dumper. This function is a fallback for objects that don't have specific representers. If the object has __qualname__ attr, the __target__ is set to f"{inspect.getmodule(obj).__name__}.{obj.__qualname__}". If the object does not have a __qualname__ attr, the __target__ is set from its __class__ attr. The __call__ key is used to indicate whether the target should be called to create an instance. Args: dumper (yaml.Dumper): The YAML dumper to use for serialization. data (Any): The data to serialize. This can be any Python object, but if it's a class or a class instance, special handling will be applied. Returns: str: The YAML representation of the data. r(FT)r)_call_)r6r7r9r:AttributeError __class__r@)rAr%objtargetcallrCr"r"r#_safe_object_representer_s  rLc@seZdZUdZejeed<ddZddZ de e e ffdd Z dejefd d Zedeefd d Zdedee fddZdejdee fddZdS)IOMixina A mixin class designed to capture the arguments passed to the `__init__` method, facilitating the re-creation of the object through `io.reinit` method using stored configurations. This class intercepts the initialization of an object to store the arguments in a configuration object, which can be serialized and later used to reinitialize the object to its original state. It utilizes `fdl.Config` from the Fiddle library to create a structured configuration object that holds the initialization parameters. This configuration object is crucial for enabling serialization and deserialization of the parameters, thus allowing the object to be reconstructed at a later time with the same initial state. Attributes ---------- __io__ (fdl.Config[Self]): A configuration object that stores the captured initialization parameters in a structured format. This object is an instance of `fdl.Config`, which allows for the serialization and deserialization of the parameters, enabling the object to be reconstructed at a later time with the same initial state. Examples -------- from nemo.lightning import io class ExampleClass(io.IOMixin): def __init__(self, param1, param2): super().__init__() self.param1 = param1 self.param2 = param2 # Creating an instance of ExampleClass example = ExampleClass('value1', 'value2') example_copy = io.reinit(example) Note: For more information on `fdl.Config`, refer to the Fiddle library documentation at [Fiddle Config Documentation](https://fiddle.readthedocs.io/en/latest/api_reference/core.html#config). __io__cOst|}t|}|S)a Overrides the default object creation process to wrap the `__init__` method, allowing initialization arguments to be captured and stored in the `__io__` attribute. Args: *args: Variable length argument list for the `__init__` method. **kwargs: Arbitrary keyword arguments for the `__init__` method. Returns ------- The newly created object instance. ) _io_wrap_initobject__new__)clsargskwargsoutputr"r"r#rQs zIOMixin.__new__cCs t|dSr)_io_register_serializationrRr"r"r#__init_subclass__s zIOMixin.__init_subclass__rcOst||g|Ri|S)aA Transforms and captures the arguments passed to the `__init__` method, filtering out any arguments that are instances of `IOProtocol` or are dataclass fields with default factories. Args: init_fn (Callable): The original `__init__` method of the class. *args: Variable length argument list for the `__init__` method. **kwargs: Arbitrary keyword arguments for the `__init__` method. Returns ------- Dict[str, Any]: A dictionary of the captured and transformed arguments. )_io_transform_args)selfinit_fnrSrTr"r"r#io_transform_argsszIOMixin.io_transform_argscKst|fi|S)a3 Initializes the configuration object (`__io__`) with the captured arguments. Args: **kwargs: A dictionary of arguments that were captured during object initialization. Returns ------- fdl.Config[Self]: The initialized configuration object. )_io_init)rZrTr"r"r#io_inits zIOMixin.io_initcCsgS)zInitialize io artifactsr"rWr"r"r# io_artifactsszIOMixin.io_artifactsrU yaml_attrscCst|}d}||}|jddd|t_|t_|d}t|d}t|t|j||}t |} | | Wdn1s?wY|j ||d} | D]\} } || d} | | qOt`t`t|sqt|dSdS) a Serializes the configuration object (`__io__`) to a file, allowing the object state to be saved and later restored. Also creates an artifacts directory and stores it in a thread-local global variable. If the artifacts directory is empty at the end, it is deleted. Args: output (Path): The path to the directory where the configuration object and artifacts will be stored. r(T)parentsexist_okio.jsonwN)attrsz.yaml)rmkdir _thread_locallocal_artifacts_dir output_pathopen_artifact_transform_saverrNr dump_jsonwrite _io_dump_yamlr- write_textanyiterdirshutilrmtree)rZrUr`rirh artifacts_dir config_pathfiojson yaml_configsattrserialized_str_pathr"r"r#io_dumps*      zIOMixin.io_dumprwrec Csddl}|jj}ddlm}m}ddlm}|j t jt |j t jt |j |t |j |t |jtt|}i} |D] } |t|| | | <qD||j_| S)Nr)r/r;)YamlSerializer)yaml SafeDumperyaml_representerscopynemo_run.configr/r; nemo_run.core.serialization.yamlr~add_representerr+rDrrEadd_multi_representerrPrL serializegetattr) rZrwreroriginal_representersr/r;r~ serializerr<rzr"r"r#rn s  zIOMixin._io_dump_yamlN)r9 __module__r:__doc__fdlr/r__annotations__rQrXrstrrr\r^ classmethodrrr_rlistr}r+rnr"r"r"r#rMs ' &rMc @sJeZdZUdZiZeeeefe d<iZ eeeefe d<e dede fddZ e dd ed eedeeegeeffd d Ze dd ed eedeeegeeffd dZe dedefddZe d edeeefdefddZddededeedefddZe dd eeefdeeeefdedefddZdS)ConnectorMixina A mixin class that provides methods to register and retrieve model connectors for importing and exporting models. This class supports dynamic registration of connectors based on file extensions, which facilitates the customization and extension of model serialization and deserialization processes. Attributes ---------- _IMPORTERS (Dict[str, Type[ModelConnector]]): A dictionary mapping file extensions to model connector classes that handle the import process. _EXPORTERS (Dict[str, Type[ModelConnector]]): A dictionary mapping file extensions to model connector classes that handle the export process. _IMPORTERS _EXPORTERSpathrcCs||}|||_|S)a Creates an instance of a model by using the appropriate importer based on the file extension of the provided path. Args: path (str): The path to the model file to be imported. Example: from nemo.collections import llm model = llm.Mistral7BModel.import_from("hf") Returns ------- Self: An instance of the model initialized from the imported data. )_get_connectorinit import_ckpt ckpt_path)rRrrUr"r"r# import_from3s zConnectorMixin.import_fromNext default_pathc&dttdttffdd }|S)a A class method decorator to register a model connector as an importer for a specific file extension. Args: ext (str): The file extension to associate with the model connector. default_path (Optional[str]): The default path to use if no path is specified during import. Returns ------- Callable[[Type[ConnT]], Type[ConnT]]: The decorator that registers the model connector. connectorrc |jt<r|_|Sr)rrrrrRrrr"r# decoratorXz3ConnectorMixin.register_importer..decoratorr rrRrrrr"rr#register_importerI"z ConnectorMixin.register_importercr)a A class method decorator to register a model connector as an exporter for a specific file extension. Args: ext (str): The file extension to associate with the model connector. default_path (Optional[str]): The default path to use if no path is specified during export. Returns ------- Callable[[Type[ConnT]], Type[ConnT]]: The decorator that registers the model connector. rrcrr)rrrrrr"r#rorz3ConnectorMixin.register_exporter..decoratorrrr"rr#register_exporter`rz ConnectorMixin.register_exportercCs|j|ddS)aJ Retrieves the appropriate model connector for importing based on the extension of the provided path. Args: path (str): The path to the model file to be imported. Returns ------- ModelConnector: The model connector instance capable of handling the import. Timporterr)rRrr"r"r#rws zConnectorMixin.importercCs|j||ddS)a Retrieves the appropriate model connector for exporting based on the extension. Args: ext (str): The file extension associated with the model connector. path (Union[str, Path]): The path where the model will be exported. Returns ------- ModelConnector: The model connector instance capable of handling the export. Frr)rRrrr"r"r#exporters zConnectorMixin.exporterF overwrite base_pathcKs8|j|fi|}|j|d}|||d}|||S)a Imports a checkpoint from a specified path, potentially overwriting existing files. Args: path (str): The path to the checkpoint file to be imported. overwrite (bool): Flag to determine if existing files should be overwritten (default is False). base_path (Optional[Path]): The base path where the checkpoint file is located; used to resolve relative paths. Returns ------- Path: The path to the imported checkpoint. Raises ------ FileNotFoundError: If the checkpoint file does not exist at the specified path. )r)r)r local_pathon_import_ckpt)rZrrrrTrrr"r"r#rs    zConnectorMixin.import_ckptTrcKsd}t|}d|vr|d\}}nt|}|r"|jt||n |jt||}|s8td|d||sI|jsFtd|dd|S||fi|S)a  Retrieves the appropriate model connector based on the file extension and path, distinguishing between importers and exporters. Args: ext (Union[str, Path]): The file extension or a URI that may include a protocol specifier. path (Optional[Union[str, Path]]): The path where the model file is located or will be saved. importer (bool): Flag to determine if the connector is for importing (True) or exporting (False). Returns ------- ModelConnector: The model connector instance capable of handling the import or export. Raises ------ ValueError: If no connector is found for the specified extension or if no default path is provided when required. Nz://z"No connector found for extension 'z' for z)No default path specified for extension 'z'. zPlease provide a path)rsplitrgetrr5r)rRrrrrTr|rr"r"r#rs,zConnectorMixin._get_connectorr)FN)NT)r9rr:rrrrr rrrrrrr rrrrrr rrboolrrr"r"r"r#r!s4 00   r artifactscsPfddfdd}ddt|tjr||St|r$|Std)a; Adds IO functionality to the target object or eligible classes in the target module by wrapping __init__ and registering serialization methods. Args: target (object or types.ModuleType): The target object or module to modify. Returns: object or types.ModuleType: The modified target with IO functionality added to eligible classes. Examples: >>> from nemo.collections.common import tokenizers >>> modified_tokenizers = track_io(tokenizers) >>> ModifiedWordTokenizer = track_io(tokenizers.WordTokenizer) c s\t|r,t|dr,t|ds,|ttttttt t dfvr|St |}t |p*g|_ |S)N__init__rN)r6isclasshasattrrintfloattuplerdictrtyperOrV__io_artifacts__rW)rr"r#_add_io_to_classs z"track_io.._add_io_to_classcs<t|D]\}}t|r||rt|||q|Sr)r6 getmembersrsetattr)moduler3rI)r#_is_defined_in_module_or_submodulesr"r#_process_modules z!track_io.._process_modulecSs |j|jkp|j|jdS)Nr()rr9 startswith)rIrr"r"r#rs z5track_io.._is_defined_in_module_or_submodulesz"Target must be a module or a class)r.types ModuleTyper6r TypeError)rJrrr")rrrr#track_ios    rc Ost|}|j|g|Ri|}dd|jD}g}|D]/}t||tr/||j||<t ||rAt j ||dd||<||j j dkrN||q|D]}||=qQ|S)a Transforms and captures the arguments passed to the `__init__` method, filtering out any arguments that are instances of `IOProtocol` or are dataclass fields with default factories. Args: init_fn (Callable): The original `__init__` method of the class. *args: Variable length argument list for the `__init__` method. **kwargs: Arbitrary keyword arguments for the `__init__` method. Returns ------- Dict[str, Any]: A dictionary of the captured and transformed arguments. cSsi|] \}}|dkr||qS)rZr").0kvr"r"r# sz&_io_transform_args..T)allow_post_init_HAS_DEFAULT_FACTORY_CLASS)r6 signature bind_partial argumentsr-r.rrNr r1fdl_dcconvert_dataclasses_to_configsrHr9append) rZr[rSrTsig bound_args config_kwargsto_delr=r"r"r#rYs  rYc Ks^z tjt|fi|WSty.}zdt|jdt|d|d}t||d}~ww)a Initializes the configuration object (`__io__`) with the captured arguments. Args: **kwargs: A dictionary of arguments that were captured during object initialization. Returns ------- fdl.Config[Self]: The initialized configuration object. zError creating fdl.Config for : z" Arguments that caused the error: zH This may be due to unsupported argument types or nested configurations.N)rr/r Exceptionr9r RuntimeError)rZrTe error_msgr"r"r#r]&s  r]cs<|jt|ddr |Stfdd}||_d|_|S)z=Wraps the __init__ method of a class to add IO functionality.__wrapped_init__Fcst|dr|jg|Ri|}n t|g|Ri|}t|dr-|jdi||_n t|fi||_|g|Ri|dS)Nr\r^r")rr\rYr^rNr])rZrSrT cfg_kwargs original_initr"r# wrapped_initCs  z#_io_wrap_init..wrapped_initT)rr functoolswrapsr)rRrr"rr#rO<s  rOcCstj|tttddS)N) flatten_fn unflatten_fnpath_elements_fn)rregister_node_traverser_io_flatten_object_io_unflatten_object_io_path_elements_fnrWr"r"r#rVUs  rVc Cszt|jWnWtjtfy_}zHttdrttds|ttjt }tj }||}t |d}t t|d||Wdn1sIwYt|fdfWYd}~Sd}~ww|jS)NrhriwbrN)rrlrNUnserializableValueErrorrGrrgrrhuuiduuid4rirjr rr __flatten__)instancerlocal_artifact_pathri artifact_pathrvr"r"r#r^s  rcCszttds tj||Stj}t|dkr6|d}tt||d }t |WdS1s1wYtj||S)N output_dirrrb) rrgrr/ __unflatten__rlenrjr pickle_load)valuesmetadatar pickle_pathrvr"r"r#ros   rc Cs>zt|jWntjtfytfYSw|jSr)rrlrNrrGIdentityElement__path_elements__)xr"r"r#r}s  rr(cfgri relative_dirc Cs&t|jdg}|D]G}|jst||js|jsq t||js*|jr*td|jdt||j}|durA|jr@td|jdq |||||}t||j|q t |D];}zt||} Wn t ygYqUwt | } zt | t jt jfrt||tt||d| ||dWqUtyYqUw|S)Nrz Artifact 'z' is required but not provided)rir)rr'skiprrzrequiredr5r rdirrGrr.rr/r;rk) rrrirrartifact current_valnew_valrzchildr"r"r#rksJ     rkrc Cst|jdgD]8}t||j}t|tjr!t||jt|jq|jr%qt||j}|dur0qt t ||}t||j|qt |D]}ztt||tjrYt t|||dWqDt ycYqDwdS)Nr)r)rr'rzr.rr/rbuildrrrr_artifact_transform_loadr5)rrrrrrzr"r"r#r s*     r rcs,ddtjdtffdd |dS)a  Analyzes config to detect unexpected keyword arguments -- for example, deprecated parameters -- and updates the config by dropping them. Returns True if the config gets updated and False otherwise. Args: config (fdl.Config): The configuration object to analyze. Frprefixcst|tjrYt|jtddjD}|s=fdd|j D}|rsz:drop_unexpected_params..analyze..csg|] }|jvr|qSr") parametersrrr"r# sz;drop_unexpected_params..analyze..Tz#Deprecated parameters to drop from rzSkip analyzing z+ as it accepts arbitrary keyword arguments.r()r.rr/r6rr'rprr __arguments__rwarningdebugr-)rr  accept_kwargsto_droprr=rCanalyzeupdatedrr#rs   z'drop_unexpected_params..analyzez)rr/rrr"rr#drop_unexpected_paramss  rT output_typesubpathr c st|}|t_t|dr|rt|d}nt|dr%|jr%t|d}|s1td|dr7dt| }t |}Wdn1sKwY| di D]4\}}d |d d |d d g} rd |vrttfdd|d rqXtt| stt| qXt|d}t |} Wdn1swYd} | di D]\}}d |vrň|d vr|} nqr| stdd|d|| r| | dd<t| j} t| |t| |s| St| S)a Loads a configuration from a pickle file and constructs an object of the specified type. Args: path (Path): The path to the pickle file or directory containing 'io.pkl'. output_type (Type[CkptType]): The type of the object to be constructed from the loaded data. subpath (Optional[str]): Subpath to selectively load only specific objects inside the output_type. Defaults to None. Returns ------- CkptType: An instance of the specified type constructed from the loaded configuration. Raises ------ FileNotFoundError: If the specified file does not exist. Example: loaded_model = load("/path/to/model", output_type=MyModel) is_dirrcisdirzNo such file: ''z.Nobjectsr(rrr3pathscs|vSrr")prr"r#szload..rzCould not find z for z in rootr=)rrgrrr r!is_fileFileNotFoundErrorrjrxrrr-joinallmaprfind_node_traverserrrloadsreadrrDeserializationr<r rrr ) rrrr r|rvjrIvalclss json_configroot_keyrr"r&r#rsP          r)r/r)Pr rr6rxrr threadingrrrrpathlibrpydocrtypingrrrrr r r r fiddler$fiddle._src.experimental.dataclasses_src experimentalr cloudpickler rr fiddle._srcrr+rfiddle._src.experimentalrtyping_extensionsrnemo.lightning.io.artifact.basernemo.lightning.io.capturernemo.lightning.io.connectorrnemo.lightning.io.fdl_torchr _enable_extnemo.lightning.io.to_configr nemo.utilsrrrlocalrgrr$r/rrr0rDrErLrMrrrYr]rOrVrrrrkr rr"r"r"r#sf   (             $ "!7,#  ,2&