o
    ॵiek                     @   s   d dl Z d dlZd dlmZ d dlZd dlZd dlZd dlZd dl	Z	d dl
mZ d dl	mZ d dlmZmZ d dlZd dlZd dlmZmZ d dlmZ e ZdZdZd	Zg d
ZG dd dejZG dd dZddeeef fddZG dd dej Z!dS )    N)Path)FunctionType)DictUnion)ConfigFields	ModelFile)
get_logger_base__delete__deprecation_)filenametextpretty_textc                       s(   e Zd ZdZdd Z fddZ  ZS )
ConfigDictz Dict which support get value through getattr

    Examples:
        >>> cdict = ConfigDict({'a':1232})
        >>> print(cdict.a)
        >>> # 1232
    c                 C   s   t |N)KeyErrorselfname r   K/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/utils/config.py__missing__(   s   zConfigDict.__missing__c              
      sn   zt t| |}W |S  ty"   td| jj d| d}Y | ty6 } z	|}W Y d }~|d }~ww )N'z' object has no attribute ')superr   __getattr__r   AttributeError	__class____name__	Exception)r   r   valueexer   r   r   r   +   s   zConfigDict.__getattr__)r   
__module____qualname____doc__r   r   __classcell__r   r   r"   r   r      s    r   c                       s"  e Zd ZdZedd Zedd Zedd Zd9 fd	d
	Ze	dd Z
e	dd Ze	dd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd  Zd!d" Z fd#d$Z fd%d&Zd:d(efd)d*Zd;d+ef fd,d-Zd< fd/d0	Zed=d2d3Zd4efd5d6Zd>d7d8Z  Z S )?Configaz  A facility for config and config files.

    It supports common file formats as configs: python/json/yaml. The interface
    is the same as a dict object and also allows access config values as
    attributes.

    Example:
        >>> cfg = Config(dict(a=1, b=dict(c=[1,2,3], d='dd')))
        >>> cfg.a
        1
        >>> cfg.b
        {'c': [1, 2, 3], 'd': 'dd'}
        >>> cfg.b.d
        'dd'
        >>> cfg = Config.from_file('configs/examples/configuration.json')
        >>> cfg.filename
       'configs/examples/configuration.json'
        >>> cfg.b
        {'c': [1, 2, 3], 'd': 'dd'}
        >>> cfg = Config.from_file('configs/examples/configuration.py')
        >>> cfg.filename
        "configs/examples/configuration.py"
        >>> cfg = Config.from_file('configs/examples/configuration.yaml')
        >>> cfg.filename
        "configs/examples/configuration.yaml"
    c                 C   s  t t | } t | std|  t | d }|dvr#tdt u}tj	||d}t
 dkr9|  t |j}t| |j | drdd	lm} |t ||\}}i }|j D ]\}	}
|	d
s{t|
tjs{t|
tjs{|
||	< qbtj|= n| drddlm} ||j}|  W d    n1 sw   Y  | d }t | ddd}||! 7 }W d    ||fS 1 sw   Y  ||fS )NzFile does not exists    .py.json.yaml.yml-Only py/yml/yaml/json type are supported now!)dirsuffixWindowsr*   r   )import_modules_from_file__)r-   r,   r+   )load
rutf-8encoding)"ospabspath
expanduserexists
ValueErrorsplitextIOErrortempfileTemporaryDirectoryNamedTemporaryFileplatformsystemclosebasenamer   shutilcopyfileendswithmodelscope.utils.import_utilsr2   join__dict__items
startswith
isinstancetypes
ModuleTyper   sysmodulesmodelscope.fileior4   openread)r   fileExtnametmp_cfg_dirtmp_cfg_filetmp_cfg_namer2   module_nanmemodcfg_dictr   r   r4   cfg_textfr   r   r   
_file2dictT   sR   











zConfig._file2dictc                 C   s.   t | tr	t| } t| \}}t||| dS )N)r_   r   )rP   r   strr'   ra   )r   r^   r_   r   r   r   	from_file   s   
zConfig.from_filec                 C   s   |dvrt d|dkrd| v rtd tjdd|dd	}||  W d
   n1 s.w   Y  t|j}t	
|j |S )a(  Generate config from config str.

        Args:
            cfg_str (str): Config str.
            file_format (str): Config file format corresponding to the
               config str. Only py/yml/yaml/json type are supported now!

        Returns:
            :obj:`Config`: Config obj.
        r)   r.   r*   dict(z6Please check "file_format", the file format may be .pywr7   F)r9   r0   deleteN)r@   loggerwarningrA   rC   writer'   rc   r   osremove)cfg_strfile_format	temp_filecfgr   r   r   from_string   s    zConfig.from_stringNc                    s   |d u rt  }nt|t stdt| |D ]}|tv r%t| dqt|tr/t|}tt	| 
dt| tt	| 
d| |rH|}n |rft|ddd}| }W d    n1 s`w   Y  nd}tt	| 
d	| d S )
Nz!cfg_dict must be a dict, but got z is reserved for config file	_cfg_dict	_filenamer6   r7   r8    _text)dictrP   	TypeErrortypeRESERVED_KEYSr   r   rb   r   r'   __setattr__r   rV   rW   )r   r^   r_   r   keyr   r`   r"   r   r   __init__   s.   


zConfig.__init__c                 C      | j S r   )rr   r   r   r   r   r         zConfig.filenamec                 C   r|   r   )rt   r}   r   r   r   r      r~   zConfig.textc                    s   ddd dfdd	dfdd	d	d
  d fdd	| j  }|dd}tdddd}ddlm} |||dd\}}|S )N   c                    sP   |  d}t|dkr| S |d} fdd|D }d|}|d | }|S )Nr5   r(   r   c                    s   g | ]} d  | qS ) r   ).0line
num_spacesr   r   
<listcomp>   s    z7Config.pretty_text.<locals>._indent.<locals>.<listcomp>)splitlenpoprL   )s_r   sfirstr   r   r   _indent   s   


z#Config.pretty_text.<locals>._indentFc                    sr   t |trd| d}nt|}|r)t | trd|  dnt| }| d| }n	t|  d| } |}|S )Nr   : =)rP   rb   kvuse_mappingv_strk_strattr_str)r   indentr   r   _format_basic_types   s   

z/Config.pretty_text.<locals>._format_basic_typesc                    s   t dd |D rId}|dfdd|D d7 }|r7t| tr+d|  dnt| }| d| }n	t|  d	| }|d
 }|S  | ||}|S )Nc                 s   s    | ]}t |tV  qd S r   )rP   ru   )r   _r   r   r   	<genexpr>   s    z;Config.pretty_text.<locals>._format_list.<locals>.<genexpr>z[
r5   c                 3   s&    | ]}d  | dV  qdS )rd   z),Nr   )r   v_)_format_dictr   r   r   r   r      s
    
,r   r   r   ])allrL   rstriprP   rb   r   )r   r   r   r   r   r   _format_list   s   z(Config.pretty_text.<locals>._format_listc                 S   s$   d}| D ]}|t |  O }q|S )NF)rb   isidentifier)dict_strcontain_invalid_identifierkey_namer   r   r   _contain_invalid_identifier   s   z7Config.pretty_text.<locals>._contain_invalid_identifierc                    s  d}g } | }|r|d7 }t |  D ]h\}\}}|t| d k}|s&|r(dnd}	t|trad| }
|rNt|trBd| dnt|}| d|
 }n	t| d|
 }|d	 |	 }nt|tro||||	 }n||||	 }|| q|d|7 }|r|d
7 }|S )Nrs   {r(   r   r5   r   z: dict(z=dict()})		enumeraterN   r   rP   ru   rb   listappendrL   )
input_dictoutest_levelr6   r   r   idxr   r   is_lastendr   r   r   r   r   r   r   r   r   r   r   r      s.   

z(Config.pretty_text.<locals>._format_dictT)r   pep8)based_on_style%blank_line_before_nested_class_or_def+split_before_expression_after_opening_parenr   )
FormatCode)style_configverifyF)rq   to_dictru   yapf.yapflib.yapf_apir   )r   r^   r   
yapf_styler   r   r   r   r   r      s    

zConfig.pretty_textc                 C   s   d| j  d| j  S )NzConfig (path: z): )r   rq   __repr__r}   r   r   r   r   "  s   zConfig.__repr__c                 C   
   t | jS r   )r   rq   r}   r   r   r   __len__%     
zConfig.__len__c                 C   s   t | j|S r   )getattrrq   r   r   r   r   r   (     zConfig.__getattr__c                 C   s   | j |S r   )rq   __getitem__r   r   r   r   r   +  r   zConfig.__getitem__c                 C   $   t |tr	t|}| j|| d S r   )rP   ru   r   rq   ry   r   r   r   r   r   r   ry   .     
zConfig.__setattr__c                 C   r   r   )rP   ru   r   rq   __setitem__r   r   r   r   r   3  r   zConfig.__setitem__c                 C   r   r   )iterrq   r}   r   r   r   __iter__8  r   zConfig.__iter__c                 C   s   | j | j| jfS r   )rq   rr   rt   r}   r   r   r   __getstate__;  s   zConfig.__getstate__c                 C   s"   | j }||}|j| j |S r   )r   __new__rM   update)r   clsotherr   r   r   __copy__>  s   
zConfig.__copy__c                    sN   | j }||}||t| < | j D ]\}}tt||t	|| q|S r   )
r   r   idrM   rN   r   r'   ry   copydeepcopy)r   memor   r   rz   r   r"   r   r   __deepcopy__E  s   
zConfig.__deepcopy__c                    sD   |\}}}t t| d| t t| d| t t| d| d S )Nrq   rr   rt   )r   r'   ry   )r   staterq   rr   rt   r"   r   r   __setstate__O  s   
zConfig.__setstate__rw   	key_chainc           
   
      s   zR| d}| j}|D ]D d}d v r!  d\ }| d\}}t|ttfrAdus0J dtt fdd|}|d }n|  }|durO|t| }q|W S  tyv }	 zt	d	| d
|  t	|	 |W  Y d}	~	S d}	~	ww )a  Get a value with a key-chain in str format, if key does not exist, the default value will be returned.

        This method is safe to call, and will not edit any value.

        Args:
            key_chain: The input key chain, for example: 'train.hooks[0].type'
            default: The default value returned when any key does not exist, default None.
            type_field: Get an object from a list or tuple for example by 'train.hooks.CheckPointHook', in which
                'hooks' is a list, and 'CheckPointHook' is a value of the content of key `type_field`.
                If there are multiple matched objects, the first element will be returned.
        Returns:
            The value, or the default value.
        .N[r   zWGetting object without an index from a list or tuple needs an valid `type_field` param.c                    s   |   kS r   r   )subrz   
type_fieldr   r   <lambda>p  s    z!Config.safe_get.<locals>.<lambda>r   zKey not valid in Config: z, return the default value: )
r   rq   rP   r   tuplefilterintr   rg   debug)
r   r   defaultr   keysrq   valr   _sub_cfg_dictr!   r   r   r   safe_getU  s6   


zConfig.safe_getfilec                    s   ddl m} tt| d }|du r0| jdu s| jdr"| jS | j	dd }|||dS |drUt
|d	d
d}|| j W d   dS 1 sNw   Y  dS |	dd }||||dS )a  Dumps config into a file or returns a string representation of the
        config.

        If a file argument is given, saves the config to that file using the
        format defined by the file argument extension.

        Otherwise, returns a string representing the config. The formatting of
        this returned string is defined by the extension of `self.filename`. If
        `self.filename` is not defined, returns a string representation of a
         dict (lowercased and using ' for strings).

        Examples:
            >>> cfg_dict = dict(item1=[1, 2], item2=dict(a=0),
            ...     item3=True, item4='test')
            >>> cfg = Config(cfg_dict=cfg_dict)
            >>> dump_file = "a.py"
            >>> cfg.dump(dump_file)

        Args:
            file (str, optional): Path of the output file where the config
                will be dumped. Defaults to None.
        r   )dumprq   Nr*   r   )rm   re   r7   r8   )r   rm   )rU   r   r   r'   __getattribute__r   r   rJ   r   r   rV   ri   )r   r   r   r^   rm   r`   r"   r   r   r   ~  s   
"zConfig.dumpTc              	      s   i }|  D ]'\}}|}|d}|dd D ]}	||	t  ||	 }q|d }	|||	< qtt| d}
tt| dtj||
||d dS )a  Merge dict into cfg_dict.

        Merge the dict parsed by MultipleKVAction into this cfg.

        Examples:
            >>> options = {'model.backbone.depth': 50,
            ...            'model.backbone.with_cp':True}
            >>> cfg = Config(dict(model=dict(backbone=dict(type='ResNet'))))
            >>> cfg.merge_from_dict(options)
            >>> cfg_dict = super(Config, self).__getattribute__('_cfg_dict')
            >>> assert cfg_dict == dict(
            ...     model=dict(backbone=dict(type='ResNet', depth=50, with_cp=True)))

            >>> # Merge list element for replace target index
            >>> cfg = Config(dict(pipeline=[
            ...     dict(type='Resize'), dict(type='RandomDistortion')]))
            >>> options = dict(pipeline={'0': dict(type='MyResize')})
            >>> cfg.merge_from_dict(options, allow_list_keys=True)
            >>> cfg_dict = super(Config, self).__getattribute__('_cfg_dict')
            >>> assert cfg_dict == dict(pipeline=[
            ...     dict(type='MyResize'), dict(type='RandomDistortion')])

            >>> # Merge list element for replace args and add to list, only support list of type dict with key ``type``,
            >>> # if you add new list element, the list does not guarantee the order,
            >>> # it is only suitable for the case where the order of the list is not concerned.
            >>> cfg = Config(dict(pipeline=[
            ...     dict(type='Resize', size=224), dict(type='RandomDistortion')]))
            >>> options = dict(pipeline=[dict(type='Resize', size=256), dict(type='RandomFlip')])
            >>> cfg.merge_from_dict(options, allow_list_keys=True)
            >>> cfg_dict = super(Config, self).__getattribute__('_cfg_dict')
            >>> assert cfg_dict == dict(pipeline=[
            ...     dict(type='Resize', size=256), dict(type='RandomDistortion'), dict(type='RandomFlip')])

            >>> # force usage
            >>> options = {'model.backbone.depth': 18,
            ...            'model.backbone.with_cp':True}
            >>> cfg = Config(dict(model=dict(backbone=dict(type='ResNet', depth=50))))
            >>> cfg.merge_from_dict(options, force=False)
            >>> cfg_dict = super(Config, self).__getattribute__('_cfg_dict')
            >>> assert cfg_dict == dict(
            ...     model=dict(backbone=dict(type='ResNet', depth=50, with_cp=True)))

        Args:
            options (dict): dict of configs to merge from.
            allow_list_keys (bool): If True, int string keys (e.g. '0', '1')
              are allowed in ``options`` and will replace the element of the
              corresponding index in the config if the config is a list.
              Or you can directly replace args for list or add new list element,
              only support list of type dict with key ``type``,
              but if you add new list element, the list does not guarantee the order,
              It is only suitable for the case where the order of the list is not concerned.
              Default: True.
            force (bool): If True, existing key-value will be replaced by new given.
                If False, existing key-value will not be updated.
        r   Nr   rq   )allow_list_keysforce)	rN   r   
setdefaultr   r   r'   r   ry   _merge_a_into_b)r   optionsr   r   option_cfg_dictfull_keyr   dkey_listsubkeyr^   r"   r   r   merge_from_dict  s&   8



zConfig.merge_from_dictFc                    s    |  D ]@\r: r:ttr:ttkr,td d tj	 d< qrttrv rt ts_t
dt dt  d d}  D ]}t|trrd|vr~vsxr|< d	}qg|rg }g g  t D ]-\}}tD ]$\}	}
|
d |d kr|tj	|
|d |	  |  nqq fd
dtt D }fddttD }|| |fddttD 7 }|< qttr=v r=td	s=rttfnt}t |s/t d d dt  dt d
tj	 d< qvsErI< qS )a  merge dict ``a`` into dict ``b`` (non-inplace).

        Values in ``a`` will overwrite ``b``. ``b`` is copied first to avoid
        in-place modifications.

        Args:
            a (dict): The source dict to be merged into ``b``.
            b (dict): The origin dict to be fetch keys from ``a``.
            allow_list_keys (bool): If True, int string keys (e.g. '0', '1')
              are allowed in source ``a`` and will replace the element of the
              corresponding index in b if b is a list. Default: False.
            force (bool): If True, existing key-value will be replaced by new given.
                If False, existing key-value will not be updated.

        Returns:
            dict: The modified dict of ``b`` using ``a``.

        Examples:
            # Normally merge a into b.
            >>> Config._merge_a_into_b(
            ...     dict(obj=dict(a=2)), dict(obj=dict(a=1)))
            {'obj': {'a': 2}}

            # Delete b first and merge a into b.
            >>> Config._merge_a_into_b(
            ...     dict(obj=dict(_delete_=True, a=2)), dict(obj=dict(a=1)))
            {'obj': {'a': 2}}

            # b is a list
            >>> Config._merge_a_into_b(
            ...     {'0': dict(a=2)}, [dict(a=1), dict(b=2)], True)
            [{'a': 2}, {'b': 2}]

            # value of a and b are both list, only support list of type dict with key ``type``,
            # You can directly replace args for list or add new list element,
            # but if you add new list element, the list does not guarantee the order,
            # it is only suitable for the case where the order of the list is not concerned.
            >>> Config._merge_a_into_b(
            ...     {'k': [dict(a=2), dict(c=3)]}, {'k': [dict(a=1), dict(b=2)]}, True)
            {'k': [dict(a=2), dict(b=2), dict(c=3)]}

            # force is False
            >>> Config._merge_a_into_b(
            ...     dict(obj=dict(a=2, b=2)), dict(obj=dict(a=1))), True, force=False)
            {'obj': {'a': 1, b=2}}
        zIndex z exceeds the length of list r   ztype mismatch z and z between a and b for key Trw   Fc                    s    g | ]}| vr | qS r   r   r   i)added_index_bkbr   r   r   r   A  s
    
z*Config._merge_a_into_b.<locals>.<listcomp>c                    s   g | ]
}| vr| qS r   r   r   )added_index_vr   r   r   r   E  s    c                    s"   g | ]}t j| i  d qS )r   )r'   r   r   )r   r   restr   r   r   I  s    r   z2 in child config cannot inherit from base because z. is a dict in the child config but is of type z in base config. You may set `z =True` to ignore the base config)r   rN   isdigitrP   r   r   r   r   r'   r   r>   rw   ru   r   r   ranger   
DELETE_KEYrv   )ar   r   r   _is_dict_with_typelist_ires_listr   b_lija_ljrest_bkrest_vallowed_typesr   )r   r   r   r   r   r   r   r   r   r     s   0 






zConfig._merge_a_into_breturnc                 C   s
   | j  S )z. Convert Config object to python dict
        )rq   r   r}   r   r   r   r   _  s   
zConfig.to_dictc                 C   s   g }| j  D ]]\}}d| }|r|dd}t|tr%|r%|| qt|tttfr:|| |t| qt|t	r^|| t|ttttfsVJ d|d  |t| qt
d| ||S )a   Convert config obj to args using parse_fn

        Args:
            parse_fn: a function object, which takes args as input,
                such as ['--foo', 'FOO'] and return parsed args, an
                example is given as follows
                including literal blocks::
                    def parse_fn(args):
                        parser = argparse.ArgumentParser(prog='PROG')
                        parser.add_argument('-x')
                        parser.add_argument('--foo')
                        return parser.parse_args(args)
            use_hyphen (bool, optional): if set true, hyphen in keyname
                will be converted to underscore
        Return:
            args: arg object parsed by argparse.ArgumentParser
        z--r   -zJElement type in list is expected to be either int,str,float, but got type r   ztype in config file which supported to be converted to args should be either bool, int, str, float or list of them but got type )rq   rN   replacerP   boolr   r   rb   floatr   r>   )r   parse_fn
use_hyphenargsr   r   arg_namer   r   r   to_argsd  s,   



zConfig.to_args)NNN)Nrw   r   )TT)FT)T)!r   r#   r$   r%   staticmethodra   rc   rp   r{   propertyr   r   r   r   r   r   r   ry   r   r   r   r   r   r   rb   r   r   r   r   r   r   r  r&   r   r   r"   r   r'   8   sB    
+




^
)&Kor'   Fro   c                    st   t  tr
t  d fdd	}|tj |tj |tj |r8|tj |tj	 |tj
 |tj dS dS )z Check whether configuration file is valid, If anything wrong, exception will be raised.

    Args:
        cfg (str or ConfigDict): Config file path or config object.
        is_training: indicate if checking training related elements
    rs   c                    s*   t  | sJ d|  dtj d| d S )Nz
Attribute z is missing from z. )hasattrr   CONFIGURATION)	attr_namemsgro   r   r   
check_attr  s
   z check_config.<locals>.check_attrN)rs   )rP   rb   r'   rc   r   	frameworktaskpipelinemodeltrainpreprocessor
evaluation)ro   is_trainingr  r   r  r   check_config  s   







r$  c                   @   s   e Zd ZdZdd ZdS )JSONIteratorEncoderzImplement this method in order that supporting arbitrary iterators, it returns
        a serializable object for ``obj``, or calls the base implementation
        (to raise a ``TypeError``).

    c                 C   sB   t |trd S z	t|}W t|S  ty   Y nw tj| |S r   )rP   r   r   rv   r   jsonJSONEncoderr   )r   objiterabler   r   r   r     s   

zJSONIteratorEncoder.defaultN)r   r#   r$   r%   r   r   r   r   r   r%    s    r%  r   )"r   rj   os.pathpathr:   rD   rH   rS   rA   rQ   pathlibr   r   typingr   r   addictr&  modelscope.utils.constantr   r   modelscope.utils.loggerr   rg   BASE_KEYr   DEPRECATION_KEYrx   r   r'   rb   r$  r'  r%  r   r   r   r   <module>   s8       Z