o
    ॵi?                     @   s   d dl Z d dlmZmZmZ d dlZd dlZd dlmZ d dl	m
Z
 d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZmZ d dlmZmZ d dlm Z  e  Z!dgZ"ej#ej$e
j%dG dd deZ&dS )    N)AnyDictUnion)BertTokenizer)	Pipelines)Model)TableQuestionAnswering)
OutputKeys)Pipeline)	PIPELINES)"TableQuestionAnsweringPreprocessor)Database)ConstantSQLQuery)	ModelFileTasks)
get_loggerTableQuestionAnsweringPipeline)module_namec                       s   e Zd Z					ddeeef dedededef
 fd	d
Zdd Z	dd Z
dd Zdeeef deeef fddZdd Z  ZS )r   NgpuTmodelpreprocessordbconfig_filedevicec           	         s  t  j||||||dd|di d t| jts#J dtj |du r2t| jj	fi || _
ttj| jj	tj| _|du r[t| jtj| jj	dtj| jj	dd	| _n|| _t }|j| _|j| _|j| _|j| _|j| _|j| _|j| _|j| _|j| _dS )
a  use `model` and `preprocessor` to create a table question answering prediction pipeline

        Args:
            model (TableQuestionAnswering): a model instance
            preprocessor (TableQuestionAnsweringPreprocessor): a preprocessor instance
            db (Database): a database to store tables in the database
            kwargs (dict, `optional`):
                Extra kwargs passed into the preprocessor's constructor.
        compileFcompile_options)r   r   r   r   auto_collater   r   z,please check whether model config exists in Nz
table.jsonzsynonym.txt)	tokenizertable_file_pathsyn_dict_file_path)super__init__pop
isinstancer   r   r   CONFIGURATIONr   	model_dirr   r   ospathjoin
VOCAB_FILEr   r   r   r   agg_opscond_opscond_conn_ops
action_opsmax_select_nummax_where_numcol_type_dictschema_link_dict
limit_dict)	selfr   r   r   r   r   r   kwargsconstant	__class__ n/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/pipelines/nlp/table_question_answering_pipeline.pyr"       sT   

	

z'TableQuestionAnsweringPipeline.__init__c                 C   s*   | j jdd | j| j | j   dS )zY Place model on certain device for pytorch models before first inference
                iX  )timeoutN)_model_prepare_lockacquirer   tor   release)r4   r9   r9   r:   prepare_model[   s   z,TableQuestionAnsweringPipeline.prepare_modelc                 C   s  | j |d  }|d }|d }|d g|d< |d u r|S |dkr"|S |dkr(|S |dkr.|S |d	kr4|S |d
kr|}g }g }	t|d D ]\}
}||d vr\|| |	|d |
  qDt|dk ro|t| |	d ||d< |	|d< g }|d D ]}|d t|k r|| q}t|dk r|t|ddg ||d< |S |dkr|}g }g }	t|d D ]:\}
}||d v r|| d}t|d D ]\}}|d | |kr|}q|	| q|| |	|d |
  q||d< |	|d< |S |dkrA|}|d }|d }	||d< |	|d< |d D ]+}|d t|k r=d}|d D ]}|d |d kr1d}q$|s=|d | q|S |dkr|}g }t|d D ]\}
}|d |d vrb|| qP||d< g }|d D ]}|d t|k r~|| qnt|dk r|t|ddg ||d< |S |dkr|}g }t|d D ]'\}
}d}|d D ]}|d |d krd}|| q|s|| q||d< g }|d D ]}|d t|k r|| qt|dk r|t|dddg ||d< |S |dkrZ|}|d }t|d D ]\}
}|d t|k r%|| q||d< g }|d D ]}|d t|k rA|| q1t|dk rT|t|ddg ||d< |S |S )Nactionheader_namesqltable_idfromout_of_scriptsswitch_tablerestart	firstTurn	del_focusselagg   r   conds   Nullchange_agg_onlychange_focus_totalFTdel_condchange_condadd_cond)r.   	enumerateappendlen)r4   history_sqlresulttablerA   headerscurrent_sqlpre_final_sqlpre_selspre_aggsidxselifinal_condscondichanged_aggi
idx_singleaggi	pre_condiin_flaghistory_condiin_single_flagsingle_condir9   r9   r:   post_process_multi_turnb   s  















z6TableQuestionAnsweringPipeline.post_process_multi_turnc           "         s  ||d d d  d dg d dg  |d }g g }}g g }}|d D ]}|d	 d
v r8| | q*| | q*|D ]F}|d  }	|	dkrMq@dd  |d  f }
| j|d	  }|d }| d|	 d | d | d  | d|
 d | d | d  q@d| j|d   d }||}||}t|dkr=dfdd|D }d fdd|D }| j|d d	  }d||f }d||f }|d d }d\}}| jD ]}||v rd}| j| } nq|r|d| 7 }|d| 7 }t|d D ]9\}}|t d	 kr:d}td D ]\}}|dkr$|} nq||d vr5||d |< q|d |= qnd}g g }}t|d D ]{\}}| }	dd  | f }
|d | dkrq| |	 | |
 qJ|d | d kr| | j|d |  d! |	 d"  | | j|d |  d! |
 d"  qJ| | j|d |  d# |	 d"  | | j|d |  d# |
 d"  qJt|dkrt|dkrd$d|d% ||f } d&d|d ||f }!nVt|dkrd'd|d% |f } d(d|d |f }!n6t|dkr/d)d|d% |f } d*d|d |f }!nd+d|d% f } d,d|d f }!t| |!|d d-}|S ).z.
        convert sql struct to string
        rC   rE   r   rB   u   空列	header_idnullrN   rM   )      	`%s`.`%s`rD   rO   z(  z "z" )cond_conn_opz, c                    s   g | ]} |d   qS )r   r9   .0cond)header_namesr9   r:   
<listcomp>  s    zBTableQuestionAnsweringPipeline.sql_dict_to_str.<locals>.<listcomp>c                    s$   g | ]}d d  |d  f qS )rs   rD   r   r9   rv   )
header_idsr\   r9   r:   rz     s    z%s %s)FrR   Tz	 LIMIT %drK   header_attributePRIMARY rL   rq   z
(DISTINCT )(z&SELECT %s FROM %s WHERE %s ORDER BY %s
table_namez(SELECT %s FROM `%s` WHERE %s ORDER BY %szSELECT %s FROM %s WHERE %szSELECT %s FROM `%s` WHERE %szSELECT %s FROM %s ORDER BY %szSELECT %s FROM `%s` ORDER BY %szSELECT %s FROM %szSELECT %s FROM `%s`)stringquery
sql_result)	rX   r,   r-   r)   rY   r3   rW   r+   r   )"r4   r[   tablesrC   str_cond_listsql_cond_listwhere_condsorderby_condsrx   rB   ro   opvaluecond_strstr_where_condssql_where_condsstr_orderby_columnsql_orderby_columnstr_orderby_opstr_orderbysql_orderby	limit_keyis_in	limit_numkeyrb   rK   primary_selindexattribstr_sel_listsql_sel_list	final_str	final_sqlr9   )r{   ry   r\   r:   sql_dict_to_str   s   














z.TableQuestionAnsweringPipeline.sql_dict_to_strinputsreturnc              
   C   sr  |d }|d }z| j ||| jj|d  d|d< W n ty'   ||d< Y nw | j|| jjd}| jjrzI| jj |j	}g g }}|j
D ]}|| jj|d  d |d |d  ||d  qHg }	| D ]	}
|	t|
 qm|||	d	}W n# ty } zt| g g g d	}W Y d
}~nd
}~ww g g g d	}tj|jtj|j	tj|d tj|i}tj|iS )zprocess the prediction results

        Args:
            inputs (Dict[str, Any]): _description_

        Returns:
            Dict[str, str]: the prediction results
        r[   rZ   rD   )rZ   r[   r\   rC   )r[   r   headerid2namer   )ro   rB   rowsN)rn   r   r   	Exceptionr   is_use_sqliteconnection_objcursorexecuter   descriptionrX   getfetchalllistloggererrorr	   
SQL_STRINGr   	SQL_QUERYHISTORYQUERY_RESULTOUTPUT)r4   r   r[   rZ   rC   r   r{   ry   r   r   res	tabledataeoutputr9   r9   r:   postprocessf  sX   	





z*TableQuestionAnsweringPipeline.postprocessc                 C   s   |S )Nr9   )r4   datar9   r9   r:   _collate_fn  s   z*TableQuestionAnsweringPipeline._collate_fn)NNNr   T)__name__
__module____qualname__r   r   strr   r   r"   r@   rn   r   r   r   r   r   __classcell__r9   r9   r7   r:   r      s.    
; "o5)'r'   typingr   r   r   jsontorchtransformersr   modelscope.metainfor   modelscope.modelsr   modelscope.models.nlpr   modelscope.outputsr	   modelscope.pipelines.baser
   modelscope.pipelines.builderr   modelscope.preprocessorsr   7modelscope.preprocessors.nlp.space_T_cn.fields.databaser   5modelscope.preprocessors.nlp.space_T_cn.fields.structr   r   modelscope.utils.constantr   r   modelscope.utils.loggerr   r   __all__register_moduletable_question_answering!table_question_answering_pipeliner   r9   r9   r9   r:   <module>   s.   