o }oi< @sddlZddlZddlZddlZejdejjddZejdejjddZejdejjddZ ejdejjd d Z ejdejjd d Z ejdejjd dZ ejdejjddZ ejdejjddZdS)NGPUcCszddlm}WntytdYdSwt}||d}||}td}| D]}| |s=Jd|dq.dS)Nr TensorRTLLMECould not import TRTLLM helpers. tensorrt_llm is likely not installed)z&model.embedding.word_embeddings.weightz8model.decoder.layers.0.self_attention.linear_proj.weightz^model\.[^.].*zKey 'z'' does not properly start with 'model.') nemo.export.tensorrt_llmr ImportErrorpytestskipobject"get_nemo_to_trtllm_conversion_dictrecompilekeysmatch)r dummy_statemodel_state_dictnemo_model_conversion_dictpatternkeyrR/home/ubuntu/.local/lib/python3.10/site-packages/tests/export/test_tensorrt_llm.py5test_get_nemo_to_trtllm_conversion_dict_on_nemo_models     rcCsfzddlm}ddlm}WntytdYdSwt}||d}||}||ks1JdS)Nr)DEFAULT_CONVERSION_DICTrr)z embedding.word_embeddings.weightz2decoder.layers.0.self_attention.linear_proj.weight) Jmegatron.core.export.trtllm.model_to_trllm_mapping.default_conversion_dictrrrrrr r r )rrrrrrrr6test_get_nemo_to_trtllm_conversion_dict_on_mcore_model0s    rcCszddlm}WntytdYdSwd}||dd}|j|ks'J|jtj |dks3J|j dus:J|j dusAJ|j dusHJdd g}|||dd }|j |ksZJ||ddddd }|jdusjJ|jdusqJ|jdusxJdS) Nrrr/tmp/test_model_dirF model_dir load_model trtllm_enginez/path/to/lora1z/path/to/lora2)rlora_ckpt_listr)ruse_python_runtimeenable_chunked_contextmax_tokens_in_paged_kv_cacher)rrrrr r engine_dirospathjoinmodel tokenizerconfigr r!r"r#)rrtrt_llmr rrr test_tensorrt_llm_initializationFs4   r,cCszddlm}WntytdYdSwd}||dd}|j}t|ts*Jt|dks2Jt dd|Ds=J|j }t|t sGJt|dksOJdS) NrrrrFrcss|]}t|tVqdS)N) isinstancestr).0r(rrr zsz5test_tensorrt_llm_supported_models..) rrrrr get_supported_models_listr-listlenallget_supported_hf_model_mappingdict)rrr+supported_models hf_mappingrrr"test_tensorrt_llm_supported_modelsjs   r9cCszddlm}WntytdYdSwd}||dd}ddlm}tj|jftj |j ftj |j fg}|D]\}}| |}||ksSJd|d |d |q9dS) NrrrrFr)DataTypez Expected z for z, got ) rrrrr megatron.core.export.data_typer:torchfloat32float16bfloat16get_input_dtype)rrr+r: test_cases storage_dtypeexpected_dtype input_dtyperrrtest_tensorrt_llm_input_dtypes"         $rEcCs|zddlm}WntytdYdSwd}||dd}|j}|dur6t|ts.J|dks4JdS|duss8       "