# You need to merge Qwen2-1.5B-Instruct with fine-tuned LLM lora first, see merge_lora.py.
# Then you can build the engine with the merged model.
# python3 convert_checkpoint.py --model_dir ${model_dir} \
#                               --output_dir ${checkpoint_dir} \
#                               --dtype float16
# We have merged the two models and convert it for trt-llm into the below checkpoint:
checkpoint_dir=tllm_checkpoint_1gpu_fp16_qwen2_1.5B_instruct_merged
# output engine directory
engine_dir=qwen2_1.5B_instruct_fp16_merged

# max_prompt_embedding_table_size should >= max_batch_size * speech_embedding_seq_length
trtllm-build --checkpoint_dir ${checkpoint_dir} \
            --output_dir $engine_dir \
            --max_prompt_embedding_table_size 4096 \
            --max_batch_size 16 \
            --gemm_plugin float16