Create README.md

README.md

@@ -0,0 +1,93 @@
+## Usage
+This model is quantized using [AutoGPTQ](https://github.com/AutoGPTQ/AutoGPTQ) for [THUDM/glm-4v-9b](https://huggingface.co/THUDM/glm-4v-9b). 
+
+The quantization script will be released later
+
+### Load model
+```python
+import os
+
+import json
+import random
+import time
+
+import torch
+import datasets
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from auto_gptq import AutoGPTQForCausalLM
+
+device = 'cuda:0'
+quantized_model_dir = 'alexwww94/glm-4v-9b-gptq'
+trust_remote_code = True
+
+tokenizer = AutoTokenizer.from_pretrained(
+    quantized_model_dir,
+    trust_remote_code=trust_remote_code,
+)
+
+model = AutoGPTQForCausalLM.from_quantized(
+    quantized_model_dir,
+    device=device,
+    trust_remote_code=trust_remote_code,
+    torch_dtype=torch.float16,
+    use_cache=True,
+    inject_fused_mlp=True,
+    inject_fused_attention=True,
+)
+```
+
+You can also load the model using HuggingFace Transformers, but it will slow down inference.
+
+```python
+model = AutoModelForCausalLM.from_pretrained(
+    quantized_model_dir,
+    torch_dtype=torch.float16,
+    device_map="auto",
+    low_cpu_mem_usage=True,
+    trust_remote_code=trust_remote_code,
+    use_cache=True
+).eval()
+```
+
+### inference test
+Load the CogVLM-SFT-311K-subset-gptq dataset as test data, which is a dataset for quantization.
+
+```python
+dataset = datasets.load_dataset('alexwww94/CogVLM-SFT-311K-subset-gptq')
+
+for example in dataset['single']:
+    # prompt = "为什么马会被围栏限制在一个区域内？"
+    prompt = json.loads(example['labels_zh'])['conversations'][0]
+    answer = json.loads(example['labels_zh'])['conversations'][1]
+    image = example['image']
+    print(f"prompt: {prompt}")
+    print("-" * 42)
+    print(f"golden: {answer}")
+    print("-" * 42)
+
+    start = time.time()
+
+    prompt.update({'image': image})
+    inputs = tokenizer.apply_chat_template([prompt],
+                                    add_generation_prompt=True, tokenize=True, return_tensors="pt",
+                                    return_dict=True, dtyp=torch.bfloat16)  # chat mode
+    inputs = inputs.to(device)
+    inputs['images'] = inputs['images'].half()
+
+    gen_kwargs = {"max_length": 2500, "do_sample": True, "top_k": 1}
+    with torch.inference_mode():
+        outputs = model.generate(**inputs, **gen_kwargs)
+        outputs = outputs[:, inputs['input_ids'].shape[1]:]
+        generated_text = tokenizer.decode(outputs[0]).split('<|endoftext|>')[0]
+
+    end = time.time()
+    print(f"quant: {generated_text}")
+    num_new_tokens = len(tokenizer(generated_text)["input_ids"])
+    print(f"generate {num_new_tokens} tokens using {end-start: .4f}s, {num_new_tokens / (end - start)} tokens/s.")
+    print("=" * 42)
+
+    # break
+```
+
+### metrics
+(to be released later)