xianchaowu commited on
Commit
4900ca9
1 Parent(s): 0b28d7a

lazy lora for llama2 13b hf

Browse files
Files changed (4) hide show
  1. README.md +157 -0
  2. adapter_config.json +316 -0
  3. adapter_model.bin +3 -0
  4. usage.py +51 -0
README.md CHANGED
@@ -1,3 +1,160 @@
1
  ---
2
  license: llama2
3
  ---
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  ---
2
  license: llama2
3
  ---
4
+
5
+ ## Lazy LoRA
6
+
7
+ ### Benefits
8
+
9
+ 0. using the updated [Meta's LLaMA-2 models](https://huggingface.co/meta-llama/Llama-2-13b-hf).
10
+ 1. support [4-bit qlora](https://arxiv.org/abs/2305.14314), extreme GPU memory and inference time saving;
11
+ 2. better MMLU evaluation dataset results, llama2-13b's 54.8% to our 57.16% (+2.36%).
12
+ 3. This lazy-lora adapter is based on [Meta's LLaMA-2-13b-hf](https://huggingface.co/meta-llama/Llama-2-13b-hf), and using the [oasst1 dataset](https://huggingface.co/datasets/OpenAssistant/oasst1), following [Guanaco](https://huggingface.co/timdettmers/guanaco-65b).
13
+
14
+ ### Introduction
15
+ Determine the rank of LoRA layers by the singular values of pretrained weight matrices.
16
+ Also, combines:
17
+ 1. LoRA: [LORA: LOW-RANK ADAPTATION OF LARGE LANGUAGE MODELS](https://arxiv.org/abs/2106.09685)
18
+ 2. Prefix Tuning: [Prefix-Tuning: Optimizing Continuous Prompts for Generation](https://aclanthology.org/2021.acl-long.3
19
+ 53/), [P-Tuning v2: Prompt Tuning Can Be Comparable to Fine-tuning Universally Across Scales and Tasks](https://arxiv.or
20
+ g/pdf/2110.07602.pdf)
21
+ 3. Prompt Tuning: [The Power of Scale for Parameter-Efficient Prompt Tuning](https://arxiv.org/abs/2104.08691)
22
+ 4. LLaMA adapter: [LLaMA-Adapter: Efficient Fine-tuning of Language Models with Zero-init Attention] (https://arxiv.org/abs/2303.16199)
23
+ in one model.
24
+
25
+ This allows you to perform LoRA (additional low rank adapters inserted to each linear layer), and prompt learning (additional virtual tokens attached to the input and to the attention layers acting as `past_key_values`)
26
+
27
+ ## Usage:
28
+ ```python
29
+ import sys
30
+ sys.path.insert(1, '/workspace/asr/peft/src')
31
+ # TODO set this path to the lazy-lora source code path,
32
+ # or you can install it from source code:
33
+ # TODO, please install lazylora for usage:
34
+ # git clone git@github.com:Xianchao-Wu/peft.git
35
+ # cd peft
36
+ # python setup.py install
37
+
38
+ from transformers import (AutoTokenizer,
39
+ AutoModelForCausalLM, BitsAndBytesConfig)
40
+ from peft import PeftModel, PeftConfig
41
+ import os
42
+ import torch
43
+
44
+ #import ipdb; ipdb.set_trace()
45
+ cache_dir="/workspace/asr/peft/qlora"
46
+ # TODO set this cache_dir to the path where you
47
+ # stored (or, want to store) llama2-13b-hf model
48
+
49
+ lazylora_dir=os.getcwd()
50
+ # the path that contains 'adapter_config.json'
51
+ # and 'adapter_model.bin'
52
+
53
+ config = PeftConfig.from_pretrained(lazylora_dir)
54
+
55
+ tokenizer = AutoTokenizer.from_pretrained(
56
+ config.base_model_name_or_path,
57
+ cache_dir=cache_dir,
58
+ use_auth_token=True
59
+ )
60
+
61
+ bnb_config = BitsAndBytesConfig(
62
+ load_in_4bit=True,
63
+ bnb_4bit_use_double_quant=True,
64
+ bnb_4bit_quant_type='nf4',
65
+ bnb_4bit_compute_dtype=torch.bfloat16
66
+ )
67
+
68
+ model = AutoModelForCausalLM.from_pretrained(
69
+ config.base_model_name_or_path,
70
+ quantization_config=bnb_config,
71
+ device_map="auto",
72
+ cache_dir=cache_dir,
73
+ use_auth_token=True
74
+ )
75
+ #model.print_trainable_parameters()
76
+ print(sum(p.numel() for p in model.parameters()))
77
+ # 6,671,979,520 -> half-size of 13B due to 4-bit loading
78
+
79
+ model = PeftModel.from_pretrained(model, lazylora_dir)
80
+ print('after adding lazy lora parameters:')
81
+ model.print_trainable_parameters()
82
+ # trainable params: 0 || all params: 6,922,290,688 || trainable%: 0.0
83
+ ```
84
+
85
+ ## MMLU result:
86
+
87
+ ```json
88
+ {"mmlu_loss": 1.7513423996918502,
89
+ "mmlu_eval_accuracy_high_school_microeconomics": 0.6538461538461539,
90
+ "mmlu_eval_accuracy_human_aging": 0.6956521739130435,
91
+ "mmlu_eval_accuracy_high_school_biology": 0.5625,
92
+ "mmlu_eval_accuracy_machine_learning": 0.45454545454545453,
93
+ "mmlu_eval_accuracy_moral_scenarios": 0.31,
94
+ "mmlu_eval_accuracy_astronomy": 0.6875,
95
+ "mmlu_eval_accuracy_medical_genetics": 0.8181818181818182,
96
+ "mmlu_eval_accuracy_high_school_government_and_politics": 0.7142857142857143,
97
+ "mmlu_eval_accuracy_anatomy": 0.35714285714285715,
98
+ "mmlu_eval_accuracy_prehistory": 0.5714285714285714,
99
+ "mmlu_eval_accuracy_high_school_mathematics": 0.20689655172413793,
100
+ "mmlu_eval_accuracy_philosophy": 0.6176470588235294,
101
+ "mmlu_eval_accuracy_high_school_computer_science": 0.5555555555555556,
102
+ "mmlu_eval_accuracy_high_school_statistics": 0.30434782608695654,
103
+ "mmlu_eval_accuracy_conceptual_physics": 0.38461538461538464,
104
+ "mmlu_eval_accuracy_global_facts": 0.5,
105
+ "mmlu_eval_accuracy_high_school_us_history": 0.7727272727272727,
106
+ "mmlu_eval_accuracy_professional_accounting": 0.41935483870967744,
107
+ "mmlu_eval_accuracy_college_computer_science": 0.5454545454545454,
108
+ "mmlu_eval_accuracy_econometrics": 0.4166666666666667,
109
+ "mmlu_eval_accuracy_high_school_physics": 0.23529411764705882,
110
+ "mmlu_eval_accuracy_public_relations": 0.6666666666666666,
111
+ "mmlu_eval_accuracy_us_foreign_policy": 0.9090909090909091,
112
+ "mmlu_eval_accuracy_miscellaneous": 0.6744186046511628,
113
+ "mmlu_eval_accuracy_college_mathematics": 0.45454545454545453,
114
+ "mmlu_eval_accuracy_management": 0.7272727272727273,
115
+ "mmlu_eval_accuracy_college_biology": 0.625,
116
+ "mmlu_eval_accuracy_high_school_world_history": 0.5384615384615384,
117
+ "mmlu_eval_accuracy_electrical_engineering": 0.5,
118
+ "mmlu_eval_accuracy_computer_security": 0.8181818181818182,
119
+ "mmlu_eval_accuracy_clinical_knowledge": 0.4482758620689655,
120
+ "mmlu_eval_accuracy_professional_psychology": 0.5507246376811594,
121
+ "mmlu_eval_accuracy_high_school_geography": 0.8181818181818182,
122
+ "mmlu_eval_accuracy_high_school_psychology": 0.8333333333333334,
123
+ "mmlu_eval_accuracy_moral_disputes": 0.631578947368421,
124
+ "mmlu_eval_accuracy_formal_logic": 0.2857142857142857,
125
+ "mmlu_eval_accuracy_international_law": 0.8461538461538461,
126
+ "mmlu_eval_accuracy_nutrition": 0.7878787878787878,
127
+ "mmlu_eval_accuracy_marketing": 0.88,
128
+ "mmlu_eval_accuracy_high_school_chemistry": 0.4090909090909091,
129
+ "mmlu_eval_accuracy_college_chemistry": 0.375,
130
+ "mmlu_eval_accuracy_professional_medicine": 0.5483870967741935,
131
+ "mmlu_eval_accuracy_virology": 0.3888888888888889,
132
+ "mmlu_eval_accuracy_logical_fallacies": 0.7222222222222222,
133
+ "mmlu_eval_accuracy_high_school_european_history": 0.7777777777777778,
134
+ "mmlu_eval_accuracy_jurisprudence": 0.6363636363636364,
135
+ "mmlu_eval_accuracy_human_sexuality": 0.5833333333333334,
136
+ "mmlu_eval_accuracy_abstract_algebra": 0.2727272727272727,
137
+ "mmlu_eval_accuracy_business_ethics": 0.2727272727272727,
138
+ "mmlu_eval_accuracy_security_studies": 0.7037037037037037,
139
+ "mmlu_eval_accuracy_professional_law": 0.4,
140
+ "mmlu_eval_accuracy_college_medicine": 0.5,
141
+ "mmlu_eval_accuracy_elementary_mathematics": 0.34146341463414637,
142
+ "mmlu_eval_accuracy_high_school_macroeconomics": 0.5813953488372093,
143
+ "mmlu_eval_accuracy_sociology": 0.8636363636363636,
144
+ "mmlu_eval_accuracy_world_religions": 0.7894736842105263,
145
+ "mmlu_eval_accuracy_college_physics": 0.6363636363636364,
146
+ "mmlu_eval_accuracy": 0.5716083571911647,
147
+ "epoch": 0.68}
148
+ ```
149
+
150
+ ## License and intended use
151
+
152
+ This lazy-lora adapter is based on [Meta's LLaMA-2-13b-hf](https://huggingface.co/meta-llama/Llama-2-13b-hf), and using the [oasst1 dataset](https://huggingface.co/datasets/OpenAssistant/oasst1), following [Guanaco](https://huggingface.co/timdettmers/guanaco-65b).
153
+
154
+ lazy lora adapter weights are available under LLAMA-2 license. Note the use of the lazy lora adapter weights, requires access to the LLaMA model weighs. Lazy lora is based on LLaMA and therefore should be used according to the LLaMA license.
155
+
156
+
157
+ ## Risks and Biases
158
+
159
+ The model can produce factually incorrect output, and should not be relied on to produce factually accurate information. The model was trained on various public datasets; it is possible that this model could generate lewd, biased, or otherwise offensive outputs.
160
+
adapter_config.json ADDED
@@ -0,0 +1,316 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "base_model_name_or_path": "meta-llama/Llama-2-13b-hf",
3
+ "bias": "none",
4
+ "fan_in_fan_out": false,
5
+ "inference_mode": true,
6
+ "init_lazy_lora_weights": true,
7
+ "is_r_by_svd": true,
8
+ "is_r_reuse": true,
9
+ "lazy_lora_alpha": 16.0,
10
+ "lazy_lora_dropout": 0.05,
11
+ "lazy_pre_adapter_type": "none",
12
+ "lazy_pre_lora_alpha": 0.1,
13
+ "modules_to_save": null,
14
+ "num_attention_heads": 40,
15
+ "num_layers": 40,
16
+ "num_transformer_submodules": 1,
17
+ "num_virtual_tokens": null,
18
+ "peft_type": "LAZY_LORA",
19
+ "prefix_tuning_config": null,
20
+ "prompt_tuning_config": null,
21
+ "r": 64,
22
+ "r_by_module_dict": {
23
+ "model.layers.0.mlp.down_proj": 56,
24
+ "model.layers.0.mlp.gate_proj": 45,
25
+ "model.layers.0.mlp.up_proj": 47,
26
+ "model.layers.0.self_attn.k_proj": 14,
27
+ "model.layers.0.self_attn.o_proj": 24,
28
+ "model.layers.0.self_attn.q_proj": 13,
29
+ "model.layers.0.self_attn.v_proj": 30,
30
+ "model.layers.1.mlp.down_proj": 61,
31
+ "model.layers.1.mlp.gate_proj": 51,
32
+ "model.layers.1.mlp.up_proj": 55,
33
+ "model.layers.1.self_attn.k_proj": 39,
34
+ "model.layers.1.self_attn.o_proj": 36,
35
+ "model.layers.1.self_attn.q_proj": 39,
36
+ "model.layers.1.self_attn.v_proj": 38,
37
+ "model.layers.10.mlp.down_proj": 62,
38
+ "model.layers.10.mlp.gate_proj": 63,
39
+ "model.layers.10.mlp.up_proj": 64,
40
+ "model.layers.10.self_attn.k_proj": 68,
41
+ "model.layers.10.self_attn.o_proj": 56,
42
+ "model.layers.10.self_attn.q_proj": 69,
43
+ "model.layers.10.self_attn.v_proj": 55,
44
+ "model.layers.11.mlp.down_proj": 63,
45
+ "model.layers.11.mlp.gate_proj": 62,
46
+ "model.layers.11.mlp.up_proj": 64,
47
+ "model.layers.11.self_attn.k_proj": 71,
48
+ "model.layers.11.self_attn.o_proj": 55,
49
+ "model.layers.11.self_attn.q_proj": 70,
50
+ "model.layers.11.self_attn.v_proj": 55,
51
+ "model.layers.12.mlp.down_proj": 63,
52
+ "model.layers.12.mlp.gate_proj": 62,
53
+ "model.layers.12.mlp.up_proj": 65,
54
+ "model.layers.12.self_attn.k_proj": 68,
55
+ "model.layers.12.self_attn.o_proj": 56,
56
+ "model.layers.12.self_attn.q_proj": 67,
57
+ "model.layers.12.self_attn.v_proj": 57,
58
+ "model.layers.13.mlp.down_proj": 64,
59
+ "model.layers.13.mlp.gate_proj": 62,
60
+ "model.layers.13.mlp.up_proj": 65,
61
+ "model.layers.13.self_attn.k_proj": 64,
62
+ "model.layers.13.self_attn.o_proj": 58,
63
+ "model.layers.13.self_attn.q_proj": 65,
64
+ "model.layers.13.self_attn.v_proj": 58,
65
+ "model.layers.14.mlp.down_proj": 64,
66
+ "model.layers.14.mlp.gate_proj": 62,
67
+ "model.layers.14.mlp.up_proj": 65,
68
+ "model.layers.14.self_attn.k_proj": 68,
69
+ "model.layers.14.self_attn.o_proj": 58,
70
+ "model.layers.14.self_attn.q_proj": 68,
71
+ "model.layers.14.self_attn.v_proj": 58,
72
+ "model.layers.15.mlp.down_proj": 65,
73
+ "model.layers.15.mlp.gate_proj": 62,
74
+ "model.layers.15.mlp.up_proj": 66,
75
+ "model.layers.15.self_attn.k_proj": 67,
76
+ "model.layers.15.self_attn.o_proj": 59,
77
+ "model.layers.15.self_attn.q_proj": 66,
78
+ "model.layers.15.self_attn.v_proj": 60,
79
+ "model.layers.16.mlp.down_proj": 65,
80
+ "model.layers.16.mlp.gate_proj": 62,
81
+ "model.layers.16.mlp.up_proj": 66,
82
+ "model.layers.16.self_attn.k_proj": 67,
83
+ "model.layers.16.self_attn.o_proj": 60,
84
+ "model.layers.16.self_attn.q_proj": 66,
85
+ "model.layers.16.self_attn.v_proj": 61,
86
+ "model.layers.17.mlp.down_proj": 65,
87
+ "model.layers.17.mlp.gate_proj": 63,
88
+ "model.layers.17.mlp.up_proj": 65,
89
+ "model.layers.17.self_attn.k_proj": 67,
90
+ "model.layers.17.self_attn.o_proj": 63,
91
+ "model.layers.17.self_attn.q_proj": 67,
92
+ "model.layers.17.self_attn.v_proj": 62,
93
+ "model.layers.18.mlp.down_proj": 65,
94
+ "model.layers.18.mlp.gate_proj": 63,
95
+ "model.layers.18.mlp.up_proj": 65,
96
+ "model.layers.18.self_attn.k_proj": 69,
97
+ "model.layers.18.self_attn.o_proj": 65,
98
+ "model.layers.18.self_attn.q_proj": 69,
99
+ "model.layers.18.self_attn.v_proj": 65,
100
+ "model.layers.19.mlp.down_proj": 65,
101
+ "model.layers.19.mlp.gate_proj": 64,
102
+ "model.layers.19.mlp.up_proj": 65,
103
+ "model.layers.19.self_attn.k_proj": 66,
104
+ "model.layers.19.self_attn.o_proj": 65,
105
+ "model.layers.19.self_attn.q_proj": 66,
106
+ "model.layers.19.self_attn.v_proj": 64,
107
+ "model.layers.2.mlp.down_proj": 62,
108
+ "model.layers.2.mlp.gate_proj": 59,
109
+ "model.layers.2.mlp.up_proj": 60,
110
+ "model.layers.2.self_attn.k_proj": 60,
111
+ "model.layers.2.self_attn.o_proj": 50,
112
+ "model.layers.2.self_attn.q_proj": 60,
113
+ "model.layers.2.self_attn.v_proj": 49,
114
+ "model.layers.20.mlp.down_proj": 65,
115
+ "model.layers.20.mlp.gate_proj": 64,
116
+ "model.layers.20.mlp.up_proj": 65,
117
+ "model.layers.20.self_attn.k_proj": 67,
118
+ "model.layers.20.self_attn.o_proj": 65,
119
+ "model.layers.20.self_attn.q_proj": 68,
120
+ "model.layers.20.self_attn.v_proj": 64,
121
+ "model.layers.21.mlp.down_proj": 65,
122
+ "model.layers.21.mlp.gate_proj": 64,
123
+ "model.layers.21.mlp.up_proj": 65,
124
+ "model.layers.21.self_attn.k_proj": 65,
125
+ "model.layers.21.self_attn.o_proj": 67,
126
+ "model.layers.21.self_attn.q_proj": 65,
127
+ "model.layers.21.self_attn.v_proj": 66,
128
+ "model.layers.22.mlp.down_proj": 65,
129
+ "model.layers.22.mlp.gate_proj": 65,
130
+ "model.layers.22.mlp.up_proj": 65,
131
+ "model.layers.22.self_attn.k_proj": 68,
132
+ "model.layers.22.self_attn.o_proj": 71,
133
+ "model.layers.22.self_attn.q_proj": 68,
134
+ "model.layers.22.self_attn.v_proj": 70,
135
+ "model.layers.23.mlp.down_proj": 65,
136
+ "model.layers.23.mlp.gate_proj": 65,
137
+ "model.layers.23.mlp.up_proj": 64,
138
+ "model.layers.23.self_attn.k_proj": 65,
139
+ "model.layers.23.self_attn.o_proj": 71,
140
+ "model.layers.23.self_attn.q_proj": 66,
141
+ "model.layers.23.self_attn.v_proj": 69,
142
+ "model.layers.24.mlp.down_proj": 65,
143
+ "model.layers.24.mlp.gate_proj": 66,
144
+ "model.layers.24.mlp.up_proj": 64,
145
+ "model.layers.24.self_attn.k_proj": 67,
146
+ "model.layers.24.self_attn.o_proj": 71,
147
+ "model.layers.24.self_attn.q_proj": 67,
148
+ "model.layers.24.self_attn.v_proj": 70,
149
+ "model.layers.25.mlp.down_proj": 65,
150
+ "model.layers.25.mlp.gate_proj": 66,
151
+ "model.layers.25.mlp.up_proj": 64,
152
+ "model.layers.25.self_attn.k_proj": 66,
153
+ "model.layers.25.self_attn.o_proj": 73,
154
+ "model.layers.25.self_attn.q_proj": 67,
155
+ "model.layers.25.self_attn.v_proj": 72,
156
+ "model.layers.26.mlp.down_proj": 65,
157
+ "model.layers.26.mlp.gate_proj": 66,
158
+ "model.layers.26.mlp.up_proj": 64,
159
+ "model.layers.26.self_attn.k_proj": 67,
160
+ "model.layers.26.self_attn.o_proj": 73,
161
+ "model.layers.26.self_attn.q_proj": 67,
162
+ "model.layers.26.self_attn.v_proj": 73,
163
+ "model.layers.27.mlp.down_proj": 65,
164
+ "model.layers.27.mlp.gate_proj": 66,
165
+ "model.layers.27.mlp.up_proj": 65,
166
+ "model.layers.27.self_attn.k_proj": 68,
167
+ "model.layers.27.self_attn.o_proj": 73,
168
+ "model.layers.27.self_attn.q_proj": 68,
169
+ "model.layers.27.self_attn.v_proj": 72,
170
+ "model.layers.28.mlp.down_proj": 65,
171
+ "model.layers.28.mlp.gate_proj": 67,
172
+ "model.layers.28.mlp.up_proj": 65,
173
+ "model.layers.28.self_attn.k_proj": 67,
174
+ "model.layers.28.self_attn.o_proj": 73,
175
+ "model.layers.28.self_attn.q_proj": 68,
176
+ "model.layers.28.self_attn.v_proj": 73,
177
+ "model.layers.29.mlp.down_proj": 65,
178
+ "model.layers.29.mlp.gate_proj": 67,
179
+ "model.layers.29.mlp.up_proj": 65,
180
+ "model.layers.29.self_attn.k_proj": 69,
181
+ "model.layers.29.self_attn.o_proj": 74,
182
+ "model.layers.29.self_attn.q_proj": 69,
183
+ "model.layers.29.self_attn.v_proj": 73,
184
+ "model.layers.3.mlp.down_proj": 62,
185
+ "model.layers.3.mlp.gate_proj": 63,
186
+ "model.layers.3.mlp.up_proj": 62,
187
+ "model.layers.3.self_attn.k_proj": 66,
188
+ "model.layers.3.self_attn.o_proj": 52,
189
+ "model.layers.3.self_attn.q_proj": 64,
190
+ "model.layers.3.self_attn.v_proj": 53,
191
+ "model.layers.30.mlp.down_proj": 65,
192
+ "model.layers.30.mlp.gate_proj": 67,
193
+ "model.layers.30.mlp.up_proj": 65,
194
+ "model.layers.30.self_attn.k_proj": 68,
195
+ "model.layers.30.self_attn.o_proj": 77,
196
+ "model.layers.30.self_attn.q_proj": 69,
197
+ "model.layers.30.self_attn.v_proj": 76,
198
+ "model.layers.31.mlp.down_proj": 65,
199
+ "model.layers.31.mlp.gate_proj": 67,
200
+ "model.layers.31.mlp.up_proj": 65,
201
+ "model.layers.31.self_attn.k_proj": 69,
202
+ "model.layers.31.self_attn.o_proj": 74,
203
+ "model.layers.31.self_attn.q_proj": 69,
204
+ "model.layers.31.self_attn.v_proj": 74,
205
+ "model.layers.32.mlp.down_proj": 65,
206
+ "model.layers.32.mlp.gate_proj": 67,
207
+ "model.layers.32.mlp.up_proj": 66,
208
+ "model.layers.32.self_attn.k_proj": 66,
209
+ "model.layers.32.self_attn.o_proj": 77,
210
+ "model.layers.32.self_attn.q_proj": 67,
211
+ "model.layers.32.self_attn.v_proj": 76,
212
+ "model.layers.33.mlp.down_proj": 65,
213
+ "model.layers.33.mlp.gate_proj": 67,
214
+ "model.layers.33.mlp.up_proj": 66,
215
+ "model.layers.33.self_attn.k_proj": 66,
216
+ "model.layers.33.self_attn.o_proj": 75,
217
+ "model.layers.33.self_attn.q_proj": 67,
218
+ "model.layers.33.self_attn.v_proj": 75,
219
+ "model.layers.34.mlp.down_proj": 65,
220
+ "model.layers.34.mlp.gate_proj": 67,
221
+ "model.layers.34.mlp.up_proj": 67,
222
+ "model.layers.34.self_attn.k_proj": 64,
223
+ "model.layers.34.self_attn.o_proj": 77,
224
+ "model.layers.34.self_attn.q_proj": 65,
225
+ "model.layers.34.self_attn.v_proj": 77,
226
+ "model.layers.35.mlp.down_proj": 66,
227
+ "model.layers.35.mlp.gate_proj": 67,
228
+ "model.layers.35.mlp.up_proj": 67,
229
+ "model.layers.35.self_attn.k_proj": 64,
230
+ "model.layers.35.self_attn.o_proj": 77,
231
+ "model.layers.35.self_attn.q_proj": 64,
232
+ "model.layers.35.self_attn.v_proj": 76,
233
+ "model.layers.36.mlp.down_proj": 67,
234
+ "model.layers.36.mlp.gate_proj": 67,
235
+ "model.layers.36.mlp.up_proj": 67,
236
+ "model.layers.36.self_attn.k_proj": 63,
237
+ "model.layers.36.self_attn.o_proj": 78,
238
+ "model.layers.36.self_attn.q_proj": 63,
239
+ "model.layers.36.self_attn.v_proj": 77,
240
+ "model.layers.37.mlp.down_proj": 67,
241
+ "model.layers.37.mlp.gate_proj": 67,
242
+ "model.layers.37.mlp.up_proj": 67,
243
+ "model.layers.37.self_attn.k_proj": 58,
244
+ "model.layers.37.self_attn.o_proj": 79,
245
+ "model.layers.37.self_attn.q_proj": 59,
246
+ "model.layers.37.self_attn.v_proj": 78,
247
+ "model.layers.38.mlp.down_proj": 66,
248
+ "model.layers.38.mlp.gate_proj": 68,
249
+ "model.layers.38.mlp.up_proj": 67,
250
+ "model.layers.38.self_attn.k_proj": 57,
251
+ "model.layers.38.self_attn.o_proj": 81,
252
+ "model.layers.38.self_attn.q_proj": 57,
253
+ "model.layers.38.self_attn.v_proj": 81,
254
+ "model.layers.39.mlp.down_proj": 66,
255
+ "model.layers.39.mlp.gate_proj": 71,
256
+ "model.layers.39.mlp.up_proj": 68,
257
+ "model.layers.39.self_attn.k_proj": 55,
258
+ "model.layers.39.self_attn.o_proj": 75,
259
+ "model.layers.39.self_attn.q_proj": 55,
260
+ "model.layers.39.self_attn.v_proj": 74,
261
+ "model.layers.4.mlp.down_proj": 62,
262
+ "model.layers.4.mlp.gate_proj": 63,
263
+ "model.layers.4.mlp.up_proj": 62,
264
+ "model.layers.4.self_attn.k_proj": 66,
265
+ "model.layers.4.self_attn.o_proj": 52,
266
+ "model.layers.4.self_attn.q_proj": 66,
267
+ "model.layers.4.self_attn.v_proj": 52,
268
+ "model.layers.5.mlp.down_proj": 62,
269
+ "model.layers.5.mlp.gate_proj": 64,
270
+ "model.layers.5.mlp.up_proj": 62,
271
+ "model.layers.5.self_attn.k_proj": 64,
272
+ "model.layers.5.self_attn.o_proj": 53,
273
+ "model.layers.5.self_attn.q_proj": 64,
274
+ "model.layers.5.self_attn.v_proj": 53,
275
+ "model.layers.6.mlp.down_proj": 61,
276
+ "model.layers.6.mlp.gate_proj": 65,
277
+ "model.layers.6.mlp.up_proj": 62,
278
+ "model.layers.6.self_attn.k_proj": 69,
279
+ "model.layers.6.self_attn.o_proj": 55,
280
+ "model.layers.6.self_attn.q_proj": 68,
281
+ "model.layers.6.self_attn.v_proj": 55,
282
+ "model.layers.7.mlp.down_proj": 61,
283
+ "model.layers.7.mlp.gate_proj": 65,
284
+ "model.layers.7.mlp.up_proj": 62,
285
+ "model.layers.7.self_attn.k_proj": 69,
286
+ "model.layers.7.self_attn.o_proj": 55,
287
+ "model.layers.7.self_attn.q_proj": 69,
288
+ "model.layers.7.self_attn.v_proj": 56,
289
+ "model.layers.8.mlp.down_proj": 62,
290
+ "model.layers.8.mlp.gate_proj": 64,
291
+ "model.layers.8.mlp.up_proj": 63,
292
+ "model.layers.8.self_attn.k_proj": 68,
293
+ "model.layers.8.self_attn.o_proj": 56,
294
+ "model.layers.8.self_attn.q_proj": 67,
295
+ "model.layers.8.self_attn.v_proj": 56,
296
+ "model.layers.9.mlp.down_proj": 62,
297
+ "model.layers.9.mlp.gate_proj": 64,
298
+ "model.layers.9.mlp.up_proj": 63,
299
+ "model.layers.9.self_attn.k_proj": 69,
300
+ "model.layers.9.self_attn.o_proj": 56,
301
+ "model.layers.9.self_attn.q_proj": 69,
302
+ "model.layers.9.self_attn.v_proj": 56
303
+ },
304
+ "rank_file": "",
305
+ "target_modules": [
306
+ "gate_proj",
307
+ "k_proj",
308
+ "q_proj",
309
+ "down_proj",
310
+ "up_proj",
311
+ "v_proj",
312
+ "o_proj"
313
+ ],
314
+ "task_type": "CAUSAL_LM",
315
+ "token_dim": 5120
316
+ }
adapter_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:946885901731680cd58d8c7d6b65785bd7c8b8463e7e9078f69daf3b91672099
3
+ size 500836813
usage.py ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import sys
2
+ sys.path.insert(1, '/workspace/asr/peft/src')
3
+ # TODO set this path to the lazy-lora source code path, or you can install it from source code:
4
+ # TODO, please install lazylora for usage:
5
+ # git clone git@github.com:Xianchao-Wu/peft.git
6
+ # cd peft
7
+ # python setup.py install
8
+
9
+ from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
10
+ from peft import PeftModel, PeftConfig
11
+ import os
12
+ import torch
13
+
14
+ import ipdb; ipdb.set_trace()
15
+ cache_dir="/workspace/asr/peft/qlora"
16
+ # TODO set this cache_dir to the path where you stored (or, want to store) llama2-7bhf model
17
+
18
+ lazylora_dir=os.getcwd() # the path that contains 'adapter_config.json' and 'adapter_model.bin'
19
+
20
+ config = PeftConfig.from_pretrained(lazylora_dir)
21
+
22
+ tokenizer = AutoTokenizer.from_pretrained(
23
+ config.base_model_name_or_path,
24
+ cache_dir=cache_dir,
25
+ use_auth_token=True
26
+ )
27
+
28
+ bnb_config = BitsAndBytesConfig(
29
+ load_in_4bit=True,
30
+ bnb_4bit_use_double_quant=True,
31
+ bnb_4bit_quant_type='nf4',
32
+ bnb_4bit_compute_dtype=torch.bfloat16
33
+ )
34
+
35
+ model = AutoModelForCausalLM.from_pretrained(
36
+ config.base_model_name_or_path,
37
+ quantization_config=bnb_config,
38
+ device_map="auto",
39
+ cache_dir=cache_dir,
40
+ use_auth_token=True
41
+ )
42
+ #model.print_trainable_parameters()
43
+ print(sum(p.numel() for p in model.parameters()))
44
+ # 6,671,979,520 -> half-size of 7B due to 4-bit loading
45
+
46
+ model = PeftModel.from_pretrained(model, lazylora_dir)
47
+ print('after adding lazy lora parameters:')
48
+ model.print_trainable_parameters()
49
+ # trainable params: 0 || all params: 6,922,290,688 || trainable%: 0.0
50
+
51
+