winglian commited on
Commit
2c9c88b
1 Parent(s): 5265cd6

fix steps check for anneal on first cycle (#1316)

Browse files
Files changed (1) hide show
  1. src/axolotl/monkeypatch/relora.py +1 -1
src/axolotl/monkeypatch/relora.py CHANGED
@@ -267,7 +267,7 @@ class ReLoRAScheduler(LRScheduler):
267
  original = self.inner_schedule.get_lr()
268
  step = self.last_epoch
269
 
270
- if step < self.relora_steps:
271
  scale = 1
272
  else:
273
  per_relora_progress = step % self.relora_steps
 
267
  original = self.inner_schedule.get_lr()
268
  step = self.last_epoch
269
 
270
+ if step < self.relora_steps - self.warmup_steps:
271
  scale = 1
272
  else:
273
  per_relora_progress = step % self.relora_steps