upd

recipes/configs/llama3/8B_full_single_device.yaml

@@ -50,6 +50,14 @@ resume_from_checkpoint: False
 # Fine-tuning arguments
 batch_size: 2
 epochs: 3
+# Note: For trading off memory efficiency with performance,
+# we enable PagedAdam optimizer, except for the embedding
+# and output projection parameters, which use AdamW8bit.
+# This allows us to retain 90% of the memory savings compared with
+# using entirely PagedAdam, but prevents a 6x perf slowdown that
+# would be incurred. This feature is only enabled for Llama-3 8b
+# single device with optimizer_in_bwd=True, please file an issue
+# if support in different scenarios is needed.
 optimizer:
   _component_: bitsandbytes.optim.PagedAdamW
   lr: 2e-5