GLM4 32B Neon V2

# Model
base_model: /home/owen/models/GLM-4-32B-0414
strict: false
model_type: AutoModelForCausalLM

# Liger Kernels and CCE (optimization)
plugins:
  - axolotl.integrations.liger.LigerPlugin
  - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin
liger_rope: false
liger_rms_norm: false
liger_glu_activation: false
liger_fused_linear_cross_entropy: false
cut_cross_entropy: true

# Output and HuggingFace
output_dir: ./GLM-32B-Neon-v2
hub_model_id: AuriAetherwiing/GLM-32B-Neon-v2-LoRA
hf_use_auth_token: true
hub_strategy: "all_checkpoints"

# WandB
wandb_project: allura-org
wandb_entity:
wandb_name: GLM-32B-Neon-v2

# Data
#chat_template: chatml
#train_on_inputs: false
group_by_length: false
datasets:
  - path: ./Neon/neon.jsonl
    type: chat_template
    field_messages: conversations
    message_field_role: from
    message_field_content: value
    train_on_eos: all
  - path: ./Neon/S2.jsonl
    type: chat_template
    field_messages: conversations
    message_field_role: from
    message_field_content: value
    train_on_eos: all
  - path: ./Neon/SystemChat_subset_filtered_sharegpt_utf8fix.jsonl
    type: chat_template
    field_messages: conversations
    message_field_role: from
    message_field_content: value
    train_on_eos: all

dataset_prepared_path: ./lora_last_run_prepared

chat_template: jinja
chat_template_jinja: |
  [gMASK]<sop>{%- for msg in messages %}{%- if msg.role == 'system' %}<|system|>
  {{ msg.content }}{%- elif msg.role == 'user' %}<|user|>
  {{ msg.content }}{%- elif msg.role == 'assistant' %}<|assistant|>
  {{ msg.content }}{%- endif %}{%- endfor %}{% if add_generation_prompt %}<|assistant|>{% endif %}

## Evaluation
val_set_size: 0.005
evals_per_epoch: 8
eval_table_size:
eval_max_new_tokens: 128

# Technical aspects
sequence_len: 16384
save_safetensors: true
saves_per_epoch: 4
logging_steps: 1
#special_tokens:
#  pad_token: <pad>
# Quantization
bf16: auto
fp16:
tf32: false
## For LoRA
load_in_8bit: false
load_in_4bit: true

# LoRA
peft_use_rslora: false
peft_use_dora: false # better but slower
adapter: qlora # lora or qlora
lora_model_dir:
lora_r: 64 # 64 is optimal for most trains on instruct
lora_alpha: 64
lora_dropout: 0.1
lora_target_linear: true
lora_fan_in_fan_out:
lora_target_modules:

# loraplus_lr_ratio: 8 # works to converge faster but is kinda cancer bc makes model unstable
#loraplus_lr_embedding:

# Training hyperparameters
# max_steps:
num_epochs: 1

# Anti Overfit and Stability
weight_decay: 0.01
max_grad_norm: 1.0

## Learning Rate
warmup_ratio: 0.05
learning_rate: 1e-5
lr_scheduler: rex
#lr_scheduler_kwargs:
#    min_lr: 0.0000024
optimizer: adamw_torch # usually adamw_torch or paged_adamw_8bit

## Batch Size
gradient_accumulation_steps: 32     # More effective batch size - stabler train, usually. MBS also speeds it up.
micro_batch_size: 1          # Batch size per gpu = micro_batch_size * gradient_accumulation_steps
eval_batch_size: 1

# Optimizations
pad_to_sequence_len: true
sample_packing: true
eval_sample_packing: false
flash_attention: true
xformers_attention:
gradient_checkpointing:
gradient_checkpointing_kwargs:
   use_reentrant: false
   
# Set to a divisor (> 1) of the number of GPUs available
sequence_parallel_degree: 4  # Split sequences across 4 GPUs
# Optional; strides across the key dimension. Larger values use more memory but should make training faster.
heads_k_stride: 1
# Optional; one of "varlen_llama3", "batch_ring", "batch_zigzag", "batch_stripe". Defaults to
# "varlen_llama3" when `sample_packing: true`, and "batch_ring" otherwise.
ring_attn_func:
   
# deepspeed: /home/owen/axolotl/deepspeed_configs/zero3_bf16_cpuoffload_all.json

fsdp:
  - full_shard
  - auto_wrap
fsdp_config:
  fsdp_limit_all_gathers: true
  fsdp_sync_module_states: true
  fsdp_offload_params: false
  fsdp_use_orig_params: false
  fsdp_cpu_ram_efficient_loading: true
  fsdp_auto_wrap_policy: TRANSFORMER_BASED_WRAP
  fsdp_transformer_layer_cls_to_wrap: Glm4DecoderLayer
  fsdp_state_dict_type: FULL_STATE_DICT
  fsdp_sharding_strategy: FULL_SHARD
  fsdp_activation_checkpointing: true