GLM4 32B Neon V2

# 模型
base_model: /home/owen/models/GLM-4-32B-0414
strict: false
model_type: AutoModelForCausalLM

# Liger內核和CCE（優化）
plugins:
  - axolotl.integrations.liger.LigerPlugin
  - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin
liger_rope: false
liger_rms_norm: false
liger_glu_activation: false
liger_fused_linear_cross_entropy: false
cut_cross_entropy: true

# 輸出和HuggingFace
output_dir: ./GLM-32B-Neon-v2
hub_model_id: AuriAetherwiing/GLM-32B-Neon-v2-LoRA
hf_use_auth_token: true
hub_strategy: "all_checkpoints"

# WandB
wandb_project: allura-org
wandb_entity:
wandb_name: GLM-32B-Neon-v2

# 數據
#chat_template: chatml
#train_on_inputs: false
group_by_length: false
datasets:
  - path: ./Neon/neon.jsonl
    type: chat_template
    field_messages: conversations
    message_field_role: from
    message_field_content: value
    train_on_eos: all
  - path: ./Neon/S2.jsonl
    type: chat_template
    field_messages: conversations
    message_field_role: from
    message_field_content: value
    train_on_eos: all
  - path: ./Neon/SystemChat_subset_filtered_sharegpt_utf8fix.jsonl
    type: chat_template
    field_messages: conversations
    message_field_role: from
    message_field_content: value
    train_on_eos: all

dataset_prepared_path: ./lora_last_run_prepared

chat_template: jinja
chat_template_jinja: |
  [gMASK]<sop>{%- for msg in messages %}{%- if msg.role == 'system' %}<|system|>
  {{ msg.content }}{%- elif msg.role == 'user' %}<|user|>
  {{ msg.content }}{%- elif msg.role == 'assistant' %}<|assistant|>
  {{ msg.content }}{%- endif %}{%- endfor %}{% if add_generation_prompt %}<|assistant|>{% endif %}

## 評估
val_set_size: 0.005
evals_per_epoch: 8
eval_table_size:
eval_max_new_tokens: 128

# 技術方面
sequence_len: 16384
save_safetensors: true
saves_per_epoch: 4
logging_steps: 1
#special_tokens:
#  pad_token: <pad>
# 量化
bf16: auto
fp16:
tf32: false
## 對於LoRA
load_in_8bit: false
load_in_4bit: true

# LoRA
peft_use_rslora: false
peft_use_dora: false # 更好但更慢
adapter: qlora # lora或qlora
lora_model_dir:
lora_r: 64 # 對於大多數指令訓練，64是最優的
lora_alpha: 64
lora_dropout: 0.1
lora_target_linear: true
lora_fan_in_fan_out:
lora_target_modules:

# loraplus_lr_ratio: 8 # 可以更快收斂，但會使模型不穩定
#loraplus_lr_embedding:

# 訓練超參數
# max_steps:
num_epochs: 1

# 抗過擬合和穩定性
weight_decay: 0.01
max_grad_norm: 1.0

## 學習率
warmup_ratio: 0.05
learning_rate: 1e-5
lr_scheduler: rex
#lr_scheduler_kwargs:
#    min_lr: 0.0000024
optimizer: adamw_torch # 通常是adamw_torch或paged_adamw_8bit

## 批量大小
gradient_accumulation_steps: 32     # 更有效的批量大小 - 通常訓練更穩定。MBS也會加快速度。
micro_batch_size: 1          # 每個GPU的批量大小 = micro_batch_size * gradient_accumulation_steps
eval_batch_size: 1

# 優化
pad_to_sequence_len: true
sample_packing: true
eval_sample_packing: false
flash_attention: true
xformers_attention:
gradient_checkpointing:
gradient_checkpointing_kwargs:
   use_reentrant: false
   
# 設置為可用GPU數量的除數（> 1）
sequence_parallel_degree: 4  # 在4個GPU上分割序列
# 可選；跨鍵維度的步長。較大的值使用更多內存，但應該使訓練更快。
heads_k_stride: 1
# 可選；"varlen_llama3"、"batch_ring"、"batch_zigzag"、"batch_stripe"之一。當 `sample_packing: true` 時默認為 "varlen_llama3"，否則為 "batch_ring"。
ring_attn_func:
   
# deepspeed: /home/owen/axolotl/deepspeed_configs/zero3_bf16_cpuoffload_all.json

fsdp:
  - full_shard
  - auto_wrap
fsdp_config:
  fsdp_limit_all_gathers: true
  fsdp_sync_module_states: true
  fsdp_offload_params: false
  fsdp_use_orig_params: false
  fsdp_cpu_ram_efficient_loading: true
  fsdp_auto_wrap_policy: TRANSFORMER_BASED_WRAP
  fsdp_transformer_layer_cls_to_wrap: Glm4DecoderLayer
  fsdp_state_dict_type: FULL_STATE_DICT
  fsdp_sharding_strategy: FULL_SHARD
  fsdp_activation_checkpointing: true