通义千问1.5

@article{qwen,
  title={Qwen Technical Report},
  author={Jinze Bai and Shuai Bai and Yunfei Chu and Zeyu Cui and Kai Dang and Xiaodong Deng and Yang Fan and Wenbin Ge and Yu Han and Fei Huang and Binyuan Hui and Luo Ji and Mei Li and Junyang Lin and Runji Lin and Dayiheng Liu and Gao Liu and Chengqiang Lu and Keming Lu and Jianxin Ma and Rui Men and Xingzhang Ren and Xuancheng Ren and Chuanqi Tan and Sinan Tan and Jianhong Tu and Peng Wang and Shijie Wang and Wei Wang and Shengguang Wu and Benfeng Xu and Jin Xu and An Yang and Hao Yang and Jian Yang and Shusheng Yang and Yang Yao and Bowen Yu and Hongyi Yuan and Zheng Yuan and Jianwei Zhang and Xingxuan Zhang and Yichang Zhang and Zhenru Zhang and Chang Zhou and Jingren Zhou and Xiaohuan Zhou and Tianhang Zhu},
  journal={arXiv preprint arXiv:2309.16609},
  year={2023}
}

pip install torch transformers transformers_stream_generator einops accelerate
# transformers版本不低于4.37.2

python research/qwen1_5/convert_weight.py \
--torch_ckpt_dir <torch_ckpt_dir> \
--mindspore_ckpt_path <mindspore_ckpt_path>
# 参数说明：
# torch_ckpt_dir: 预训练权重文件所在的目录，此参数必须。
# mindspore_ckpt_path: 转换后的输出文件存放路径。可选，如果不给出，默认为`./transform.ckpt`

python convert_reversed.py --mindspore_ckpt_path /path/your.ckpt --torch_ckpt_path /path/your.bin
# 参数说明：
# mindspore_ckpt_path: 待转换的mindspore权重，此参数必须。
# torch_ckpt_path: 转换后的输出文件存放路径，此参数必须。

python qwen1_5/alpaca_converter.py \
--data_path path/alpaca_data.json \
--output_path /path/alpaca-data-messages.json
# 参数说明
# data_path: 存放alpaca数据的路径
# output_path: 输出转换后对话格式的数据路径

  {
    "type": "chatml",
    "messages": [
      {
        "role": "system",
        "content": "You are a helpful assistant."
      },
      {
        "role": "user",
        "content": "Give three tips for staying healthy."
      },
      {
        "role": "assistant",
        "content": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."
      }
    ],
    "source": "unknown"
  },

python qwen1_5/qwen1_5_preprocess.py \
--input_glob /path/alpaca-data-messages.json \
--vocab_file /path/vocab.json \
--merges_file /path/merges.txt \
--seq_length 2048 \
--output_file /path/alpaca-messages.mindrecord

# 节点0，节点ip为192.168.1.1，作为主节点，总共32卡且每个节点8卡
bash ../../scripts/msrun_launcher.sh "run_qwen1_5.py \
--config run_qwen1_5_72b.yaml \
--load_checkpoint /path/model_dir \
--use_parallel True \
--run_mode finetune \
--auto_trans_ckpt True \
--train_data /path/alpaca.mindrecord" \
32 8 192.168.1.1 8118 0 output/msrun_log False 300

# 节点1，节点ip为192.168.1.2，节点0与节点1启动命令仅参数NODE_RANK不同
bash ../../scripts/msrun_launcher.sh "run_qwen1_5.py \
--config run_qwen1_5_72b.yaml \
--load_checkpoint /path/model_dir \
--use_parallel True \
--run_mode finetune \
--auto_trans_ckpt True \
--train_data /path/alpaca.mindrecord" \
32 8 192.168.1.1 8118 1 output/msrun_log False 300

# 节点2，节点ip为192.168.1.3，节点0与节点2启动命令仅参数NODE_RANK不同
bash ../../scripts/msrun_launcher.sh "run_qwen1_5.py \
--config run_qwen1_5_72b.yaml \
--load_checkpoint /path/model_dir \
--use_parallel True \
--run_mode finetune \
--auto_trans_ckpt True \
--train_data /path/alpaca.mindrecord" \
32 8 192.168.1.1 8118 2 output/msrun_log False 300

# 节点3，节点ip为192.168.1.4，节点0与节点3启动命令仅参数NODE_RANK不同
bash ../../scripts/msrun_launcher.sh "run_qwen1_5.py \
--config run_qwen1_5_72b.yaml \
--load_checkpoint /path/model_dir \
--use_parallel True \
--run_mode finetune \
--auto_trans_ckpt True \
--train_data /path/alpaca.mindrecord" \
32 8 192.168.1.1 8118 3 output/msrun_log False 300

# 参数说明
# config: 配置文件路径
# load_checkpoint: 权重文件夹路径，权重按照'model_dir/rank_0/xxx.ckpt'格式存放
# auto_trans_ckpt: 自动权重转换开关
# run_mode: 运行模式，微调时设置为finetune
# train_data: 训练数据集文件夹路径