Mirrors/modelscope
1
0
Fork 0
mirror of https://github.com/modelscope/modelscope.git synced 2026-09-01 19:49:03 +02:00
Code Issues Packages Projects Releases Wiki Activity

add example/llm (#372)

Browse Source 
* add example/llm

* fix lint test
This commit is contained in:
Jintao
2023-07-11 17:35:11 +08:00
committed by GitHub
parent d8dd3989de
commit d20d033e07
11 changed files with 1124 additions and 180 deletions
Download Patch File Download Diff File Expand all files Collapse all files

449
examples/pytorch/llm/_common.py Normal file
View File

@@ -0,0 +1,449 @@
import ast
import datetime as dt
import math
import os
import random
import re
import sys
from functools import partial
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
import json
import matplotlib.pyplot as plt
import numpy as np
#
import torch
import torch.nn as nn
import torch.optim as optim
from datasets import Dataset as HFDataset
from datasets import concatenate_datasets
from matplotlib.axes import Axes
from matplotlib.figure import Figure
from numpy import ndarray
from tensorboard.backend.event_processing.event_accumulator import \
EventAccumulator
from torch import Tensor
from torch import device as Device
from torch import dtype as Dtype
from torch.nn import Module
from torch.nn.parameter import Parameter
from torch.nn.utils.rnn import pad_sequence
from torch.optim import Optimizer
from torch.optim import lr_scheduler as lrs
from torch.optim.lr_scheduler import _LRScheduler as LRScheduler
from torch.utils.data import Dataset
#
from torchmetrics import Accuracy, MeanMetric
#
from tqdm import tqdm
#
from modelscope import (Model, MsDataset, get_logger, read_config,
snapshot_download)
from modelscope.metrics.base import Metric
from modelscope.metrics.builder import METRICS
from modelscope.models.nlp.chatglm2 import ChatGLM2Tokenizer
from modelscope.msdatasets.dataset_cls.custom_datasets import \
TorchCustomDataset
from modelscope.swift import LoRAConfig, Swift
from modelscope.trainers import EpochBasedTrainer
from modelscope.utils.config import Config, ConfigDict
from modelscope.utils.registry import default_group
#
TEST_SPLIT_P = 0.01
SPLIT_SEED = 42
MAX_LENGTH: Optional[int] = 2048
COLOR, COLOR_S = '#FFE2D9', '#FF7043'
PROMPT = """### 用户
{instruction}
### AI助手
"""
logger = get_logger()
#
def get_model_dir(model_id: str, model_revision: Optional[str] = None) -> str:
model_dir = snapshot_download(model_id, model_revision)
return model_dir
def _get_version(work_dir: str) -> int:
if os.path.isdir(work_dir):
fnames = os.listdir(work_dir)
else:
fnames = []
v_list = [-1]
for fname in fnames:
m = re.match(r'v(\d+)', fname)
if m is None:
continue
v = m.group(1)
v_list.append(int(v))
return max(v_list) + 1
def get_work_dir(work_dir: str) -> str:
"""add version"""
work_dir = os.path.abspath(work_dir)
version = _get_version(work_dir)
time = dt.datetime.now().strftime('%Y%m%d-%H%M%S')
#
work_dir = os.path.join(work_dir, f'v{version}-{time}')
logger.info(f'work_dir: {work_dir}')
return work_dir
def select_device(device_ids: List[int]) -> Device:
"""Call this function before cuda is initialized.
Return: master device
"""
if torch.cuda.is_initialized():
logger.warning('CUDA has been initialized! Device selection fails!')
return torch.device('cuda:0')
#
log_s = 'Using device: '
if len(device_ids) == 0: # cpu
os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
device: str = 'cpu'
log_s += device
else:
os.environ['CUDA_VISIBLE_DEVICES'] = ','.join(
[str(d) for d in device_ids])
assert torch.cuda.is_available(
) and torch.cuda.device_count() >= len(device_ids)
log_s += f"cuda:{','.join([str(d) for d in device_ids])}" # e.g. 'cuda:1,7,8'
device = 'cuda:0'
logger.info(log_s)
return torch.device(device)
def seed_everything(seed: Optional[int] = None, gpu_dtm: bool = False) -> int:
if seed is None:
seed_max = np.iinfo(np.int32).max
seed = random.randint(0, seed_max)
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed_all(seed)
logger.info(f'Global seed set to {seed}')
if gpu_dtm:
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
logger.info(f'Setting deterministic: {True}, benchmark: {False}')
return seed
def get_T_max(dataset_len: int, batch_size: int, max_epochs: int,
drop_last: bool) -> int:
"""Calculate T_max in CosineAnnealingLR"""
if drop_last:
T_max = dataset_len // batch_size
else:
T_max = math.ceil(dataset_len / batch_size)
T_max *= max_epochs
return T_max
def tokenize_function(example: Dict[str, str], tokenizer) -> Dict[str, Any]:
"""Only applicable to baichuan and chatglm2. Other models need to be tested"""
instruction = example['instruction']
input_: str = example['input']
if input_ is not None and input_ != '':
# instruction = instruction + '\n'
if input_.startswith('输入：'):
instruction = instruction + input_[3:]
else:
instruction = instruction + input_
output = example['output']
src_text = PROMPT.format(instruction=instruction, add_special_tokens=False)
src_input_ids: List[int] = tokenizer(
src_text, return_attention_mask=False,
add_special_tokens=True)['input_ids']
# tokenizer.bos_token_id: Avoid `tgt_input_ids` being empty
tgt_input_ids = [tokenizer.bos_token_id]
if output is not None:
tgt_input_ids += tokenizer(
output, return_attention_mask=False,
add_special_tokens=False)['input_ids']
tgt_input_ids += [tokenizer.eos_token_id]
labels = [-100] * len(src_input_ids) + tgt_input_ids
else:
labels = None
input_ids = src_input_ids + tgt_input_ids
#
if MAX_LENGTH is not None:
input_ids = input_ids[-MAX_LENGTH:]
if labels is not None:
labels = labels[-MAX_LENGTH:]
#
return {'input_ids': input_ids, 'labels': labels}
def stat_dataset(dataset: HFDataset) -> None:
"""Statistical analysis was performed on the data set"""
_token_len = []
for d in dataset:
_token_len.append(len(d['input_ids']))
_token_len = np.array(_token_len)
mean = _token_len.mean().item()
std = _token_len.std().item()
min_ = _token_len.min().item()
max_ = _token_len.max().item()
logger.info(
f'Dataset Token Length: {mean:.6f}±{std:.6f}, min={min_:.6f}, max={max_:.6f}, size={_token_len.shape[0]}'
)
def print_examples(examples: Dict[str, Any], tokenizer) -> None:
input_ids, labels = examples['input_ids'], examples['labels']
print(f'[INPUT_IDS] {tokenizer.decode(input_ids)}')
print()
print(
f'[LABLES] {tokenizer.decode([lb if lb != -100 else 0 for lb in labels])}'
)
def data_collate_fn(batch: List[Dict[str, Any]], tokenizer) -> Dict[str, Any]:
input_ids = [torch.tensor(b['input_ids']) for b in batch]
labels = [torch.tensor(b['labels']) for b in batch]
attention_mask = [
torch.ones(len(input_ids[i]), dtype=torch.int64)
for i in range(len(input_ids))
]
#
input_ids = pad_sequence(
input_ids, batch_first=True, padding_value=tokenizer.pad_token_id)
attention_mask = pad_sequence(
attention_mask, batch_first=True, padding_value=0)
labels = pad_sequence(labels, batch_first=True, padding_value=-100)
return {
'input_ids': input_ids,
'attention_mask': attention_mask,
'labels': labels
}
def print_model_info(model: Module, name: Optional[str] = None) -> None:
if name is None:
name = model.__class__.__name__
#
n_params = sum(p.numel() for p in model.parameters())
n_grads = sum(p.numel() for p in model.parameters() if p.requires_grad)
n_buffers = sum(p.numel() for p in model.buffers())
#
n_params /= 1e6
n_grads /= 1e6
n_buffers /= 1e6
s = [
f'{name}: ',
f'{n_params:.4f}M Params ({n_grads:.4f}M Trainable), ',
f'{n_buffers:.4f}M Buffers',
]
s += '.'
logger.info(''.join(s))
def show_freeze_layers(model: Module, max_lines: int = 20) -> None:
named_p = list(model.named_parameters())
for i, (n, p) in enumerate(named_p):
if i >= max_lines:
logger.info('...')
break
logger.info(f'{n}: requires_grad={p.requires_grad}')
@METRICS.register_module(group_key=default_group, module_name='my_metric')
class MyMetric(Metric):
def __init__(self, vocab_size: int):
self.acc = Accuracy('multiclass', num_classes=vocab_size)
self.loss = MeanMetric()
def add(self, outputs: Dict[str, Any], inputs: Dict[str, Any]) -> None:
loss: Tensor = outputs.loss
self.loss.update(loss)
#
labels: Tensor = inputs['labels']
labels = labels[:, 1:]
labels_mask = labels != -100
logits: Tensor = outputs.logits[:, :-1]
logits = logits[labels_mask].contiguous().view(-1, logits.shape[-1])
pred = logits.argmax(dim=-1)
labels = labels[labels_mask].to(logits.device)
self.acc.update(pred, labels)
def evaluate(self):
return {
'acc': self.acc.compute().item(),
'loss': self.loss.compute().item()
}
def merge(self, other: 'MyMetric') -> None:
"""This script does not support ddp"""
raise NotImplementedError
def get_baichuan7B_model_tokenizer(model_dir: Optional[str] = None,
load_model: bool = True):
if model_dir is None:
model_id = 'baichuan-inc/baichuan-7B'
model_dir = get_model_dir(model_id, None)
#
sys.path.insert(0, model_dir)
from configuration_baichuan import BaiChuanConfig
from tokenization_baichuan import BaiChuanTokenizer
from modeling_baichuan import BaiChuanForCausalLM
model_config = BaiChuanConfig.from_pretrained(model_dir)
model_config.torch_dtype = torch.float16
logger.info(f'model_config: {model_config}')
tokenizer = BaiChuanTokenizer.from_pretrained(model_dir)
model = None
if load_model:
model = BaiChuanForCausalLM.from_pretrained(
model_dir,
config=model_config,
device_map='auto',
torch_dtype=torch.float16)
#
return model, tokenizer
def get_baichuan13B_model_tokenizer(model_dir: Optional[str] = None,
load_model: bool = True):
if model_dir is None:
model_id = 'baichuan-inc/Baichuan-13B-Base'
model_dir = get_model_dir(model_id, 'v1.0.1')
#
sys.path.insert(0, model_dir)
from configuration_baichuan import BaichuanConfig
from tokenization_baichuan import BaichuanTokenizer
from modeling_baichuan import BaichuanForCausalLM
model_config = BaichuanConfig.from_pretrained(model_dir)
model_config.torch_dtype = torch.float16
logger.info(f'model_config: {model_config}')
tokenizer = BaichuanTokenizer.from_pretrained(model_dir)
model = None
if load_model:
model = BaichuanForCausalLM.from_pretrained(
model_dir,
config=model_config,
device_map='auto',
torch_dtype=torch.float16)
#
return model, tokenizer
def get_chatglm2_model_tokenizer(model_dir: Optional[str] = None,
load_model: bool = True):
if model_dir is None:
model_id = 'ZhipuAI/chatglm2-6b'
model_dir = snapshot_download(model_id, None)
#
config = read_config(model_dir)
config['model'] = ConfigDict({'type': 'chatglm2-6b'})
tokenizer = ChatGLM2Tokenizer.from_pretrained(model_dir)
model = None
if load_model:
model = Model.from_pretrained(
model_dir,
cfg_dict=config,
device_map='auto',
torch_dtype=torch.float16)
return model, tokenizer
def get_alpaca_en_zh_dataset(
tokenize_function,
only_val: bool = False) -> Tuple[HFDataset, HFDataset]:
"""
split: Literal['train', 'validation', None]
"""
dataset_en: HFDataset = MsDataset.load(
'AI-ModelScope/alpaca-gpt4-data-en', split='train').to_hf_dataset()
dataset_zh: HFDataset = MsDataset.load(
'AI-ModelScope/alpaca-gpt4-data-zh', split='train').to_hf_dataset()
dataset_en = dataset_en.remove_columns(['text'])
dataset: HFDataset = concatenate_datasets([dataset_zh, dataset_en])
#
# dataset = dataset.select(range(1000)) # for debug
dataset = dataset.train_test_split(TEST_SPLIT_P, seed=SPLIT_SEED)
if only_val:
dataset = dataset['test']
if tokenize_function is not None:
dataset = dataset.map(tokenize_function)
dataset = dataset.remove_columns(['instruction', 'input', 'output'])
#
if only_val:
return None, dataset
else:
return dataset['train'], dataset['test']
Item = Dict[str, float]
def read_tensorboard_file(fpath: str) -> Dict[str, List[Item]]:
if not os.path.isfile(fpath):
raise FileNotFoundError(f'fpath: {fpath}')
ea = EventAccumulator(fpath)
ea.Reload()
res = {}
tags = ea.Tags()['scalars']
for tag in tags:
values = ea.Scalars(tag)
r = []
for v in values:
r.append({'step': v.step, 'value': v.value})
res[tag] = r
return res
def tensorboard_smoothing(values: List[float],
smooth: float = 0.9) -> List[float]:
norm_factor = 1
x = 0
res = []
for i in range(len(values)):
x = x * smooth + values[i] # Exponential decay
res.append(x / norm_factor)
#
norm_factor *= smooth
norm_factor += 1
return res
def plot_image(tb_dir: str,
smooth_key: List[str],
smooth_val: float = 0.9,
figsize: Tuple[int, int] = (8, 5),
dpi: int = 100) -> None:
image_dir = os.path.join(os.path.dirname(tb_dir), 'images')
os.makedirs(image_dir, exist_ok=True)
#
fname = os.listdir(tb_dir)[0]
tb_path = os.path.join(tb_dir, fname)
data = read_tensorboard_file(tb_path)
#
for k in data.keys():
_data = data[k]
steps = [d['step'] for d in _data]
values = [d['value'] for d in _data]
if len(values) == 0:
continue
_, ax = plt.subplots(1, 1, squeeze=True, figsize=figsize, dpi=dpi)
ax.set_title(k)
if len(values) == 1:
ax.scatter(steps, values, color=COLOR_S)
elif k in smooth_key:
ax.plot(steps, values, color=COLOR)
values_s = tensorboard_smoothing(values, smooth_val)
ax.plot(steps, values_s, color=COLOR_S)
else:
ax.plot(steps, values, color=COLOR_S)
fpath = os.path.join(image_dir, k.replace('/', '_'))
plt.savefig(fpath, dpi=dpi, bbox_inches='tight')

62
examples/pytorch/llm/baichuan_infer.py Normal file
View File

@@ -0,0 +1,62 @@
# ### Setting up experimental environment.
from _common import *
from transformers import TextStreamer
device_ids = [0, 1]
logger.info(device_ids)
select_device(device_ids)
# ### Loading Model and Tokenizer
# Note: You need to set the value of `CKPT_FPATH`
BAICHUAN_TYPE = '13B' # Literal['7B', '13B']
CKPT_FAPTH = '/path/to/your/xxx.pth'
LORA_TARGET_MODULES = ['W_pack']
if BAICHUAN_TYPE == '7B':
model, tokenizer = get_baichuan7B_model_tokenizer()
else:
model, tokenizer = get_baichuan13B_model_tokenizer()
if tokenizer.pad_token_id is None:
tokenizer.pad_token_id = tokenizer.eos_token_id
model.bfloat16() # Consistent with training
# ### Preparing lora
LORA_RANK = 8
LORA_ALPHA = 32
LORA_DROPOUT_P = 0 # Arbitrary value
lora_config = LoRAConfig(
replace_modules=LORA_TARGET_MODULES,
rank=LORA_RANK,
lora_alpha=LORA_ALPHA,
lora_dropout=LORA_DROPOUT_P,
pretrained_weights=CKPT_FAPTH)
logger.info(f'lora_config: {lora_config}')
Swift.prepare_model(model, lora_config)
# ### Loading Dataset
_, test_dataset = get_alpaca_en_zh_dataset(None, True)
# ### Inference
streamer = TextStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
for d in test_dataset[:5]:
output = d['output']
d['output'] = None
input_ids = tokenize_function(d, tokenizer)['input_ids']
print(f'[TEST]{tokenizer.decode(input_ids)}', end='')
input_ids = torch.tensor(input_ids)[None].cuda()
attention_mask = torch.ones_like(input_ids)
generate_ids = model.generate(
input_ids=input_ids,
max_new_tokens=512,
attention_mask=attention_mask,
streamer=streamer,
pad_token_id=tokenizer.pad_token_id,
temperature=0.7,
top_k=50,
do_sample=True)
print()
print(f'[LABELS]{output}')
print(
'-----------------------------------------------------------------------------------'
)
# input('next[ENTER]')

199
examples/pytorch/llm/baichuan_sft.py Normal file
View File

@@ -0,0 +1,199 @@
# ### Setting up experimental environment.
"""
pip install modelscope
pip install numpy pandas matplotlib scikit-learn
pip install transformers datasets
pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118
pip install tqdm
pip install tensorboard
pip install torchmetrics
pip install sentencepiece
pip install accelerate
pip install numpy -U # Resolve torchmetrics dependencies and update numpy
"""
from _common import *
device_ids = [0, 1, 2, 3]
logger.info(device_ids)
select_device(device_ids)
seed_everything(42)
# ### Loading Model and Tokenizer
BAICHUAN_TYPE = '13B' # Literal['7B', '13B']
WORK_DIR = f'runs/baichuan_{BAICHUAN_TYPE}'
LORA_TARGET_MODULES = ['W_pack']
#
if BAICHUAN_TYPE == '7B':
model_id = 'baichuan-inc/baichuan-7B'
model_dir = get_model_dir(model_id, None)
model, tokenizer = get_baichuan7B_model_tokenizer(model_dir)
else:
model_id = 'baichuan-inc/Baichuan-13B-Base'
model_dir = get_model_dir(model_id, 'v1.0.1')
model, tokenizer = get_baichuan13B_model_tokenizer(model_dir)
#
GRADIENT_CHECKPOINTING = True
if GRADIENT_CHECKPOINTING:
# baichuan13B does not implement the `get_input_embeddings` function
if BAICHUAN_TYPE == '13B':
def get_input_embeddings(self):
return self.model.embed_tokens
model.__class__.get_input_embeddings = get_input_embeddings.__get__(
model)
model.gradient_checkpointing_enable()
model.enable_input_require_grads()
if tokenizer.pad_token_id is None:
tokenizer.pad_token_id = tokenizer.eos_token_id
#
logger.info(
f'bos_token_id: {tokenizer.bos_token_id}, eos_token_id: {tokenizer.eos_token_id}, '
f'pad_token_id: {tokenizer.pad_token_id}')
# ### Preparing lora
LORA_RANK = 8
LORA_ALPHA = 32
LORA_DROPOUT_P = 0.1
lora_config = LoRAConfig(
replace_modules=LORA_TARGET_MODULES,
rank=LORA_RANK,
lora_alpha=LORA_ALPHA,
lora_dropout=LORA_DROPOUT_P)
logger.info(f'lora_config: {lora_config}')
Swift.prepare_model(model, lora_config)
#
show_freeze_layers(model)
print_model_info(model)
_p = list(model.parameters())[100]
logger.info(f'device: {_p.device}, dtype: {_p.dtype}')
model.bfloat16()
# ### Loading Dataset
tokenize_function = partial(tokenize_function, tokenizer=tokenizer)
train_dataset, val_dataset = get_alpaca_en_zh_dataset(tokenize_function)
# Data analysis
stat_dataset(train_dataset)
stat_dataset(val_dataset)
data_collate_fn = partial(data_collate_fn, tokenizer=tokenizer)
print_examples(train_dataset[0], tokenizer)
# ### Setting Config
cfg_file = os.path.join(model_dir, 'configuration.json')
#
BATCH_SIZE = 1
MAX_EPOCHS = 1
T_max = get_T_max(len(train_dataset), BATCH_SIZE, MAX_EPOCHS, True)
WORK_DIR = get_work_dir(WORK_DIR)
EVAL_INTERVAL = 500
CONFIG = Config({
'train': {
'dataloader': {
'batch_size_per_gpu': BATCH_SIZE,
'workers_per_gpu': 1,
'shuffle': True,
'drop_last': True,
'pin_memory': True
},
'max_epochs':
MAX_EPOCHS,
'work_dir':
WORK_DIR,
'optimizer': {
'type': 'AdamW',
'lr': 1e-4,
'weight_decay': 0.01,
'options': {
'cumulative_iters': 16,
'grad_clip': {
'norm_type': 2,
'max_norm': 2.0
}
}
},
'lr_scheduler': {
'type': 'CosineAnnealingLR',
'T_max': T_max,
'eta_min': 1e-5,
'options': {
'by_epoch': False,
'warmup': {
'type': 'LinearWarmup',
'warmup_ratio': 0.1,
'warmup_iters': 200
}
}
},
'hooks': [
{
'type': 'CheckpointHook',
'by_epoch': False,
'interval': EVAL_INTERVAL,
'max_checkpoint_num': 1
},
{
'type': 'EvaluationHook',
'by_epoch': False,
'interval': EVAL_INTERVAL
},
{
'type': 'BestCkptSaverHook',
'metric_key': 'acc',
'save_best': True,
'rule': 'max',
'max_checkpoint_num': 1
},
{
'type': 'TextLoggerHook',
'by_epoch': True, # Whether EpochBasedTrainer is used
'interval': 5
},
{
'type': 'TensorboardHook',
'by_epoch': False,
'interval': 5
}
]
},
'evaluation': {
'dataloader': {
'batch_size_per_gpu': BATCH_SIZE,
'workers_per_gpu': 1,
'shuffle': False,
'drop_last': False,
'pin_memory': True
},
'metrics': [{
'type': 'my_metric',
'vocab_size': tokenizer.vocab_size
}]
}
})
# ### Finetuning
def cfg_modify_fn(cfg: Config) -> Config:
cfg.update(CONFIG)
return cfg
trainer = EpochBasedTrainer(
model=model,
cfg_file=cfg_file,
data_collator=data_collate_fn,
train_dataset=train_dataset,
eval_dataset=val_dataset,
remove_unused_data=True,
seed=42,
device='cpu', # No placement for model, leave the model to `device_map`
cfg_modify_fn=cfg_modify_fn,
)
trainer.train()
# ### Visualization
tb_dir = os.path.join(WORK_DIR, 'tensorboard_output')
plot_image(tb_dir, ['loss'], 0.9)

60
examples/pytorch/llm/chatglm2_infer.py Normal file
View File

@@ -0,0 +1,60 @@
# ### Setting up experimental environment.
from _common import *
from transformers import TextStreamer
device_ids = [0, 1]
logger.info(device_ids)
select_device(device_ids)
# ### Loading Model and Tokenizer
# Note: You need to set the value of `CKPT_FPATH`
CKPT_FAPTH = '/path/to/your/xxx.pth'
LORA_TARGET_MODULES = ['query_key_value']
model, tokenizer = get_chatglm2_model_tokenizer()
if tokenizer.eos_token_id is None:
tokenizer.eos_token_id = tokenizer.pad_token_id
if tokenizer.bos_token_id is None:
tokenizer.bos_token_id = 1
model.bfloat16() # Consistent with training
# ### Preparing lora
LORA_RANK = 8
LORA_ALPHA = 32
LORA_DROPOUT_P = 0 # Arbitrary value
lora_config = LoRAConfig(
replace_modules=LORA_TARGET_MODULES,
rank=LORA_RANK,
lora_alpha=LORA_ALPHA,
lora_dropout=LORA_DROPOUT_P,
pretrained_weights=CKPT_FAPTH)
logger.info(f'lora_config: {lora_config}')
Swift.prepare_model(model, lora_config)
# ### Loading Dataset
_, test_dataset = get_alpaca_en_zh_dataset(None, True)
# ### Inference
streamer = TextStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
for d in test_dataset[:5]:
output = d['output']
d['output'] = None
input_ids = tokenize_function(d, tokenizer)['input_ids']
print(f'[TEST]{tokenizer.decode(input_ids)}', end='')
input_ids = torch.tensor(input_ids)[None].cuda()
attention_mask = torch.ones_like(input_ids)
generate_ids = model.generate(
input_ids=input_ids,
max_new_tokens=512,
attention_mask=attention_mask,
streamer=streamer,
pad_token_id=tokenizer.pad_token_id,
temperature=0.7,
top_k=50,
do_sample=True)
print()
print(f'[LABELS]{output}')
print(
'-----------------------------------------------------------------------------------'
)
# input('next[ENTER]')

188
examples/pytorch/llm/chatglm2_sft.py Normal file
View File

@@ -0,0 +1,188 @@
# ### Setting up experimental environment.
"""
pip install modelscope
pip install numpy pandas matplotlib scikit-learn
pip install transformers datasets
pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118
pip install tqdm
pip install tensorboard
pip install torchmetrics
pip install sentencepiece
pip install accelerate
pip install numpy -U # Resolve torchmetrics dependencies and update numpy
"""
from _common import *
device_ids = [0, 1, 2, 3]
logger.info(device_ids)
select_device(device_ids)
seed_everything(42)
# ### Loading Model and Tokenizer
model_id = 'ZhipuAI/chatglm2-6b'
WORK_DIR = 'runs/chatglm2'
LORA_TARGET_MODULES = ['query_key_value']
#
model_dir = get_model_dir(model_id, None)
model, tokenizer = get_chatglm2_model_tokenizer(model_dir)
# chatglm2 does not support gradient_checkpointing
GRADIENT_CHECKPOINTING = False
if GRADIENT_CHECKPOINTING:
model.gradient_checkpointing_enable()
model.enable_input_require_grads()
logger.info(tokenizer.special_tokens)
if tokenizer.eos_token_id is None:
tokenizer.eos_token_id = tokenizer.pad_token_id
if tokenizer.bos_token_id is None:
tokenizer.bos_token_id = 1
#
logger.info(
f'bos_token_id: {tokenizer.bos_token_id}, eos_token_id: {tokenizer.eos_token_id}, '
f'pad_token_id: {tokenizer.pad_token_id}')
# ### Preparing lora
LORA_RANK = 8
LORA_ALPHA = 32
LORA_DROPOUT_P = 0.1
lora_config = LoRAConfig(
replace_modules=LORA_TARGET_MODULES,
rank=LORA_RANK,
lora_alpha=LORA_ALPHA,
lora_dropout=LORA_DROPOUT_P)
logger.info(f'lora_config: {lora_config}')
Swift.prepare_model(model, lora_config)
#
show_freeze_layers(model)
print_model_info(model)
_p = list(model.parameters())[100]
logger.info(f'device: {_p.device}, dtype: {_p.dtype}')
model.bfloat16()
# ### Loading Dataset
tokenize_function = partial(tokenize_function, tokenizer=tokenizer)
train_dataset, val_dataset = get_alpaca_en_zh_dataset(tokenize_function)
# Data analysis
stat_dataset(train_dataset)
stat_dataset(val_dataset)
data_collate_fn = partial(data_collate_fn, tokenizer=tokenizer)
print_examples(train_dataset[0], tokenizer)
# ### Setting Config
cfg_file = os.path.join(model_dir, 'configuration.json')
#
BATCH_SIZE = 1
MAX_EPOCHS = 1
T_max = get_T_max(len(train_dataset), BATCH_SIZE, MAX_EPOCHS, True)
WORK_DIR = get_work_dir(WORK_DIR)
EVAL_INTERVAL = 500
CONFIG = Config({
'train': {
'dataloader': {
'batch_size_per_gpu': BATCH_SIZE,
'workers_per_gpu': 1,
'shuffle': True,
'drop_last': True,
'pin_memory': True
},
'max_epochs':
MAX_EPOCHS,
'work_dir':
WORK_DIR,
'optimizer': {
'type': 'AdamW',
'lr': 1e-4,
'weight_decay': 0.01,
'options': {
'cumulative_iters': 16,
'grad_clip': {
'norm_type': 2,
'max_norm': 2.0
}
}
},
'lr_scheduler': {
'type': 'CosineAnnealingLR',
'T_max': T_max,
'eta_min': 1e-5,
'options': {
'by_epoch': False,
'warmup': {
'type': 'LinearWarmup',
'warmup_ratio': 0.1,
'warmup_iters': 200
}
}
},
'hooks': [
{
'type': 'CheckpointHook',
'by_epoch': False,
'interval': EVAL_INTERVAL,
'max_checkpoint_num': 1
},
{
'type': 'EvaluationHook',
'by_epoch': False,
'interval': EVAL_INTERVAL
},
{
'type': 'BestCkptSaverHook',
'metric_key': 'acc',
'save_best': True,
'rule': 'max',
'max_checkpoint_num': 1
},
{
'type': 'TextLoggerHook',
'by_epoch': True, # Whether EpochBasedTrainer is used
'interval': 5
},
{
'type': 'TensorboardHook',
'by_epoch': False,
'interval': 5
}
]
},
'evaluation': {
'dataloader': {
'batch_size_per_gpu': BATCH_SIZE,
'workers_per_gpu': 1,
'shuffle': False,
'drop_last': False,
'pin_memory': True
},
'metrics': [{
'type': 'my_metric',
'vocab_size': tokenizer.vocab_size
}]
}
})
# ### Finetuning
def cfg_modify_fn(cfg: Config) -> Config:
cfg.update(CONFIG)
return cfg
trainer = EpochBasedTrainer(
model=model,
cfg_file=cfg_file,
data_collator=data_collate_fn,
train_dataset=train_dataset,
eval_dataset=val_dataset,
remove_unused_data=True,
seed=42,
device='cpu', # No placement for model, leave the model to `device_map`
cfg_modify_fn=cfg_modify_fn,
)
trainer.train()
# ### Visualization
tb_dir = os.path.join(WORK_DIR, 'tensorboard_output')
plot_image(tb_dir, ['loss'], 0.9)

9
examples/pytorch/llm_agent/_common.py
View File

@@ -111,7 +111,7 @@ def select_device(device_ids: List[int]) -> Device:
[str(d) for d in device_ids])
assert torch.cuda.is_available(
) and torch.cuda.device_count() >= len(device_ids)
log_s += f"cuda:{','.join([str(d) for d in device_ids])}" # e.g. "cuda:1,7,8"
log_s += f"cuda:{','.join([str(d) for d in device_ids])}" # e.g. 'cuda:1,7,8'
device = 'cuda:0'
logger.info(log_s)
return torch.device(device)
@@ -221,7 +221,7 @@ def print_examples(examples: Dict[str, Any], tokenizer) -> None:
print(f'[INPUT_IDS] {tokenizer.decode(input_ids)}')
print()
print(
f'[LABLES] {tokenizer.decode([l if l != -100 else 0 for l in labels])}'
f'[LABLES] {tokenizer.decode([lb if lb != -100 else 0 for lb in labels])}'
)
@@ -334,8 +334,7 @@ def get_chatglm2_model_tokenizer(model_dir: Optional[str] = None,
load_model: bool = True):
if model_dir is None:
model_id = 'ZhipuAI/chatglm2-6b'
model_revision = 'v1.0.3'
model_dir = snapshot_download(model_id, model_revision)
model_dir = snapshot_download(model_id, None)
#
config = read_config(model_dir)
config['model'] = ConfigDict({'type': 'chatglm2-6b'})
@@ -355,7 +354,7 @@ def make_dataset(
Dict[str, Any]]
) -> MyDataset:
"""
split: Literal["train", "validation"]
split: Literal['train', 'validation']
"""
dataset = MsDataset.load(
'modelscope/ms_hackathon_23_agent_train_dev', split=split)

42
examples/pytorch/llm_agent/baichuan_infer.ipynb
View File

@@ -16,15 +16,6 @@
"### 配置实验环境"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# !pip install transformers"
]
},
{
"cell_type": "code",
"execution_count": 1,
@@ -62,7 +53,7 @@
"source": [
"from _common import *\n",
"from transformers import TextStreamer\n",
"device_ids = list(range(min(4, torch.cuda.device_count())))\n",
"device_ids = [0, 1]\n",
"logger.info(device_ids)\n",
"select_device(device_ids)"
]
@@ -152,8 +143,8 @@
}
],
"source": [
"CKPT_FAPTH = \"/home/hackathon/my_git/agent/runs/baichuan/v10-20230702-172449/output_best/pytorch_model.bin\"\n",
"LORA_TARGET_MODULES = [\"W_pack\"]\n",
"CKPT_FAPTH = '/home/hackathon/my_git/agent/runs/baichuan/v10-20230702-172449/output_best/pytorch_model.bin'\n",
"LORA_TARGET_MODULES = ['W_pack']\n",
"\n",
"model, tokenizer = get_baichuan_model_tokenizer()\n",
"if tokenizer.pad_token_id is None:\n",
@@ -225,7 +216,7 @@
" lora_alpha=LORA_ALPHA,\n",
" lora_dropout=LORA_DROPOUT_P,\n",
" pretrained_weights=CKPT_FAPTH)\n",
"logger.info(f\"lora_config: {lora_config}\")\n",
"logger.info(f'lora_config: {lora_config}')\n",
"Swift.prepare_model(model, lora_config)"
]
},
@@ -289,8 +280,8 @@
}
],
"source": [
"test_dataset = make_dataset(\"validation\", lambda system, user, assistant:\n",
" {\"system\": system, \"user\": user, \"assistant\": assistant})"
"test_dataset = make_dataset('validation', lambda system, user, assistant:\n",
" {'system': system, 'user': user, 'assistant': assistant})"
]
},
{
@@ -451,20 +442,21 @@
"source": [
"streamer = TextStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)\n",
"for d in test_dataset[:5]:\n",
" system = d[\"system\"]\n",
" user = d[\"user\"]\n",
" assistant = d[\"assistant\"]\n",
" input_ids = tokenize_function(system, user, None, tokenizer)[\"input_ids\"]\n",
" print(f\"[TEST]{tokenizer.decode(input_ids)}\", end=\"\")\n",
" system = d['system']\n",
" user = d['user']\n",
" assistant = d['assistant']\n",
" input_ids = tokenize_function(system, user, None, tokenizer)['input_ids']\n",
" print(f'[TEST]{tokenizer.decode(input_ids)}', end='')\n",
" input_ids = torch.tensor(input_ids)[None].cuda()\n",
" attention_mask = torch.ones_like(input_ids)\n",
" generate_ids = model.generate(input_ids=input_ids, max_new_tokens=512,\n",
" attention_mask=attention_mask,\n",
" streamer=streamer, pad_token_id=tokenizer.pad_token_id)\n",
" streamer=streamer, pad_token_id=tokenizer.pad_token_id, \n",
" temperature=0.7, top_k=50, do_sample=True)\n",
" print()\n",
" print(f\"[LABELS]{assistant}\")\n",
" print(\"-----------------------------------------------------------------------------------\")\n",
" # input(\"next[ENTER]\")"
" print(f'[LABELS]{assistant}')\n",
" print('-----------------------------------------------------------------------------------')\n",
" # input('next[ENTER]')"
]
}
],
@@ -484,7 +476,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.11"
"version": "3.10.12"
}
},
"nbformat": 4,

124
examples/pytorch/llm_agent/baichuan_sft.ipynb
View File

@@ -36,10 +36,12 @@
"# !pip install modelscope -U\n",
"# !pip install numpy pandas matplotlib scikit-learn\n",
"# !pip install transformers datasets\n",
"# !conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch -c nvidia\n",
"# !pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118\n",
"# !pip install tqdm\n",
"# !pip install tensorboard\n",
"# !pip install torchmetrics\n",
"# !pip install sentencepiece\n",
"# !pip install accelerate\n",
"#\n",
"# !pip install numpy -U # Resolve torchmetrics dependencies and update numpy"
]
@@ -73,7 +75,7 @@
],
"source": [
"from _common import *\n",
"device_ids = list(range(min(4, torch.cuda.device_count())))\n",
"device_ids = [0, 1, 2, 3]\n",
"logger.info(device_ids)\n",
"select_device(device_ids)\n",
"_ = seed_everything(42)"
@@ -130,9 +132,9 @@
}
],
"source": [
"model_id = \"baichuan-inc/baichuan-7B\"\n",
"WORK_DIR = \"runs/baichuan\"\n",
"LORA_TARGET_MODULES = [\"W_pack\"]\n",
"model_id = 'baichuan-inc/baichuan-7B'\n",
"WORK_DIR = 'runs/baichuan'\n",
"LORA_TARGET_MODULES = ['W_pack']\n",
"#\n",
"model_dir = get_model_dir(model_id, None)\n",
"model, tokenizer = get_baichuan_model_tokenizer(model_dir)\n",
@@ -144,8 +146,8 @@
"if tokenizer.pad_token_id is None:\n",
" tokenizer.pad_token_id = tokenizer.eos_token_id\n",
"#\n",
"logger.info(f\"bos_token_id: {tokenizer.bos_token_id}, eos_token_id: {tokenizer.eos_token_id}, \"\n",
" f\"pad_token_id: {tokenizer.pad_token_id}\")"
"logger.info(f'bos_token_id: {tokenizer.bos_token_id}, eos_token_id: {tokenizer.eos_token_id}, '\n",
" f'pad_token_id: {tokenizer.pad_token_id}')"
]
},
{
@@ -237,13 +239,13 @@
" rank=LORA_RANK,\n",
" lora_alpha=LORA_ALPHA,\n",
" lora_dropout=LORA_DROPOUT_P)\n",
"logger.info(f\"lora_config: {lora_config}\")\n",
"logger.info(f'lora_config: {lora_config}')\n",
"Swift.prepare_model(model, lora_config)\n",
"#\n",
"show_freeze_layers(model)\n",
"print_model_info(model)\n",
"_p = list(model.parameters())[100]\n",
"logger.info(f\"device: {_p.device}, dtype: {_p.dtype}\")\n",
"logger.info(f'device: {_p.device}, dtype: {_p.dtype}')\n",
"model.bfloat16()"
]
},
@@ -308,8 +310,8 @@
],
"source": [
"tokenize_function = partial(tokenize_function, tokenizer=tokenizer)\n",
"train_dataset = make_dataset(\"train\", tokenize_function)\n",
"val_dataset = make_dataset(\"validation\", tokenize_function)\n",
"train_dataset = make_dataset('train', tokenize_function)\n",
"val_dataset = make_dataset('validation', tokenize_function)\n",
"# Data analysis\n",
"stat_dataset(train_dataset)\n",
"stat_dataset(val_dataset)\n",
@@ -339,7 +341,7 @@
}
],
"source": [
"cfg_file = os.path.join(model_dir, \"configuration.json\")\n",
"cfg_file = os.path.join(model_dir, 'configuration.json')\n",
"#\n",
"BATCH_SIZE = 1\n",
"MAX_EPOCHS = 1\n",
@@ -347,62 +349,62 @@
"WORK_DIR = get_work_dir(WORK_DIR)\n",
"EVAL_INTERVAL = 200\n",
"CONFIG = Config({\n",
" \"train\": {\n",
" \"dataloader\": {\n",
" \"batch_size_per_gpu\": BATCH_SIZE,\n",
" \"workers_per_gpu\": 1,\n",
" \"shuffle\": True,\n",
" \"drop_last\": True,\n",
" \"pin_memory\": True\n",
" 'train': {\n",
" 'dataloader': {\n",
" 'batch_size_per_gpu': BATCH_SIZE,\n",
" 'workers_per_gpu': 1,\n",
" 'shuffle': True,\n",
" 'drop_last': True,\n",
" 'pin_memory': True\n",
" },\n",
" \"max_epochs\": MAX_EPOCHS,\n",
" \"work_dir\": WORK_DIR,\n",
" \"optimizer\": {\n",
" \"type\": \"AdamW\",\n",
" \"lr\": 1e-4,\n",
" \"weight_decay\": 0.01,\n",
" \"options\": {\n",
" \"cumulative_iters\": 16, \"grad_clip\": {\n",
" \"norm_type\": 2,\n",
" \"max_norm\": 2.0\n",
" 'max_epochs': MAX_EPOCHS,\n",
" 'work_dir': WORK_DIR,\n",
" 'optimizer': {\n",
" 'type': 'AdamW',\n",
" 'lr': 1e-4,\n",
" 'weight_decay': 0.01,\n",
" 'options': {\n",
" 'cumulative_iters': 16, 'grad_clip': {\n",
" 'norm_type': 2,\n",
" 'max_norm': 2.0\n",
" }\n",
" }\n",
" },\n",
" \"lr_scheduler\": {\n",
" \"type\": \"CosineAnnealingLR\",\n",
" \"T_max\": T_max,\n",
" \"eta_min\": 1e-5,\n",
" \"options\": {\n",
" \"by_epoch\": False,\n",
" \"warmup\": {\n",
" 'lr_scheduler': {\n",
" 'type': 'CosineAnnealingLR',\n",
" 'T_max': T_max,\n",
" 'eta_min': 1e-5,\n",
" 'options': {\n",
" 'by_epoch': False,\n",
" 'warmup': {\n",
" 'type': 'LinearWarmup',\n",
" 'warmup_ratio': 0.1,\n",
" \"warmup_iters\": 200\n",
" 'warmup_iters': 200\n",
" }\n",
" }\n",
" },\n",
" \"hooks\": [\n",
" {\"type\": \"CheckpointHook\", \"by_epoch\": False, \"interval\": EVAL_INTERVAL, \"max_checkpoint_num\": 1},\n",
" {\"type\": \"EvaluationHook\", \"by_epoch\": False, \"interval\": EVAL_INTERVAL},\n",
" {\"type\": \"BestCkptSaverHook\",\n",
" \"metric_key\": \"acc\",\n",
" \"save_best\": True, \"rule\": \"max\", \"max_checkpoint_num\": 1},\n",
" {\"type\": \"TextLoggerHook\",\n",
" \"by_epoch\": True, # Whether EpochBasedTrainer is used\n",
" \"interval\": 5},\n",
" {\"type\": \"TensorboardHook\", \"by_epoch\": False, \"interval\": 5}\n",
" 'hooks': [\n",
" {'type': 'CheckpointHook', 'by_epoch': False, 'interval': EVAL_INTERVAL, 'max_checkpoint_num': 1},\n",
" {'type': 'EvaluationHook', 'by_epoch': False, 'interval': EVAL_INTERVAL},\n",
" {'type': 'BestCkptSaverHook',\n",
" 'metric_key': 'acc',\n",
" 'save_best': True, 'rule': 'max', 'max_checkpoint_num': 1},\n",
" {'type': 'TextLoggerHook',\n",
" 'by_epoch': True, # Whether EpochBasedTrainer is used\n",
" 'interval': 5},\n",
" {'type': 'TensorboardHook', 'by_epoch': False, 'interval': 5}\n",
" ]\n",
" },\n",
" \"evaluation\": {\n",
" \"dataloader\": {\n",
" \"batch_size_per_gpu\": BATCH_SIZE,\n",
" \"workers_per_gpu\": 1,\n",
" \"shuffle\": False,\n",
" \"drop_last\": False,\n",
" \"pin_memory\": True\n",
" 'evaluation': {\n",
" 'dataloader': {\n",
" 'batch_size_per_gpu': BATCH_SIZE,\n",
" 'workers_per_gpu': 1,\n",
" 'shuffle': False,\n",
" 'drop_last': False,\n",
" 'pin_memory': True\n",
" },\n",
" \"metrics\": [\n",
" {\"type\": \"my_metric\", \"vocab_size\": tokenizer.vocab_size}\n",
" 'metrics': [\n",
" {'type': 'my_metric', 'vocab_size': tokenizer.vocab_size}\n",
" ]\n",
" }\n",
"})"
@@ -1778,16 +1780,16 @@
}
],
"source": [
"tb_dir = os.path.join(WORK_DIR, \"tensorboard_output\")\n",
"tb_dir = os.path.join(WORK_DIR, 'tensorboard_output')\n",
"fname = os.listdir(tb_dir)[0]\n",
"tb_path = os.path.join(tb_dir, fname)\n",
"#\n",
"data = read_tensorboard_file(tb_path)\n",
"print(data.keys())\n",
"_ = plot_image(data, \"loss\", 0.9)\n",
"_ = plot_image(data, \"lr\", 0)\n",
"_ = plot_image(data, \"evaluation/acc\", 0)\n",
"_ = plot_image(data, \"evaluation/loss\", 0)"
"_ = plot_image(data, 'loss', 0.9)\n",
"_ = plot_image(data, 'lr', 0)\n",
"_ = plot_image(data, 'evaluation/acc', 0)\n",
"_ = plot_image(data, 'evaluation/loss', 0)"
]
},
{

42
examples/pytorch/llm_agent/chatglm2_infer.ipynb
View File

@@ -17,15 +17,6 @@
"The following code is copied from baichuan_infer.ipynb"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"# !pip install transformers"
]
},
{
"cell_type": "code",
"execution_count": 2,
@@ -63,7 +54,7 @@
"source": [
"from _common import *\n",
"from transformers import TextStreamer\n",
"device_ids = list(range(min(4, torch.cuda.device_count())))\n",
"device_ids = [0, 1]\n",
"logger.info(device_ids)\n",
"select_device(device_ids)"
]
@@ -149,8 +140,8 @@
}
],
"source": [
"CKPT_FAPTH = \"/home/hackathon/my_git/agent/runs/chatglm2/v1-20230702-203505/output_best/pytorch_model.bin\"\n",
"LORA_TARGET_MODULES = [\"query_key_value\"]\n",
"CKPT_FAPTH = '/home/hackathon/my_git/agent/runs/chatglm2/v1-20230702-203505/output_best/pytorch_model.bin'\n",
"LORA_TARGET_MODULES = ['query_key_value']\n",
"\n",
"model, tokenizer = get_chatglm2_model_tokenizer()\n",
"if tokenizer.eos_token_id is None:\n",
@@ -230,7 +221,7 @@
" lora_alpha=LORA_ALPHA,\n",
" lora_dropout=LORA_DROPOUT_P,\n",
" pretrained_weights=CKPT_FAPTH)\n",
"logger.info(f\"lora_config: {lora_config}\")\n",
"logger.info(f'lora_config: {lora_config}')\n",
"Swift.prepare_model(model, lora_config)"
]
},
@@ -295,8 +286,8 @@
}
],
"source": [
"test_dataset = make_dataset(\"validation\", lambda system, user, assistant:\n",
" {\"system\": system, \"user\": user, \"assistant\": assistant})"
"test_dataset = make_dataset('validation', lambda system, user, assistant:\n",
" {'system': system, 'user': user, 'assistant': assistant})"
]
},
{
@@ -484,20 +475,21 @@
"source": [
"streamer = TextStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)\n",
"for d in test_dataset[:5]:\n",
" system = d[\"system\"]\n",
" user = d[\"user\"]\n",
" assistant = d[\"assistant\"]\n",
" input_ids = tokenize_function(system, user, None, tokenizer)[\"input_ids\"]\n",
" print(f\"[TEST]{tokenizer.decode(input_ids)}\", end=\"\")\n",
" system = d['system']\n",
" user = d['user']\n",
" assistant = d['assistant']\n",
" input_ids = tokenize_function(system, user, None, tokenizer)['input_ids']\n",
" print(f'[TEST]{tokenizer.decode(input_ids)}', end='')\n",
" input_ids = torch.tensor(input_ids)[None].cuda()\n",
" attention_mask = torch.ones_like(input_ids)\n",
" generate_ids = model.generate(input_ids=input_ids, max_new_tokens=512,\n",
" attention_mask=attention_mask,\n",
" streamer=streamer, pad_token_id=tokenizer.pad_token_id)\n",
" streamer=streamer, pad_token_id=tokenizer.pad_token_id, \n",
" temperature=0.7, top_k=50, do_sample=True)\n",
" print()\n",
" print(f\"[LABELS]{assistant}\")\n",
" print(\"-----------------------------------------------------------------------------------\")\n",
" # input(\"next[ENTER]\")"
" print(f'[LABELS]{assistant}')\n",
" print('-----------------------------------------------------------------------------------')\n",
" # input('next[ENTER]')"
]
}
],
@@ -517,7 +509,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.11"
"version": "3.10.12"
},
"orig_nbformat": 4
},

127
examples/pytorch/llm_agent/chatglm2_sft.ipynb
View File

@@ -43,10 +43,12 @@
"# !pip install modelscope -U\n",
"# !pip install numpy pandas matplotlib scikit-learn\n",
"# !pip install transformers datasets\n",
"# !conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch -c nvidia\n",
"# !pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118\n",
"# !pip install tqdm\n",
"# !pip install tensorboard\n",
"# !pip install torchmetrics\n",
"# !pip install sentencepiece\n",
"# !pip install accelerate\n",
"#\n",
"# !pip install numpy -U # Resolve torchmetrics dependencies and update numpy"
]
@@ -78,7 +80,7 @@
],
"source": [
"from _common import *\n",
"device_ids = list(range(min(4, torch.cuda.device_count())))\n",
"device_ids = [0, 1, 2, 3]\n",
"logger.info(device_ids)\n",
"select_device(device_ids)\n",
"_ = seed_everything(42)"
@@ -134,12 +136,11 @@
}
],
"source": [
"model_id = \"ZhipuAI/chatglm2-6b\"\n",
"model_revision = \"v1.0.3\"\n",
"WORK_DIR = \"runs/chatglm2\"\n",
"LORA_TARGET_MODULES = [\"query_key_value\"]\n",
"model_id = 'ZhipuAI/chatglm2-6b'\n",
"WORK_DIR = 'runs/chatglm2'\n",
"LORA_TARGET_MODULES = ['query_key_value']\n",
"#\n",
"model_dir = get_model_dir(model_id, model_revision)\n",
"model_dir = get_model_dir(model_id, None)\n",
"model, tokenizer = get_chatglm2_model_tokenizer(model_dir)\n",
"# chatglm2 does not support gradient_checkpointing\n",
"GRADIENT_CHECKPOINTING = False\n",
@@ -152,8 +153,8 @@
"if tokenizer.bos_token_id is None:\n",
" tokenizer.bos_token_id = 1\n",
"#\n",
"logger.info(f\"bos_token_id: {tokenizer.bos_token_id}, eos_token_id: {tokenizer.eos_token_id}, \"\n",
" f\"pad_token_id: {tokenizer.pad_token_id}\")"
"logger.info(f'bos_token_id: {tokenizer.bos_token_id}, eos_token_id: {tokenizer.eos_token_id}, '\n",
" f'pad_token_id: {tokenizer.pad_token_id}')"
]
},
{
@@ -251,13 +252,13 @@
" rank=LORA_RANK,\n",
" lora_alpha=LORA_ALPHA,\n",
" lora_dropout=LORA_DROPOUT_P)\n",
"logger.info(f\"lora_config: {lora_config}\")\n",
"logger.info(f'lora_config: {lora_config}')\n",
"Swift.prepare_model(model, lora_config)\n",
"#\n",
"show_freeze_layers(model)\n",
"print_model_info(model)\n",
"_p = list(model.parameters())[100]\n",
"logger.info(f\"device: {_p.device}, dtype: {_p.dtype}\")\n",
"logger.info(f'device: {_p.device}, dtype: {_p.dtype}')\n",
"model.bfloat16()"
]
},
@@ -399,8 +400,8 @@
],
"source": [
"tokenize_function = partial(tokenize_function, tokenizer=tokenizer)\n",
"train_dataset = make_dataset(\"train\", tokenize_function)\n",
"val_dataset = make_dataset(\"validation\", tokenize_function)\n",
"train_dataset = make_dataset('train', tokenize_function)\n",
"val_dataset = make_dataset('validation', tokenize_function)\n",
"# Data analysis\n",
"stat_dataset(train_dataset)\n",
"stat_dataset(val_dataset)\n",
@@ -431,7 +432,7 @@
}
],
"source": [
"cfg_file = os.path.join(model_dir, \"configuration.json\")\n",
"cfg_file = os.path.join(model_dir, 'configuration.json')\n",
"#\n",
"BATCH_SIZE = 1\n",
"MAX_EPOCHS = 1\n",
@@ -439,62 +440,62 @@
"WORK_DIR = get_work_dir(WORK_DIR)\n",
"EVAL_INTERVAL = 200\n",
"CONFIG = Config({\n",
" \"train\": {\n",
" \"dataloader\": {\n",
" \"batch_size_per_gpu\": BATCH_SIZE,\n",
" \"workers_per_gpu\": 1,\n",
" \"shuffle\": True,\n",
" \"drop_last\": True,\n",
" \"pin_memory\": True\n",
" 'train': {\n",
" 'dataloader': {\n",
" 'batch_size_per_gpu': BATCH_SIZE,\n",
" 'workers_per_gpu': 1,\n",
" 'shuffle': True,\n",
" 'drop_last': True,\n",
" 'pin_memory': True\n",
" },\n",
" \"max_epochs\": MAX_EPOCHS,\n",
" \"work_dir\": WORK_DIR,\n",
" \"optimizer\": {\n",
" \"type\": \"AdamW\",\n",
" \"lr\": 1e-4,\n",
" \"weight_decay\": 0.01,\n",
" \"options\": {\n",
" \"cumulative_iters\": 16, \"grad_clip\": {\n",
" \"norm_type\": 2,\n",
" \"max_norm\": 2.0\n",
" 'max_epochs': MAX_EPOCHS,\n",
" 'work_dir': WORK_DIR,\n",
" 'optimizer': {\n",
" 'type': 'AdamW',\n",
" 'lr': 1e-4,\n",
" 'weight_decay': 0.01,\n",
" 'options': {\n",
" 'cumulative_iters': 16, 'grad_clip': {\n",
" 'norm_type': 2,\n",
" 'max_norm': 2.0\n",
" }\n",
" }\n",
" },\n",
" \"lr_scheduler\": {\n",
" \"type\": \"CosineAnnealingLR\",\n",
" \"T_max\": T_max,\n",
" \"eta_min\": 1e-5,\n",
" \"options\": {\n",
" \"by_epoch\": False,\n",
" \"warmup\": {\n",
" 'lr_scheduler': {\n",
" 'type': 'CosineAnnealingLR',\n",
" 'T_max': T_max,\n",
" 'eta_min': 1e-5,\n",
" 'options': {\n",
" 'by_epoch': False,\n",
" 'warmup': {\n",
" 'type': 'LinearWarmup',\n",
" 'warmup_ratio': 0.1,\n",
" \"warmup_iters\": 200\n",
" 'warmup_iters': 200\n",
" }\n",
" }\n",
" },\n",
" \"hooks\": [\n",
" {\"type\": \"CheckpointHook\", \"by_epoch\": False, \"interval\": EVAL_INTERVAL, \"max_checkpoint_num\": 1},\n",
" {\"type\": \"EvaluationHook\", \"by_epoch\": False, \"interval\": EVAL_INTERVAL},\n",
" {\"type\": \"BestCkptSaverHook\",\n",
" \"metric_key\": \"acc\",\n",
" \"save_best\": True, \"rule\": \"max\", \"max_checkpoint_num\": 1},\n",
" {\"type\": \"TextLoggerHook\",\n",
" \"by_epoch\": True, # Whether EpochBasedTrainer is used\n",
" \"interval\": 5},\n",
" {\"type\": \"TensorboardHook\", \"by_epoch\": False, \"interval\": 5}\n",
" 'hooks': [\n",
" {'type': 'CheckpointHook', 'by_epoch': False, 'interval': EVAL_INTERVAL, 'max_checkpoint_num': 1},\n",
" {'type': 'EvaluationHook', 'by_epoch': False, 'interval': EVAL_INTERVAL},\n",
" {'type': 'BestCkptSaverHook',\n",
" 'metric_key': 'acc',\n",
" 'save_best': True, 'rule': 'max', 'max_checkpoint_num': 1},\n",
" {'type': 'TextLoggerHook',\n",
" 'by_epoch': True, # Whether EpochBasedTrainer is used\n",
" 'interval': 5},\n",
" {'type': 'TensorboardHook', 'by_epoch': False, 'interval': 5}\n",
" ]\n",
" },\n",
" \"evaluation\": {\n",
" \"dataloader\": {\n",
" \"batch_size_per_gpu\": BATCH_SIZE,\n",
" \"workers_per_gpu\": 1,\n",
" \"shuffle\": False,\n",
" \"drop_last\": False,\n",
" \"pin_memory\": True\n",
" 'evaluation': {\n",
" 'dataloader': {\n",
" 'batch_size_per_gpu': BATCH_SIZE,\n",
" 'workers_per_gpu': 1,\n",
" 'shuffle': False,\n",
" 'drop_last': False,\n",
" 'pin_memory': True\n",
" },\n",
" \"metrics\": [\n",
" {\"type\": \"my_metric\", \"vocab_size\": tokenizer.vocab_size}\n",
" 'metrics': [\n",
" {'type': 'my_metric', 'vocab_size': tokenizer.vocab_size}\n",
" ]\n",
" }\n",
"})"
@@ -1884,16 +1885,16 @@
}
],
"source": [
"tb_dir = os.path.join(WORK_DIR, \"tensorboard_output\")\n",
"tb_dir = os.path.join(WORK_DIR, 'tensorboard_output')\n",
"fname = os.listdir(tb_dir)[0]\n",
"tb_path = os.path.join(tb_dir, fname)\n",
"#\n",
"data = read_tensorboard_file(tb_path)\n",
"print(data.keys())\n",
"_ = plot_image(data, \"loss\", 0.9)\n",
"_ = plot_image(data, \"lr\", 0)\n",
"_ = plot_image(data, \"evaluation/acc\", 0)\n",
"_ = plot_image(data, \"evaluation/loss\", 0)"
"_ = plot_image(data, 'loss', 0.9)\n",
"_ = plot_image(data, 'lr', 0)\n",
"_ = plot_image(data, 'evaluation/acc', 0)\n",
"_ = plot_image(data, 'evaluation/loss', 0)"
]
},
{

2
setup.cfg
View File

@@ -21,7 +21,7 @@ ignore-words-list = patten,nd,ty,mot,hist,formating,winn,gool,datas,wan,confids
[flake8]
max-line-length = 120
select = B,C,E,F,P,T4,W,B9
ignore = F401,F405,F821,W503,E251
ignore = F401,F403,F405,F821,W503,E251
exclude = docs/src,*.pyi,.git
[darglint]