diff --git a/examples/pytorch/chatglm6b/chatglm_trainer.py b/examples/pytorch/chatglm6b/chatglm_trainer.py index efa4dfce..84167713 100644 --- a/examples/pytorch/chatglm6b/chatglm_trainer.py +++ b/examples/pytorch/chatglm6b/chatglm_trainer.py @@ -6,7 +6,7 @@ from transformers.deepspeed import is_deepspeed_zero3_enabled from modelscope import EpochBasedTrainer, get_logger -logger = get_logger(__name__) +logger = get_logger() class Seq2SeqTrainer(EpochBasedTrainer): diff --git a/examples/pytorch/llm/_common.py b/examples/pytorch/llm/_common.py index 77952f3c..86531b0e 100644 --- a/examples/pytorch/llm/_common.py +++ b/examples/pytorch/llm/_common.py @@ -1,4 +1,3 @@ -import ast import datetime as dt import math import os @@ -7,12 +6,12 @@ import re import sys from dataclasses import dataclass, field from functools import partial +from types import MethodType from typing import Any, Callable, Dict, List, Optional, Tuple, Union import json import matplotlib.pyplot as plt import numpy as np -# import torch import torch.nn as nn import torch.optim as optim @@ -33,35 +32,35 @@ from torch.optim import Optimizer from torch.optim import lr_scheduler as lrs from torch.optim.lr_scheduler import _LRScheduler as LRScheduler from torch.utils.data import Dataset -# from torchmetrics import Accuracy, MeanMetric -# from tqdm import tqdm from transformers import (AutoConfig, AutoModelForCausalLM, AutoTokenizer, GenerationConfig, HfArgumentParser, TextStreamer) -# from modelscope import (Model, MsDataset, get_logger, read_config, snapshot_download) from modelscope.metrics.base import Metric from modelscope.metrics.builder import METRICS -from modelscope.models.nlp.chatglm2 import ChatGLM2Tokenizer -from modelscope.msdatasets.dataset_cls.custom_datasets import \ - TorchCustomDataset +from modelscope.models.nlp.chatglm2 import ChatGLM2Config, ChatGLM2Tokenizer +from modelscope.models.nlp.llama2 import Llama2Config, Llama2Tokenizer from modelscope.swift import LoRAConfig, Swift from modelscope.trainers import EpochBasedTrainer from modelscope.utils.config import Config, ConfigDict from modelscope.utils.registry import default_group -# COLOR, COLOR_S = '#FFE2D9', '#FF7043' -PROMPT = """Human: {instruction} -AI: """ +PROMPT = """Here's a conversation between a human and an AI assistant. \ +The AI assistant provides detailed, friendly answers for the human. + +### Human: +{instruction} + +### AI: +""" logger = get_logger() os.environ['TOKENIZERS_PARALLELISM'] = 'true' -# def _get_version(work_dir: str) -> int: @@ -84,7 +83,7 @@ def get_work_dir(work_dir: str) -> str: work_dir = os.path.abspath(work_dir) version = _get_version(work_dir) time = dt.datetime.now().strftime('%Y%m%d-%H%M%S') - # + work_dir = os.path.join(work_dir, f'v{version}-{time}') logger.info(f'work_dir: {work_dir}') return work_dir @@ -109,9 +108,8 @@ def select_device(device: Union[List[int], str]) -> Device: if torch.cuda.is_initialized(): logger.warning('CUDA has been initialized! Device selection fails!') return torch.device('cuda:0') - # + device_ids, device_str = _format_device(device) - # os.environ['CUDA_VISIBLE_DEVICES'] = device_str log_s = 'Using device: ' if len(device_ids) == 0: @@ -157,11 +155,9 @@ def get_T_max(dataset_len: int, batch_size: int, max_epochs: int, def tokenize_function(example: Dict[str, Optional[str]], tokenizer, max_length: Optional[int] = 2048) -> Dict[str, Any]: - """Only applicable to baichuan and chatglm2. Other models need to be tested""" instruction: str = example['instruction'] input_ = example['input'] if input_ is not None and input_ != '': - # instruction = instruction + '\n' if input_.startswith('输入：'): instruction = instruction + input_[3:] else: @@ -171,7 +167,7 @@ def tokenize_function(example: Dict[str, Optional[str]], src_input_ids: List[int] = tokenizer( src_text, return_attention_mask=False, add_special_tokens=True)['input_ids'] - # + tgt_input_ids = [] if output is not None: tgt_input_ids += tokenizer( @@ -182,17 +178,17 @@ def tokenize_function(example: Dict[str, Optional[str]], else: labels = None input_ids = src_input_ids + tgt_input_ids - # + if max_length is not None: input_ids = input_ids[-max_length:] if labels is not None: labels = labels[-max_length:] - # + return {'input_ids': input_ids, 'labels': labels} def stat_dataset(dataset: HfDataset) -> None: - """Statistical analysis was performed on the data set""" + """Statistical analysis was performed on the dataset""" _token_len = [] for d in dataset: _token_len.append(len(d['input_ids'])) @@ -224,7 +220,7 @@ def data_collate_fn(batch: List[Dict[str, Any]], tokenizer) -> Dict[str, Any]: torch.ones(len(input_ids[i]), dtype=torch.int64) for i in range(len(input_ids)) ] - # + input_ids = pad_sequence( input_ids, batch_first=True, padding_value=tokenizer.pad_token_id) attention_mask = pad_sequence( @@ -240,11 +236,11 @@ def data_collate_fn(batch: List[Dict[str, Any]], tokenizer) -> Dict[str, Any]: def print_model_info(model: Module, name: Optional[str] = None) -> None: if name is None: name = model.__class__.__name__ - # + n_params = sum(p.numel() for p in model.parameters()) n_grads = sum(p.numel() for p in model.parameters() if p.requires_grad) n_buffers = sum(p.numel() for p in model.buffers()) - # + n_params /= 1e6 n_grads /= 1e6 n_buffers /= 1e6 @@ -276,7 +272,7 @@ class MyMetric(Metric): def add(self, outputs: Dict[str, Any], inputs: Dict[str, Any]) -> None: loss: Tensor = outputs.loss self.loss.update(loss) - # + labels: Tensor = inputs['labels'] labels = labels[:, 1:] labels_mask = labels != -100 @@ -311,11 +307,11 @@ def _add_special_token(tokenizer): def get_baichuan_model_tokenizer(model_dir: str, load_model: bool = True, - add_special_token: bool = True): - sys.path.insert(0, model_dir) + add_special_token: bool = True, + torch_dtype: Dtype = torch.float16): model_config = AutoConfig.from_pretrained( model_dir, trust_remote_code=True) - model_config.torch_dtype = torch.float16 + model_config.torch_dtype = torch_dtype logger.info(f'model_config: {model_config}') tokenizer = AutoTokenizer.from_pretrained( model_dir, trust_remote_code=True) @@ -325,9 +321,9 @@ def get_baichuan_model_tokenizer(model_dir: str, model_dir, config=model_config, device_map='auto', - torch_dtype=torch.float16, + torch_dtype=torch_dtype, trust_remote_code=True) - # + if add_special_token: _add_special_token(tokenizer) return model, tokenizer @@ -335,17 +331,22 @@ def get_baichuan_model_tokenizer(model_dir: str, def get_chatglm2_model_tokenizer(model_dir: str, load_model: bool = True, - add_special_token: bool = True): + add_special_token: bool = True, + torch_dtype: Dtype = torch.float16): config = read_config(model_dir) - config['model'] = ConfigDict({'type': 'chatglm2-6b'}) + logger.info(config) + model_config = ChatGLM2Config.from_pretrained(model_dir) + model_config.torch_dtype = torch_dtype + logger.info(model_config) tokenizer = ChatGLM2Tokenizer.from_pretrained(model_dir) model = None if load_model: model = Model.from_pretrained( model_dir, cfg_dict=config, + config=model_config, device_map='auto', - torch_dtype=torch.float16) + torch_dtype=torch_dtype) if add_special_token: _add_special_token(tokenizer) return model, tokenizer @@ -353,39 +354,68 @@ def get_chatglm2_model_tokenizer(model_dir: str, def get_llama2_model_tokenizer(model_dir: str, load_model: bool = True, - add_special_token: bool = True): - config = AutoConfig.from_pretrained(model_dir) - tokenizer = AutoTokenizer.from_pretrained(model_dir) + add_special_token: bool = True, + torch_dtype: Dtype = torch.float16): + config = read_config(model_dir) + logger.info(config) + model_config = Llama2Config.from_pretrained(model_dir) + model_config.torch_dtype = torch_dtype + logger.info(model_config) + tokenizer = Llama2Tokenizer.from_pretrained(model_dir) model = None if load_model: - model = AutoModelForCausalLM.from_pretrained( + model = Model.from_pretrained( model_dir, - config=config, + cfg_dict=config, + config=model_config, device_map='auto', - torch_dtype=torch.float16, - ) + torch_dtype=torch_dtype) if add_special_token: _add_special_token(tokenizer) return model, tokenizer +def get_model_tokenizer(model_type: str, + load_model: bool = True, + add_special_token: bool = True, + torch_dtype: Dtype = torch.float16): + # ### Loading Model and Tokenizer + if model_type == 'baichuan-7b': + model_dir = snapshot_download('baichuan-inc/baichuan-7B', 'v1.0.7') + model, tokenizer = get_baichuan_model_tokenizer( + model_dir, load_model, add_special_token, torch_dtype) + elif model_type == 'baichuan-13b': + model_dir = snapshot_download('baichuan-inc/Baichuan-13B-Base', + 'v1.0.3') + model, tokenizer = get_baichuan_model_tokenizer( + model_dir, load_model, add_special_token, torch_dtype) + elif model_type == 'chatglm2': + model_dir = snapshot_download('ZhipuAI/chatglm2-6b', 'v1.0.6') + model, tokenizer = get_chatglm2_model_tokenizer( + model_dir, load_model, add_special_token, torch_dtype) + elif model_type == 'llama2-7b': + model_dir = snapshot_download('modelscope/Llama-2-7b-ms', 'v1.0.2') + model, tokenizer = get_llama2_model_tokenizer(model_dir, load_model, + add_special_token, + torch_dtype) + else: + raise ValueError(f'model_type: {model_type}') + return model, tokenizer, model_dir + + def get_alpaca_en_zh_dataset( tokenize_function, only_val: bool = False, test_split_p: float = 0.01, split_seed: int = 42, data_sample: Optional[int] = None) -> Tuple[HfDataset, HfDataset]: - """ - split: Literal['train', 'validation', None] - """ - dataset_en: HfDataset = MsDataset.load( 'AI-ModelScope/alpaca-gpt4-data-en', split='train').to_hf_dataset() dataset_zh: HfDataset = MsDataset.load( 'AI-ModelScope/alpaca-gpt4-data-zh', split='train').to_hf_dataset() dataset_en = dataset_en.remove_columns(['text']) dataset: HfDataset = concatenate_datasets([dataset_zh, dataset_en]) - # + if data_sample is not None: dataset = dataset.select(range(data_sample)) dataset = dataset.train_test_split(test_split_p, seed=split_seed) @@ -394,7 +424,7 @@ def get_alpaca_en_zh_dataset( if tokenize_function is not None: dataset = dataset.map(tokenize_function) dataset = dataset.remove_columns(['instruction', 'input', 'output']) - # + if only_val: return None, dataset else: @@ -428,7 +458,7 @@ def tensorboard_smoothing(values: List[float], for i in range(len(values)): x = x * smooth + values[i] # Exponential decay res.append(x / norm_factor) - # + norm_factor *= smooth norm_factor += 1 return res @@ -441,7 +471,7 @@ def plot_image(tb_dir: str, dpi: int = 100) -> None: image_dir = os.path.join(os.path.dirname(tb_dir), 'images') os.makedirs(image_dir, exist_ok=True) - # + fname = os.listdir(tb_dir)[0] tb_path = os.path.join(tb_dir, fname) data = read_tensorboard_file(tb_path) @@ -464,3 +494,22 @@ def plot_image(tb_dir: str, ax.plot(steps, values, color=COLOR_S) fpath = os.path.join(image_dir, k.replace('/', '_')) plt.savefig(fpath, dpi=dpi, bbox_inches='tight') + + +def inference(data: Dict[str, Optional[str]], + model, + tokenizer, + streamer: Optional[TextStreamer] = None, + generation_config: Optional[GenerationConfig] = None, + tag: str = '[INFERENCE]') -> str: + input_ids = tokenize_function(data, tokenizer)['input_ids'] + print(f'{tag}{tokenizer.decode(input_ids)}', end='') + input_ids = torch.tensor(input_ids)[None].cuda() + attention_mask = torch.ones_like(input_ids) + generate_ids = model.generate( + input_ids=input_ids, + attention_mask=attention_mask, + streamer=streamer, + generation_config=generation_config) + output_text = tokenizer.decode(generate_ids[0]) + return output_text diff --git a/examples/pytorch/llm/llm_infer.py b/examples/pytorch/llm/llm_infer.py index cac59bee..a83dd6fe 100644 --- a/examples/pytorch/llm/llm_infer.py +++ b/examples/pytorch/llm/llm_infer.py @@ -3,7 +3,7 @@ from _common import * @dataclass -class Arguments: +class InferArguments: device: str = '0' # e.g. '-1'; '0'; '0,1' model_type: str = field( default='baichuan-7b', @@ -11,15 +11,17 @@ class Arguments: 'choices': ['baichuan-7b', 'baichuan-13b', 'chatglm2', 'llama2-7b'] }) - ckpt_fpath: str = '' # e.g. '/path/to/your/iter_xxx.pth' + sft_type: str = field( + default='lora', metadata={'choices': ['lora', 'full']}) + ckpt_fpath: str = '/path/to/your/iter_xxx.pth' eval_human: bool = False # False: eval test_dataset data_sample: Optional[int] = None - # + # sft_type: lora lora_target_modules: Optional[List[str]] = None lora_rank: int = 8 lora_alpha: int = 32 lora_dropout_p: float = 0.1 - # + max_new_tokens: int = 512 temperature: float = 0.9 top_k: int = 50 @@ -35,88 +37,79 @@ class Arguments: self.lora_target_modules = ['q_proj', 'k_proj', 'v_proj'] else: raise ValueError(f'model_type: {self.model_type}') - # + if not os.path.isfile(self.ckpt_fpath): - raise ValueError('Please enter a valid fpath') + raise ValueError(f'Please enter a valid fpath: {self.ckpt_fpath}') -def parse_args() -> Arguments: - args, = HfArgumentParser([Arguments]).parse_args_into_dataclasses() +def parse_args() -> InferArguments: + # return_remaining_strings=True for notebook compatibility + args, remaining_args = HfArgumentParser([ + InferArguments + ]).parse_args_into_dataclasses(return_remaining_strings=True) + logger.info(f'args: {args}') + if len(remaining_args) > 0: + logger.warning(f'remaining_args: {remaining_args}') return args -args = parse_args() -logger.info(args) -select_device(args.device) +def llm_infer(args: InferArguments) -> None: + select_device(args.device) + # ### Loading Model and Tokenizer + support_bf16 = torch.cuda.is_bf16_supported() + if not support_bf16: + logger.warning(f'support_bf16: {support_bf16}') + model, tokenizer, _ = get_model_tokenizer( + args.model_type, torch_dtype=torch.bfloat16) -# ### Loading Model and Tokenizer -if args.model_type == 'baichuan-7b': - model_dir = snapshot_download('baichuan-inc/baichuan-7B', 'v1.0.5') - model, tokenizer = get_baichuan_model_tokenizer(model_dir) -elif args.model_type == 'baichuan-13b': - model_dir = snapshot_download('baichuan-inc/Baichuan-13B-Base', 'v1.0.2') - model, tokenizer = get_baichuan_model_tokenizer(model_dir) -elif args.model_type == 'chatglm2': - model_dir = snapshot_download('ZhipuAI/chatglm2-6b', 'v1.0.6') - model, tokenizer = get_chatglm2_model_tokenizer(model_dir) -elif args.model_type == 'llama2-7b': - model_dir = snapshot_download('modelscope/Llama-2-7b-ms', 'v1.0.0') - model, tokenizer = get_llama2_model_tokenizer(model_dir) -else: - raise ValueError(f'model_type: {args.model_type}') + # ### Preparing lora + if args.sft_type == 'lora': + lora_config = LoRAConfig( + replace_modules=args.lora_target_modules, + rank=args.lora_rank, + lora_alpha=args.lora_alpha, + lora_dropout=args.lora_dropout_p, + pretrained_weights=args.ckpt_fpath) + logger.info(f'lora_config: {lora_config}') + Swift.prepare_model(model, lora_config) + elif args.sft_type == 'full': + state_dict = torch.load(args.ckpt_fpath, map_location='cpu') + model.load_state_dict(state_dict) + else: + raise ValueError(f'args.sft_type: {args.sft_type}') -# ### Preparing lora -lora_config = LoRAConfig( - replace_modules=args.lora_target_modules, - rank=args.lora_rank, - lora_alpha=args.lora_alpha, - lora_dropout=args.lora_dropout_p, - pretrained_weights=args.ckpt_fpath) -logger.info(f'lora_config: {lora_config}') -Swift.prepare_model(model, lora_config) -model.bfloat16() # Consistent with training + # ### Inference + streamer = TextStreamer( + tokenizer, skip_prompt=True, skip_special_tokens=True) + generation_config = GenerationConfig( + max_new_tokens=args.max_new_tokens, + temperature=args.temperature, + top_k=args.top_k, + top_p=args.top_p, + do_sample=True, + pad_token_id=tokenizer.eos_token_id) + logger.info(f'generation_config: {generation_config}') -# ### Inference -streamer = TextStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True) -generation_config = GenerationConfig( - max_new_tokens=args.max_new_tokens, - temperature=args.temperature, - top_k=args.top_k, - top_p=args.top_p, - do_sample=True, - pad_token_id=tokenizer.eos_token_id) -logger.info(generation_config) + if args.eval_human: + while True: + instruction = input('<<< ') + data = {'instruction': instruction, 'input': None, 'output': None} + inference(data, model, tokenizer, streamer, generation_config) + print('-' * 80) + else: + _, test_dataset = get_alpaca_en_zh_dataset( + None, True, split_seed=42, data_sample=args.data_sample) + mini_test_dataset = test_dataset.select(range(10)) + for data in mini_test_dataset: + output = data['output'] + data['output'] = None + inference(data, model, tokenizer, streamer, generation_config) + print() + print(f'[LABELS]{output}') + print('-' * 80) + # input('next[ENTER]') -def inference(data: Dict[str, Optional[str]]) -> str: - input_ids = tokenize_function(data, tokenizer)['input_ids'] - print(f'[TEST]{tokenizer.decode(input_ids)}', end='') - input_ids = torch.tensor(input_ids)[None].cuda() - attention_mask = torch.ones_like(input_ids) - generate_ids = model.generate( - input_ids=input_ids, - attention_mask=attention_mask, - streamer=streamer, - generation_config=generation_config) - output_text = tokenizer.decode(generate_ids[0]) - return output_text - - -if args.eval_human: - while True: - instruction = input('<<< ') - data = {'instruction': instruction, 'input': None, 'output': None} - inference(data) - print('-' * 80) -else: - _, test_dataset = get_alpaca_en_zh_dataset( - None, True, split_seed=42, data_sample=None) - mini_test_dataset = test_dataset.select(range(10)) - for data in mini_test_dataset: - output = data['output'] - data['output'] = None - inference(data) - print() - print(f'[LABELS]{output}') - print('-' * 80) - # input('next[ENTER]') +if __name__ == '__main__': + args = parse_args() + llm_infer(args) diff --git a/examples/pytorch/llm/llm_sft.py b/examples/pytorch/llm/llm_sft.py index 5e835625..eb830abf 100644 --- a/examples/pytorch/llm/llm_sft.py +++ b/examples/pytorch/llm/llm_sft.py @@ -1,20 +1,25 @@ # ### Setting up experimental environment. """ -pip install modelscope pip install numpy pandas matplotlib scikit-learn pip install transformers datasets conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch -c nvidia pip install tqdm tensorboard torchmetrics sentencepiece charset_normalizer pip install accelerate transformers_stream_generator -pip install numpy -U # Resolve torchmetrics dependencies and update numpy +# Install the latest version of modelscope from source +git clone https://github.com/modelscope/modelscope.git +cd modelscope +pip install . + +# Resolve torchmetrics dependencies and update numpy +pip install numpy -U """ from _common import * @dataclass -class Arguments: +class SftArguments: device: str = '0,1' # e.g. '-1'; '0'; '0,1' seed: int = 42 model_type: str = field( @@ -23,29 +28,52 @@ class Arguments: 'choices': ['baichuan-7b', 'baichuan-13b', 'chatglm2', 'llama2-7b'] }) + # baichuan-7b: 'lora': 16G; 'full': 80G + sft_type: str = field( + default='lora', metadata={'choices': ['lora', 'full']}) data_sample: Optional[int] = None - # + lora_target_modules: Optional[List[str]] = None lora_rank: int = 8 lora_alpha: int = 32 lora_dropout_p: float = 0.1 - # + gradient_checkpoint: bool = True batch_size: int = 1 max_epochs: int = 1 - eval_interval: int = 500 - learning_rate: float = 1e-4 + learning_rate: Optional[float] = None weight_decay: float = 0.01 n_accumulate_grad: int = 16 grad_clip_norm: float = 1. warmup_iters: int = 200 + + save_trainer_state: Optional[bool] = None + eval_interval: int = 500 + last_save_interval: Optional[int] = None last_max_checkpoint_num: int = 1 best_max_checkpoint_num: int = 1 - # logging_interval: int = 5 tb_interval: int = 5 def __post_init__(self): + if self.sft_type == 'lora': + if self.learning_rate is None: + self.learning_rate = 1e-4 + if self.save_trainer_state is None: + self.save_trainer_state = True + if self.last_save_interval is None: + self.last_save_interval = self.eval_interval + elif self.sft_type == 'full': + if self.learning_rate is None: + self.learning_rate = 1e-5 + if self.save_trainer_state is None: + self.save_trainer_state = False # save disk space + if self.last_save_interval is None: + # Saving the model takes a long time + self.last_save_interval = self.eval_interval * 4 + else: + raise ValueError(f'sft_type: {self.sft_type}') + if self.lora_target_modules is None: if self.model_type in {'baichuan-7b', 'baichuan-13b'}: self.lora_target_modules = ['W_pack'] @@ -57,181 +85,179 @@ class Arguments: raise ValueError(f'model_type: {self.model_type}') -def parse_args() -> Arguments: - args, = HfArgumentParser([Arguments]).parse_args_into_dataclasses() +def parse_args() -> SftArguments: + # return_remaining_strings=True for notebook compatibility + args, remaining_args = HfArgumentParser([ + SftArguments + ]).parse_args_into_dataclasses(return_remaining_strings=True) + logger.info(f'args: {args}') + if len(remaining_args) > 0: + logger.warning(f'remaining_args: {remaining_args}') return args -args = parse_args() -logger.info(args) -select_device(args.device) -seed_everything(args.seed) +def llm_sft(args: SftArguments) -> None: + select_device(args.device) + seed_everything(args.seed) -# ### Loading Model and Tokenizer -if args.model_type == 'baichuan-7b': - model_dir = snapshot_download('baichuan-inc/baichuan-7B', 'v1.0.5') - model, tokenizer = get_baichuan_model_tokenizer(model_dir) -elif args.model_type == 'baichuan-13b': - model_dir = snapshot_download('baichuan-inc/Baichuan-13B-Base', 'v1.0.2') - model, tokenizer = get_baichuan_model_tokenizer(model_dir) -elif args.model_type == 'chatglm2': - model_dir = snapshot_download('ZhipuAI/chatglm2-6b', 'v1.0.6') - model, tokenizer = get_chatglm2_model_tokenizer(model_dir) -elif args.model_type == 'llama2-7b': - model_dir = snapshot_download('modelscope/Llama-2-7b-ms', 'v1.0.0') - model, tokenizer = get_llama2_model_tokenizer(model_dir) -else: - raise ValueError(f'model_type: {args.model_type}') + # ### Loading Model and Tokenizer + support_bf16 = torch.cuda.is_bf16_supported() + if not support_bf16: + logger.warning(f'support_bf16: {support_bf16}') + model, tokenizer, model_dir = get_model_tokenizer( + args.model_type, torch_dtype=torch.bfloat16) -# -if args.gradient_checkpoint: - # baichuan13B does not implement the `get_input_embeddings` function - if args.model_type == 'baichuan-13b': + if args.gradient_checkpoint: + # baichuan-13b does not implement the `get_input_embeddings` function + if args.model_type == 'baichuan-13b': + model.__class__.get_input_embeddings = MethodType( + lambda self: self.model.embed_tokens, model) + model.gradient_checkpointing_enable() + model.enable_input_require_grads() - def get_input_embeddings(self): - return self.model.embed_tokens + # ### Preparing lora + if args.sft_type == 'lora': + lora_config = LoRAConfig( + replace_modules=args.lora_target_modules, + rank=args.lora_rank, + lora_alpha=args.lora_alpha, + lora_dropout=args.lora_dropout_p) + logger.info(f'lora_config: {lora_config}') + Swift.prepare_model(model, lora_config) - model.__class__.get_input_embeddings = get_input_embeddings.__get__( - model) - model.gradient_checkpointing_enable() - model.enable_input_require_grads() + show_freeze_layers(model) + print_model_info(model) + # check the device and dtype of the model + _p: Parameter = list(model.parameters())[-1] + logger.info(f'device: {_p.device}, dtype: {_p.dtype}') -# ### Preparing lora -lora_config = LoRAConfig( - replace_modules=args.lora_target_modules, - rank=args.lora_rank, - lora_alpha=args.lora_alpha, - lora_dropout=args.lora_dropout_p) -logger.info(f'lora_config: {lora_config}') -Swift.prepare_model(model, lora_config) -# -show_freeze_layers(model) -print_model_info(model) -_p: Parameter = list(model.parameters())[100] -logger.info(f'device: {_p.device}, dtype: {_p.dtype}') -model.bfloat16() + # ### Loading Dataset + tokenize_func = partial(tokenize_function, tokenizer=tokenizer) + train_dataset, val_dataset = get_alpaca_en_zh_dataset( + tokenize_func, split_seed=42, data_sample=args.data_sample) + # Data analysis + stat_dataset(train_dataset) + stat_dataset(val_dataset) + data_collator = partial(data_collate_fn, tokenizer=tokenizer) + print_example(train_dataset[0], tokenizer) -# ### Loading Dataset -tokenize_function = partial(tokenize_function, tokenizer=tokenizer) -train_dataset, val_dataset = get_alpaca_en_zh_dataset( - tokenize_function, split_seed=42, data_sample=args.data_sample) -# Data analysis -stat_dataset(train_dataset) -stat_dataset(val_dataset) -data_collate_fn = partial(data_collate_fn, tokenizer=tokenizer) -print_example(train_dataset[0], tokenizer) + # ### Setting Config + cfg_file = os.path.join(model_dir, 'configuration.json') -# ### Setting Config -cfg_file = os.path.join(model_dir, 'configuration.json') -# -T_max = get_T_max(len(train_dataset), args.batch_size, args.max_epochs, True) -work_dir = get_work_dir(f'runs/{args.model_type}') -config = Config({ - 'train': { - 'dataloader': { - 'batch_size_per_gpu': args.batch_size, - 'workers_per_gpu': 1, - 'shuffle': True, - 'drop_last': True, - 'pin_memory': True - }, - 'max_epochs': - args.max_epochs, - 'work_dir': - work_dir, - 'optimizer': { - 'type': 'AdamW', - 'lr': args.learning_rate, - 'weight_decay': args.weight_decay, - 'options': { - 'cumulative_iters': args.n_accumulate_grad, - 'grad_clip': { - 'norm_type': 2, - 'max_norm': args.grad_clip_norm + T_max = get_T_max( + len(train_dataset), args.batch_size, args.max_epochs, True) + work_dir = get_work_dir(f'runs/{args.model_type}') + config = Config({ + 'train': { + 'dataloader': { + 'batch_size_per_gpu': args.batch_size, + 'workers_per_gpu': 1, + 'shuffle': True, + 'drop_last': True, + 'pin_memory': True + }, + 'max_epochs': + args.max_epochs, + 'work_dir': + work_dir, + 'optimizer': { + 'type': 'AdamW', + 'lr': args.learning_rate, + 'weight_decay': args.weight_decay, + 'options': { + 'cumulative_iters': args.n_accumulate_grad, + 'grad_clip': { + 'norm_type': 2, + 'max_norm': args.grad_clip_norm + } } - } - }, - 'lr_scheduler': { - 'type': 'CosineAnnealingLR', - 'T_max': T_max, - 'eta_min': 0, - 'options': { - 'by_epoch': False, - 'warmup': { - 'type': 'LinearWarmup', - 'warmup_ratio': 0.1, - 'warmup_iters': args.warmup_iters + }, + 'lr_scheduler': { + 'type': 'CosineAnnealingLR', + 'T_max': T_max, + 'eta_min': args.learning_rate * 0.1, + 'options': { + 'by_epoch': False, + 'warmup': { + 'type': 'LinearWarmup', + 'warmup_ratio': 0.1, + 'warmup_iters': args.warmup_iters + } } - } + }, + 'hooks': [ + { + 'type': 'CheckpointHook', + 'by_epoch': False, + 'interval': args.last_save_interval, + 'max_checkpoint_num': args.last_max_checkpoint_num, + 'save_trainer_state': args.save_trainer_state + }, + { + 'type': 'EvaluationHook', + 'by_epoch': False, + 'interval': args.eval_interval + }, + { + 'type': 'BestCkptSaverHook', + 'metric_key': 'loss', + 'save_best': True, + 'rule': 'min', + 'max_checkpoint_num': args.best_max_checkpoint_num, + 'save_trainer_state': args.save_trainer_state + }, + { + 'type': 'TextLoggerHook', + 'by_epoch': True, # Whether EpochBasedTrainer is used + 'interval': args.logging_interval + }, + { + 'type': 'TensorboardHook', + 'by_epoch': False, + 'interval': args.tb_interval + } + ] }, - 'hooks': [ - { - 'type': 'CheckpointHook', - 'by_epoch': False, - 'interval': args.eval_interval, - 'max_checkpoint_num': args.last_max_checkpoint_num + 'evaluation': { + 'dataloader': { + 'batch_size_per_gpu': args.batch_size, + 'workers_per_gpu': 1, + 'shuffle': False, + 'drop_last': False, + 'pin_memory': True }, - { - 'type': 'EvaluationHook', - 'by_epoch': False, - 'interval': args.eval_interval - }, - { - 'type': 'BestCkptSaverHook', - 'metric_key': 'loss', - 'save_best': True, - 'rule': 'min', - 'max_checkpoint_num': args.best_max_checkpoint_num - }, - { - 'type': 'TextLoggerHook', - 'by_epoch': True, # Whether EpochBasedTrainer is used - 'interval': args.logging_interval - }, - { - 'type': 'TensorboardHook', - 'by_epoch': False, - 'interval': args.tb_interval - } - ] - }, - 'evaluation': { - 'dataloader': { - 'batch_size_per_gpu': args.batch_size, - 'workers_per_gpu': 1, - 'shuffle': False, - 'drop_last': False, - 'pin_memory': True - }, - 'metrics': [{ - 'type': 'my_metric', - 'vocab_size': tokenizer.vocab_size - }] - } -}) + 'metrics': [{ + 'type': 'my_metric', + 'vocab_size': tokenizer.vocab_size + }] + } + }) -# ### Finetuning + # ### Finetuning + + def cfg_modify_fn(cfg: Config) -> Config: + cfg.update(config) + return cfg + + trainer = EpochBasedTrainer( + model=model, + cfg_file=cfg_file, + data_collator=data_collator, + train_dataset=train_dataset, + eval_dataset=val_dataset, + remove_unused_data=True, + seed=42, + device='cpu', # No placement for model, leave the model to `device_map` + cfg_modify_fn=cfg_modify_fn, + ) + + trainer.train() + + # ### Visualization + tb_dir = os.path.join(work_dir, 'tensorboard_output') + plot_image(tb_dir, ['loss'], 0.9) -def cfg_modify_fn(cfg: Config) -> Config: - cfg.update(config) - return cfg - - -trainer = EpochBasedTrainer( - model=model, - cfg_file=cfg_file, - data_collator=data_collate_fn, - train_dataset=train_dataset, - eval_dataset=val_dataset, - remove_unused_data=True, - seed=42, - device='cpu', # No placement for model, leave the model to `device_map` - cfg_modify_fn=cfg_modify_fn, -) - -trainer.train() - -# ### Visualization -tb_dir = os.path.join(work_dir, 'tensorboard_output') -plot_image(tb_dir, ['loss'], 0.9) +if __name__ == '__main__': + args = parse_args() + llm_sft(args) diff --git a/modelscope/exporters/base.py b/modelscope/exporters/base.py index d105afd2..1bfed176 100644 --- a/modelscope/exporters/base.py +++ b/modelscope/exporters/base.py @@ -9,7 +9,7 @@ from modelscope.utils.constant import ModelFile from modelscope.utils.logger import get_logger from .builder import build_exporter -logger = get_logger(__name__) +logger = get_logger() class Exporter(ABC): diff --git a/modelscope/exporters/cv/cartoon_translation_exporter.py b/modelscope/exporters/cv/cartoon_translation_exporter.py index 79b859cb..0cfd746f 100644 --- a/modelscope/exporters/cv/cartoon_translation_exporter.py +++ b/modelscope/exporters/cv/cartoon_translation_exporter.py @@ -9,7 +9,7 @@ from modelscope.exporters.tf_model_exporter import TfModelExporter from modelscope.models.cv.cartoon import CartoonModel from modelscope.utils.logger import get_logger -logger = get_logger(__name__) +logger = get_logger() if version.parse(tf.__version__) < version.parse('2'): pass diff --git a/modelscope/exporters/nlp/csanmt_for_translation_exporter.py b/modelscope/exporters/nlp/csanmt_for_translation_exporter.py index 6b69595d..65b55b43 100644 --- a/modelscope/exporters/nlp/csanmt_for_translation_exporter.py +++ b/modelscope/exporters/nlp/csanmt_for_translation_exporter.py @@ -13,7 +13,7 @@ from modelscope.utils.constant import Tasks from modelscope.utils.logger import get_logger from modelscope.utils.test_utils import compare_arguments_nested -logger = get_logger(__name__) +logger = get_logger() if tf.__version__ >= '2.0': tf = tf.compat.v1 diff --git a/modelscope/models/nlp/chatglm/configuration.py b/modelscope/models/nlp/chatglm/configuration.py index 18fdca0f..5ecf3484 100644 --- a/modelscope/models/nlp/chatglm/configuration.py +++ b/modelscope/models/nlp/chatglm/configuration.py @@ -1,9 +1,10 @@ """ ChatGLM model configuration """ from transformers.configuration_utils import PretrainedConfig -from transformers.utils import logging -logger = logging.get_logger(__name__) +from modelscope.utils import logger as logging + +logger = logging.get_logger() class ChatGLMConfig(PretrainedConfig): diff --git a/modelscope/models/nlp/chatglm/quantization.py b/modelscope/models/nlp/chatglm/quantization.py index 9994d9c4..4e568c71 100644 --- a/modelscope/models/nlp/chatglm/quantization.py +++ b/modelscope/models/nlp/chatglm/quantization.py @@ -6,9 +6,10 @@ from typing import List import torch from torch.nn import Linear from torch.nn.parameter import Parameter -from transformers.utils import logging -logger = logging.get_logger(__name__) +from modelscope.utils import logger as logging + +logger = logging.get_logger() try: from cpm_kernels.kernels.base import LazyKernelCModule, KernelFunction, round_up diff --git a/modelscope/models/nlp/chatglm/text_generation.py b/modelscope/models/nlp/chatglm/text_generation.py index 64b82862..95ea33db 100644 --- a/modelscope/models/nlp/chatglm/text_generation.py +++ b/modelscope/models/nlp/chatglm/text_generation.py @@ -24,11 +24,12 @@ from transformers.modeling_outputs import ( from transformers.modeling_utils import PreTrainedModel from transformers.utils import (add_code_sample_docstrings, add_start_docstrings, - add_start_docstrings_to_model_forward, logging) + add_start_docstrings_to_model_forward) from modelscope.metainfo import Models from modelscope.models import MODELS, Model, TorchModel from modelscope.outputs import OutputKeys +from modelscope.utils import logger as logging from modelscope.utils.constant import Tasks from .configuration import ChatGLMConfig from .tokenization import ChatGLMTokenizer @@ -41,7 +42,7 @@ if sys.platform != 'darwin': torch._C._jit_override_can_fuse_on_cpu(True) torch._C._jit_override_can_fuse_on_gpu(True) -logger = logging.get_logger(__name__) +logger = logging.get_logger() _CHECKPOINT_FOR_DOC = 'THUDM/ChatGLM-6B' _CONFIG_FOR_DOC = 'ChatGLM6BConfig' diff --git a/modelscope/models/nlp/chatglm/tokenization.py b/modelscope/models/nlp/chatglm/tokenization.py index 77bcde55..f5f8cd0c 100644 --- a/modelscope/models/nlp/chatglm/tokenization.py +++ b/modelscope/models/nlp/chatglm/tokenization.py @@ -6,9 +6,11 @@ import numpy as np import sentencepiece as spm from transformers.tokenization_utils import PreTrainedTokenizer from transformers.tokenization_utils_base import BatchEncoding, EncodedInput -from transformers.utils import PaddingStrategy, logging +from transformers.utils import PaddingStrategy -logger = logging.get_logger(__name__) +from modelscope.utils import logger as logging + +logger = logging.get_logger() PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { 'THUDM/chatglm-6b': 2048, diff --git a/modelscope/models/nlp/chatglm2/configuration.py b/modelscope/models/nlp/chatglm2/configuration.py index 1583e886..ab40de0e 100644 --- a/modelscope/models/nlp/chatglm2/configuration.py +++ b/modelscope/models/nlp/chatglm2/configuration.py @@ -1,9 +1,10 @@ """ ChatGLM model configuration """ from transformers import PretrainedConfig -from transformers.utils import logging -logger = logging.get_logger(__name__) +from modelscope.utils import logger as logging + +logger = logging.get_logger() class ChatGLM2Config(PretrainedConfig): diff --git a/modelscope/models/nlp/chatglm2/quantization.py b/modelscope/models/nlp/chatglm2/quantization.py index 116bc0ea..a1e8b8f2 100644 --- a/modelscope/models/nlp/chatglm2/quantization.py +++ b/modelscope/models/nlp/chatglm2/quantization.py @@ -5,9 +5,10 @@ from typing import List import torch from torch.nn.parameter import Parameter -from transformers.utils import logging -logger = logging.get_logger(__name__) +from modelscope.utils import logger as logging + +logger = logging.get_logger() try: from cpm_kernels.kernels.base import LazyKernelCModule, KernelFunction, round_up diff --git a/modelscope/models/nlp/chatglm2/text_generation.py b/modelscope/models/nlp/chatglm2/text_generation.py index 082e16e7..1052b875 100644 --- a/modelscope/models/nlp/chatglm2/text_generation.py +++ b/modelscope/models/nlp/chatglm2/text_generation.py @@ -19,11 +19,11 @@ from transformers.generation.utils import (GenerationConfig, from transformers.modeling_outputs import (BaseModelOutputWithPast, CausalLMOutputWithPast) from transformers.modeling_utils import PreTrainedModel -from transformers.utils import logging from modelscope import Model, TorchModel from modelscope.metainfo import Models from modelscope.outputs import OutputKeys +from modelscope.utils import logger as logging from modelscope.utils.constant import Tasks from ... import MODELS from .configuration import ChatGLM2Config @@ -36,7 +36,7 @@ if sys.platform != 'darwin': torch._C._jit_override_can_fuse_on_cpu(True) torch._C._jit_override_can_fuse_on_gpu(True) -logger = logging.get_logger(__name__) +logger = logging.get_logger() _CHECKPOINT_FOR_DOC = 'THUDM/ChatGLM2-6B' _CONFIG_FOR_DOC = 'ChatGLM6BConfig' diff --git a/modelscope/models/nlp/fid_plug/backbone.py b/modelscope/models/nlp/fid_plug/backbone.py index 70c45633..5dcddcc1 100644 --- a/modelscope/models/nlp/fid_plug/backbone.py +++ b/modelscope/models/nlp/fid_plug/backbone.py @@ -26,10 +26,11 @@ import torch.nn.functional as F from torch import Tensor, nn from torch.nn.init import xavier_uniform_ from transformers import (BertConfig, BertModel, BertTokenizer, RobertaConfig, - RobertaModel, RobertaTokenizer, logging) + RobertaModel, RobertaTokenizer) from transformers.activations import ACT2FN from transformers.modeling_utils import PreTrainedModel +from modelscope.utils import logger as logging from .configuration import PlugConfig CONFIG_NAME = 'config.json' @@ -729,7 +730,7 @@ class PlugForConditionalGeneration(PlugPreTrainedModel): def __init__(self, config, checkpoint=None, dataset: str = 'default'): super().__init__(config) - self.logger = logging.get_logger(__name__) + self.logger = logging.get_logger() self.config = config if config.encoder == 'roberta': tokenizer = RobertaTokenizer.from_pretrained( diff --git a/modelscope/models/nlp/llama/backbone.py b/modelscope/models/nlp/llama/backbone.py index 120581a9..16be099f 100755 --- a/modelscope/models/nlp/llama/backbone.py +++ b/modelscope/models/nlp/llama/backbone.py @@ -35,7 +35,7 @@ from modelscope.utils.constant import Tasks from modelscope.utils.logger import get_logger from .configuration import LlamaConfig -logger = get_logger(__name__) +logger = get_logger() _CONFIG_FOR_DOC = 'LlamaConfig' diff --git a/modelscope/models/nlp/llama/tokenization.py b/modelscope/models/nlp/llama/tokenization.py index b3d24dd9..cd423683 100644 --- a/modelscope/models/nlp/llama/tokenization.py +++ b/modelscope/models/nlp/llama/tokenization.py @@ -29,7 +29,7 @@ from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer from modelscope.utils.logger import get_logger # This file is mainly copied from the llama code of transformers -logger = get_logger(__name__) +logger = get_logger() VOCAB_FILES_NAMES = {'vocab_file': 'tokenizer.model'} diff --git a/modelscope/models/nlp/llama/tokenization_fast.py b/modelscope/models/nlp/llama/tokenization_fast.py index 7aa0ac1b..13696b59 100644 --- a/modelscope/models/nlp/llama/tokenization_fast.py +++ b/modelscope/models/nlp/llama/tokenization_fast.py @@ -31,7 +31,7 @@ if is_sentencepiece_available(): else: LlamaTokenizer = None -logger = get_logger(__name__) +logger = get_logger() VOCAB_FILES_NAMES = { 'vocab_file': 'tokenizer.model', 'tokenizer_file': 'tokenizer.json' diff --git a/modelscope/models/nlp/llama2/backbone.py b/modelscope/models/nlp/llama2/backbone.py index c0983478..ee0d742b 100755 --- a/modelscope/models/nlp/llama2/backbone.py +++ b/modelscope/models/nlp/llama2/backbone.py @@ -36,7 +36,7 @@ from modelscope.utils.logger import get_logger from ... import MODELS from .configuration import Llama2Config -logger = get_logger(__name__) +logger = get_logger() _CONFIG_FOR_DOC = 'Llama2Config' @@ -570,7 +570,7 @@ class LlamaPreTrainedModel(TorchModel, PreTrainedModel): module.weight.data[module.padding_idx].zero_() def _set_gradient_checkpointing(self, module, value=False): - if isinstance(module, LlamaModel): + if isinstance(module, Llama2Model): module.gradient_checkpointing = value @classmethod diff --git a/modelscope/models/nlp/llama2/configuration.py b/modelscope/models/nlp/llama2/configuration.py index b95f9ddd..c9f38fe4 100644 --- a/modelscope/models/nlp/llama2/configuration.py +++ b/modelscope/models/nlp/llama2/configuration.py @@ -23,7 +23,7 @@ from transformers.configuration_utils import PretrainedConfig from modelscope.utils.logger import get_logger -logger = get_logger(__name__) +logger = get_logger() LLAMA_PRETRAINED_CONFIG_ARCHIVE_MAP = {} diff --git a/modelscope/models/nlp/llama2/tokenization.py b/modelscope/models/nlp/llama2/tokenization.py index d57c6017..bb276621 100644 --- a/modelscope/models/nlp/llama2/tokenization.py +++ b/modelscope/models/nlp/llama2/tokenization.py @@ -30,7 +30,7 @@ from modelscope.utils.logger import get_logger if TYPE_CHECKING: from transformers.pipelines.conversational import Conversation -logger = get_logger(__name__) +logger = get_logger() VOCAB_FILES_NAMES = {'vocab_file': 'tokenizer.model'} diff --git a/modelscope/models/nlp/llama2/tokenization_fast.py b/modelscope/models/nlp/llama2/tokenization_fast.py index 6cfae2ff..13862955 100644 --- a/modelscope/models/nlp/llama2/tokenization_fast.py +++ b/modelscope/models/nlp/llama2/tokenization_fast.py @@ -18,9 +18,11 @@ from typing import TYPE_CHECKING, Optional, Tuple from tokenizers import processors from transformers.tokenization_utils_fast import PreTrainedTokenizerFast -from transformers.utils import is_sentencepiece_available, logging +from transformers.utils import is_sentencepiece_available from transformers.utils.versions import require_version +from modelscope.utils import logger as logging + if TYPE_CHECKING: from transformers.pipelines.conversational import Conversation @@ -31,7 +33,7 @@ if is_sentencepiece_available(): else: Llama2Tokenizer = None -logger = logging.get_logger(__name__) +logger = logging.get_logger() VOCAB_FILES_NAMES = { 'vocab_file': 'tokenizer.model', 'tokenizer_file': 'tokenizer.json' diff --git a/modelscope/models/nlp/peer/backbone.py b/modelscope/models/nlp/peer/backbone.py index 2dca8dda..4bf376cd 100644 --- a/modelscope/models/nlp/peer/backbone.py +++ b/modelscope/models/nlp/peer/backbone.py @@ -36,7 +36,7 @@ from modelscope.utils.nlp.utils import parse_labels_in_order from .configuration import PeerConfig from .sas_utils import SequenceSideInfo -logger = logging.get_logger(__name__) +logger = logging.get_logger() PEER_PRETRAINED_MODEL_ARCHIVE_LIST = [ 'google/peer-small-generator', diff --git a/modelscope/models/nlp/peer/configuration.py b/modelscope/models/nlp/peer/configuration.py index da8b0a74..794b89f7 100644 --- a/modelscope/models/nlp/peer/configuration.py +++ b/modelscope/models/nlp/peer/configuration.py @@ -20,7 +20,7 @@ from transformers.configuration_utils import PretrainedConfig from modelscope.utils import logger as logging -logger = logging.get_logger(__name__) +logger = logging.get_logger() class PeerConfig(PretrainedConfig): diff --git a/modelscope/trainers/hooks/checkpoint/checkpoint_hook.py b/modelscope/trainers/hooks/checkpoint/checkpoint_hook.py index 49d20278..86ca61dd 100644 --- a/modelscope/trainers/hooks/checkpoint/checkpoint_hook.py +++ b/modelscope/trainers/hooks/checkpoint/checkpoint_hook.py @@ -76,7 +76,7 @@ class CheckpointHook(Hook): private_hub: Optional[bool] = True, hub_revision: Optional[str] = DEFAULT_REPOSITORY_REVISION, upload_strategy: Optional[str] = UploadStrategy.cancel, - save_trainer_state: Optional[bool] = True, + save_trainer_state: bool = True, **kwargs): self.interval = interval self.save_dir = save_dir @@ -302,6 +302,7 @@ class BestCkptSaverHook(CheckpointHook): max_checkpoint_num (int): The max number of checkpoint files, default None which means never delete anything. If the number exceeding the limit, checkpoints with worse metric will be deleted, which is judged by the `rule` and `metric_key` arguments. + save_trainer_state (bool): Save the trainer state for continue training, default True. The `BestCkptSaverHook` class accepts `output_sub_dir` and `output_dir` argument as its super class do. If neither of them are passed, the default value is `{save_dir}/output_best`. @@ -320,6 +321,7 @@ class BestCkptSaverHook(CheckpointHook): save_file_name: Optional[str] = None, restore_best: Optional[bool] = False, max_checkpoint_num: Optional[int] = 1, + save_trainer_state: bool = True, **kwargs): assert rule in ['max', 'min'], 'Only support "max" or "min" rule now.' output_kwargs = {} @@ -329,6 +331,7 @@ class BestCkptSaverHook(CheckpointHook): kwargs.pop('save_strategy', None) super().__init__( max_checkpoint_num=max_checkpoint_num, + save_trainer_state=save_trainer_state, **kwargs, **output_kwargs, ) diff --git a/modelscope/utils/data_collators.py b/modelscope/utils/data_collators.py index 044b1993..0981c836 100644 --- a/modelscope/utils/data_collators.py +++ b/modelscope/utils/data_collators.py @@ -7,7 +7,7 @@ from typing import Any, List, Optional, Tuple from .logger import get_logger -logger = get_logger(__name__) +logger = get_logger() class RemoveColumnsCollator: