diff --git a/atroposlib/utils/tokenize_for_trainer.py b/atroposlib/utils/tokenize_for_trainer.py index 722b09ea..c1187fe1 100644 --- a/atroposlib/utils/tokenize_for_trainer.py +++ b/atroposlib/utils/tokenize_for_trainer.py @@ -6,6 +6,7 @@ from atroposlib.type_definitions import Message # Roles that should be masked in the loss calculation (not used for training) UNMASKED_ROLES = ["assistant", "agent"] + def tokenize_for_trainer( tokenizer: PreTrainedTokenizer, chat: list[Message],