NousResearch/atropos
1
0
Fork 0
mirror of https://github.com/NousResearch/atropos.git synced 2026-04-19 12:57:58 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

tokenizer bug

Browse source
This commit is contained in:
Jai Suphavadeeprasit 2026-03-11 19:35:12 -04:00
parent b457a678ce
commit 2f371e03fc
1 changed files with 1 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
atroposlib/envs/base.py
View file

@ -907,7 +907,7 @@ class BaseEnv(ABC):
"ensure your trainer handles this appropriately."
)
elif abort_on_any_max_length_exceeded and any(
[len(x) >= self.max_token_len for x in group["tokens"]]
[len(x) > self.max_token_len for x in group["tokens"]]
):
logger.warning("Token length is too long in a group, skipping...")
continue