diff --git a/eval/eval.py b/eval/eval.py
index 8c3f2f96..5626aa79 100755
--- a/eval/eval.py
+++ b/eval/eval.py
@@ -139,6 +139,9 @@ class AsyncModelEvaluator:
                         params["extra_body"] = {"provider": {"order": [self.config.provider], "allow_fallbacks": False}}
 
                     completion = await self.client.chat.completions.create(**params)
+                    if completion.error:
+                        raise Exception(completion.error["message"])
+
                     response = completion.choices[0].message.content
 
                     if self.verbose: