Eval script consolidation (#238)

The script now supports:
   - YAML and JSON configurations
   - Dataset-specific parameters
   - Overriding configuration via command line
   - Detailed logging and error handling
This commit is contained in:
Andreas Köpf 2025-02-27 17:39:14 +01:00 committed by GitHub
parent 8a66d2a216
commit 850c1cf6f4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
40 changed files with 1111 additions and 670 deletions

View file

@ -0,0 +1,34 @@
# Combined configuration for Claude 3.5 Sonnet
model: "anthropic/claude-3.5-sonnet"
provider: "Anthropic"
output_dir: "results"
max_concurrent: 10
default_size: 50
default_seed: 45
categories:
- category: "algorithmic"
datasets:
- dataset: "count_primes"
- dataset: "game_of_life"
- dataset: "graph_color"
- dataset: "group_anagrams"
- dataset: "isomorphic_strings"
- dataset: "letter_counting"
- dataset: "letter_jumble"
- dataset: "manipulate_matrix"
- dataset: "number_filtering"
- dataset: "number_sorting"
- dataset: "palindrome"
- dataset: "pool_matrix"
- dataset: "ransom_note"
- dataset: "rotate_matrix"
- dataset: "sentence_reordering"
- dataset: "spell_backward"
- dataset: "spiral_matrix"
- dataset: "string_insertion"
- dataset: "string_manipulation"
- dataset: "string_synthesis"
- dataset: "word_ladder"
- dataset: "word_sequence_reversal"
- dataset: "word_sorting"