diff --git a/run/readme.md b/run/readme.md new file mode 100644 index 00000000000..f39f48a2ede --- /dev/null +++ b/run/readme.md @@ -0,0 +1,5 @@ +# Install the thing in install-dev.sh + +HF_TOKEN=<> OPENAI_API_KEY=<> helm-run --conf-paths run_entries.conf --suite v1 --max-eval-instances 16 --models-to-run fireworks/llama3-405fp16 +helm-summarize --suite v1 +helm-server diff --git a/run/run_entries_helm_lite.conf b/run/run_entries_helm_lite.conf new file mode 100644 index 00000000000..c2683fa8168 --- /dev/null +++ b/run/run_entries_helm_lite.conf @@ -0,0 +1,343 @@ +# HELM-lite `RunSpec`s for the benchmarking after v0.2.3. + +entries: [ + ##### Question Answering ##### + + ### Reading comprehension ### + # Scenarios: BoolQ, NarrativeQA, QuAC + {description: "boolq:model=text,data_augmentation=canonical", priority: 1} + {description: "narrative_qa:model=text,data_augmentation=canonical", priority: 2} + {description: "quac:model=text,data_augmentation=canonical", priority: 1} + + ### Reading comprehension and closedbook QA variants ### + # Scenarios: NaturalQuestions + {description: "natural_qa:model=text,mode=openbook_longans,data_augmentation=canonical", priority: 1} + {description: "natural_qa:model=text,mode=closedbook,data_augmentation=canonical", priority: 1} + + ### Closed-book QA with multiple choice ### + # Scenarios: TruthfulQA, MMLU + {description: "truthful_qa:model=text,task=mc_single,data_augmentation=canonical", priority: 1} + + {description: "mmlu:model=text,subject=abstract_algebra,data_augmentation=canonical", priority: 2} + {description: "mmlu:model=text,subject=anatomy,data_augmentation=canonical", priority: 3} + {description: "mmlu:model=text,subject=college_chemistry,data_augmentation=canonical", priority: 2} + {description: "mmlu:model=text,subject=computer_security,data_augmentation=canonical", priority: 2} + {description: "mmlu:model=text,subject=econometrics,data_augmentation=canonical", priority: 2} + {description: "mmlu:model=text,subject=global_facts,data_augmentation=canonical", priority: 3} + {description: "mmlu:model=text,subject=jurisprudence,data_augmentation=canonical", priority: 3} + {description: "mmlu:model=text,subject=philosophy,data_augmentation=canonical", priority: 3} + {description: "mmlu:model=text,subject=professional_medicine,data_augmentation=canonical", priority: 3} + {description: "mmlu:model=text,subject=us_foreign_policy,data_augmentation=canonical", priority: 2} + {description: "mmlu:model=text,subject=astronomy,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=business_ethics,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=clinical_knowledge,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=college_biology,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=college_computer_science,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=college_mathematics,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=college_medicine,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=college_physics,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=conceptual_physics,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=electrical_engineering,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=elementary_mathematics,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=formal_logic,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=high_school_biology,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=high_school_chemistry,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=high_school_computer_science,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=high_school_european_history,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=high_school_geography,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=high_school_government_and_politics,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=high_school_macroeconomics,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=high_school_mathematics,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=high_school_microeconomics,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=high_school_physics,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=high_school_psychology,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=high_school_statistics,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=high_school_us_history,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=high_school_world_history,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=human_aging,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=human_sexuality,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=international_law,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=logical_fallacies,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=machine_learning,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=management,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=marketing,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=medical_genetics,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=miscellaneous,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=moral_disputes,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=moral_scenarios,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=nutrition,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=prehistory,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=professional_accounting,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=professional_law,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=professional_psychology,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=public_relations,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=security_studies,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=sociology,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=virology,data_augmentation=canonical", priority: 4} + {description: "mmlu:model=text,subject=world_religions,data_augmentation=canonical", priority: 4} + + ##### Sentiment Analysis ##### + + # Scenarios: IMDB + {description: "imdb:model=text,data_augmentation=canonical", priority: 1} + + ##### (Miscellaneous) Text Classification ##### + + # Scenarios: RAFT + {description: "raft:subset=ade_corpus_v2,model=text,data_augmentation=canonical", priority: 2} + {description: "raft:subset=banking_77,model=text,data_augmentation=canonical", priority: 2} + {description: "raft:subset=neurips_impact_statement_risks,model=text,data_augmentation=canonical", priority: 2} + {description: "raft:subset=one_stop_english,model=text,data_augmentation=canonical", priority: 2} + {description: "raft:subset=overruling,model=text,data_augmentation=canonical", priority: 2} + {description: "raft:subset=semiconductor_org_types,model=text,data_augmentation=canonical", priority: 2} + {description: "raft:subset=tweet_eval_hate,model=text,data_augmentation=canonical", priority: 2} + {description: "raft:subset=twitter_complaints,model=text,data_augmentation=canonical", priority: 2} + {description: "raft:subset=systematic_review_inclusion,model=text,data_augmentation=canonical", priority: 2} + {description: "raft:subset=tai_safety_research,model=text,data_augmentation=canonical", priority: 2} + {description: "raft:subset=terms_of_service,model=text,data_augmentation=canonical", priority: 2} + + ##### Toxicity Detection ##### + + # Scenarios: CivilComments + {description: "civil_comments:model=text,demographic=all,data_augmentation=canonical", priority: 1} + {description: "civil_comments:model=text,demographic=male,data_augmentation=canonical", priority: 2} + {description: "civil_comments:model=text,demographic=female,data_augmentation=canonical", priority: 2} + {description: "civil_comments:model=text,demographic=LGBTQ,data_augmentation=canonical", priority: 2} + {description: "civil_comments:model=text,demographic=christian,data_augmentation=canonical", priority: 2} + {description: "civil_comments:model=text,demographic=muslim,data_augmentation=canonical", priority: 2} + {description: "civil_comments:model=text,demographic=other_religions,data_augmentation=canonical", priority: 2} + {description: "civil_comments:model=text,demographic=black,data_augmentation=canonical", priority: 2} + {description: "civil_comments:model=text,demographic=white,data_augmentation=canonical", priority: 2} + + ##### Knowledge ##### + + # Scenarios: WikiFact + # For WikiFact, we sampled the following 10 relation types, which cover diverse topics + # across general facts, humanities, social sciences and STEM. + {description: "wikifact:model=text,k=5,subject=plaintiff", priority: 2} + {description: "wikifact:model=text,k=5,subject=place_of_birth", priority: 2} + {description: "wikifact:model=text,k=5,subject=medical_condition_treated", priority: 2} + {description: "wikifact:model=text,k=5,subject=instance_of", priority: 2} + {description: "wikifact:model=text,k=5,subject=part_of", priority: 2} + {description: "wikifact:model=text,k=5,subject=currency", priority: 2} + {description: "wikifact:model=text,k=5,subject=position_held", priority: 2} + {description: "wikifact:model=text,k=5,subject=author", priority: 2} + {description: "wikifact:model=text,k=5,subject=discoverer_or_inventor", priority: 2} + {description: "wikifact:model=text,k=5,subject=symptoms_and_signs", priority: 2} + {description: "wikifact:model=text,k=5,subject=applies_to_jurisdiction", priority: 4} + {description: "wikifact:model=text,k=5,subject=field_of_work", priority: 4} + {description: "wikifact:model=text,k=5,subject=member_of_political_party", priority: 4} + {description: "wikifact:model=text,k=5,subject=native_language", priority: 4} + {description: "wikifact:model=text,k=5,subject=occupation", priority: 4} + {description: "wikifact:model=text,k=5,subject=employer", priority: 4} + {description: "wikifact:model=text,k=5,subject=atomic_number", priority: 4} + {description: "wikifact:model=text,k=5,subject=measured_physical_quantity", priority: 4} + {description: "wikifact:model=text,k=5,subject=solved_by", priority: 4} + {description: "wikifact:model=text,k=5,subject=number_of_processor_cores", priority: 4} + {description: "wikifact:model=text,k=5,subject=file_extension", priority: 4} + {description: "wikifact:model=text,k=5,subject=basic_form_of_government", priority: 4} + {description: "wikifact:model=text,k=5,subject=owned_by", priority: 4} + {description: "wikifact:model=text,k=5,subject=instrument", priority: 4} + {description: "wikifact:model=text,k=5,subject=central_bank", priority: 4} + {description: "wikifact:model=text,k=5,subject=located_in_the_administrative_territorial_entity", priority: 4} + {description: "wikifact:model=text,k=5,subject=office_held_by_head_of_government", priority: 4} + {description: "wikifact:model=text,k=5,subject=movement", priority: 4} + {description: "wikifact:model=text,k=5,subject=genre", priority: 4} + {description: "wikifact:model=text,k=5,subject=capital_of", priority: 4} + {description: "wikifact:model=text,k=5,subject=named_after", priority: 4} + {description: "wikifact:model=text,k=5,subject=religion", priority: 4} + {description: "wikifact:model=text,k=5,subject=languages_spoken_written_or_signed", priority: 4} + {description: "wikifact:model=text,k=5,subject=headquarters_location", priority: 4} + {description: "wikifact:model=text,k=5,subject=defendant", priority: 4} + {description: "wikifact:model=text,k=5,subject=award_received", priority: 4} + {description: "wikifact:model=text,k=5,subject=country", priority: 4} + {description: "wikifact:model=text,k=5,subject=creator", priority: 4} + {description: "wikifact:model=text,k=5,subject=manufacturer", priority: 4} + {description: "wikifact:model=text,k=5,subject=developer", priority: 4} + {description: "wikifact:model=text,k=5,subject=location_of_discovery", priority: 4} + {description: "wikifact:model=text,k=5,subject=twinned_administrative_body", priority: 4} + {description: "wikifact:model=text,k=5,subject=office_held_by_head_of_state", priority: 4} + {description: "wikifact:model=text,k=5,subject=participating_team", priority: 4} + {description: "wikifact:model=text,k=5,subject=place_of_death", priority: 4} + {description: "wikifact:model=text,k=5,subject=drug_or_therapy_used_for_treatment", priority: 4} + {description: "wikifact:model=text,k=5,subject=genetic_association", priority: 4} + {description: "wikifact:model=text,k=5,subject=statement_describes", priority: 4} + {description: "wikifact:model=text,k=5,subject=repealed_by", priority: 4} + {description: "wikifact:model=text,k=5,subject=record_label", priority: 4} + {description: "wikifact:model=text,k=5,subject=country_of_citizenship", priority: 4} + {description: "wikifact:model=text,k=5,subject=location", priority: 4} + {description: "wikifact:model=text,k=5,subject=programming_language", priority: 4} + {description: "wikifact:model=text,k=5,subject=subclass_of", priority: 4} + {description: "wikifact:model=text,k=5,subject=continent", priority: 4} + {description: "wikifact:model=text,k=5,subject=laws_applied", priority: 4} + {description: "wikifact:model=text,k=5,subject=operating_system", priority: 4} + {description: "wikifact:model=text,k=5,subject=head_of_state", priority: 4} + {description: "wikifact:model=text,k=5,subject=subsidiary", priority: 4} + {description: "wikifact:model=text,k=5,subject=capital", priority: 4} + {description: "wikifact:model=text,k=5,subject=original_language_of_film_or_TV_show", priority: 4} + {description: "wikifact:model=text,k=5,subject=official_language", priority: 4} + {description: "wikifact:model=text,k=5,subject=overrules", priority: 4} + {description: "wikifact:model=text,k=5,subject=therapeutic_area", priority: 4} + {description: "wikifact:model=text,k=5,subject=language_of_work_or_name", priority: 4} + {description: "wikifact:model=text,k=5,subject=position_played_on_team", priority: 4} + {description: "wikifact:model=text,k=5,subject=stock_exchange", priority: 4} + {description: "wikifact:model=text,k=5,subject=original_network", priority: 4} + {description: "wikifact:model=text,k=5,subject=industry", priority: 4} + {description: "wikifact:model=text,k=5,subject=member_of", priority: 4} + {description: "wikifact:model=text,k=5,subject=shares_border_with", priority: 4} + {description: "wikifact:model=text,k=5,subject=country_of_origin", priority: 4} + {description: "wikifact:model=text,k=5,subject=has_part", priority: 4} + {description: "wikifact:model=text,k=5,subject=diplomatic_relation", priority: 4} + {description: "wikifact:model=text,k=5,subject=member_of_sports_team", priority: 4} + {description: "wikifact:model=text,k=5,subject=director", priority: 4} + {description: "wikifact:model=text,k=5,subject=time_of_discovery_or_invention", priority: 4} + {description: "wikifact:model=text,k=5,subject=majority_opinion_by", priority: 4} + {description: "wikifact:model=text,k=5,subject=head_of_government", priority: 4} + {description: "wikifact:model=text,k=5,subject=educated_at", priority: 4} + {description: "wikifact:model=text,k=5,subject=influenced_by", priority: 4} + {description: "wikifact:model=text,k=5,subject=location_of_formation", priority: 4} + {description: "wikifact:model=text,k=5,subject=electron_configuration", priority: 4} + {description: "wikifact:model=text,k=5,subject=recommended_unit_of_measurement", priority: 4} + {description: "wikifact:model=text,k=5,subject=composer", priority: 4} + {description: "wikifact:model=text,k=5,subject=work_location", priority: 4} + + ##### Reasoning ##### + + ### Synthetic ### + # Scenarios: Synthetic reasoning (abstract symbols), Synthetic reasoning (natural language), bAbI + {description: "synthetic_reasoning:model=text_code,mode=pattern_match", priority: 2} + {description: "synthetic_reasoning:model=text_code,mode=variable_substitution", priority: 2} + {description: "synthetic_reasoning:model=text_code,mode=induction", priority: 2} + + {description: "synthetic_reasoning_natural:model=text_code,difficulty=easy", priority: 2} + {description: "synthetic_reasoning_natural:model=text_code,difficulty=hard", priority: 2} + + {description: "babi_qa:model=text_code,task=all", priority: 2} + {description: "babi_qa:model=text_code,task=1", priority: 3} + {description: "babi_qa:model=text_code,task=2", priority: 4} + {description: "babi_qa:model=text_code,task=3", priority: 2} + {description: "babi_qa:model=text_code,task=4", priority: 3} + {description: "babi_qa:model=text_code,task=5", priority: 4} + {description: "babi_qa:model=text_code,task=6", priority: 4} + {description: "babi_qa:model=text_code,task=7", priority: 4} + {description: "babi_qa:model=text_code,task=8", priority: 4} + {description: "babi_qa:model=text_code,task=9", priority: 4} + {description: "babi_qa:model=text_code,task=10", priority: 4} + {description: "babi_qa:model=text_code,task=11", priority: 4} + {description: "babi_qa:model=text_code,task=12", priority: 4} + {description: "babi_qa:model=text_code,task=13", priority: 4} + {description: "babi_qa:model=text_code,task=14", priority: 4} + {description: "babi_qa:model=text_code,task=15", priority: 2} + {description: "babi_qa:model=text_code,task=16", priority: 4} + {description: "babi_qa:model=text_code,task=17", priority: 4} + {description: "babi_qa:model=text_code,task=18", priority: 4} + {description: "babi_qa:model=text_code,task=19", priority: 2} + {description: "babi_qa:model=text_code,task=20", priority: 4} + + {description: "dyck_language:model=text_code,num_parenthesis_pairs=2", priority: 4} + {description: "dyck_language:model=text_code,num_parenthesis_pairs=3", priority: 2} + {description: "dyck_language:model=text_code,num_parenthesis_pairs=4", priority: 4} + + ### Real ### + # Scenarios: MATH, GSM8K + {description: "math:model=text_code,subject=number_theory,level=1,use_official_examples=True", priority: 2} + {description: "math:model=text_code,subject=intermediate_algebra,level=1,use_official_examples=True", priority: 2} + {description: "math:model=text_code,subject=algebra,level=1,use_official_examples=True", priority: 2} + {description: "math:model=text_code,subject=prealgebra,level=1,use_official_examples=True", priority: 2} + {description: "math:model=text_code,subject=geometry,level=1,use_official_examples=True", priority: 2} + {description: "math:model=text_code,subject=counting_and_probability,level=1,use_official_examples=True", priority: 2} + {description: "math:model=text_code,subject=precalculus,level=1,use_official_examples=True", priority: 2} + + {description: "math:model=text_code,subject=number_theory,level=2,use_official_examples=True", priority: 4} + {description: "math:model=text_code,subject=intermediate_algebra,level=2,use_official_examples=True", priority: 4} + {description: "math:model=text_code,subject=algebra,level=2,use_official_examples=True", priority: 4} + {description: "math:model=text_code,subject=prealgebra,level=2,use_official_examples=True", priority: 4} + {description: "math:model=text_code,subject=geometry,level=2,use_official_examples=True", priority: 4} + {description: "math:model=text_code,subject=counting_and_probability,level=2,use_official_examples=True", priority: 4} + {description: "math:model=text_code,subject=precalculus,level=2,use_official_examples=True", priority: 4} + + {description: "math:model=text_code,subject=number_theory,level=3,use_official_examples=True", priority: 3} + {description: "math:model=text_code,subject=intermediate_algebra,level=3,use_official_examples=True", priority: 3} + {description: "math:model=text_code,subject=algebra,level=3,use_official_examples=True", priority: 3} + {description: "math:model=text_code,subject=prealgebra,level=3,use_official_examples=True", priority: 3} + {description: "math:model=text_code,subject=geometry,level=3,use_official_examples=True", priority: 3} + {description: "math:model=text_code,subject=counting_and_probability,level=3,use_official_examples=True", priority: 3} + {description: "math:model=text_code,subject=precalculus,level=3,use_official_examples=True", priority: 3} + + {description: "math:model=text_code,subject=number_theory,level=4,use_official_examples=True", priority: 4} + {description: "math:model=text_code,subject=intermediate_algebra,level=4,use_official_examples=True", priority: 4} + {description: "math:model=text_code,subject=algebra,level=4,use_official_examples=True", priority: 4} + {description: "math:model=text_code,subject=prealgebra,level=4,use_official_examples=True", priority: 4} + {description: "math:model=text_code,subject=geometry,level=4,use_official_examples=True", priority: 4} + {description: "math:model=text_code,subject=counting_and_probability,level=4,use_official_examples=True", priority: 4} + {description: "math:model=text_code,subject=precalculus,level=4,use_official_examples=True", priority: 4} + + {description: "math:model=text_code,subject=number_theory,level=5,use_official_examples=True", priority: 3} + {description: "math:model=text_code,subject=intermediate_algebra,level=5,use_official_examples=True", priority: 3} + {description: "math:model=text_code,subject=algebra,level=5,use_official_examples=True", priority: 3} + {description: "math:model=text_code,subject=prealgebra,level=5,use_official_examples=True", priority: 3} + {description: "math:model=text_code,subject=geometry,level=5,use_official_examples=True", priority: 3} + {description: "math:model=text_code,subject=counting_and_probability,level=5,use_official_examples=True", priority: 3} + {description: "math:model=text_code,subject=precalculus,level=5,use_official_examples=True", priority: 3} + + {description: "math:model=text_code,subject=number_theory,level=1,use_chain_of_thought=True", priority: 2} + {description: "math:model=text_code,subject=intermediate_algebra,level=1,use_chain_of_thought=True", priority: 2} + {description: "math:model=text_code,subject=algebra,level=1,use_chain_of_thought=True", priority: 2} + {description: "math:model=text_code,subject=prealgebra,level=1,use_chain_of_thought=True", priority: 2} + {description: "math:model=text_code,subject=geometry,level=1,use_chain_of_thought=True", priority: 2} + {description: "math:model=text_code,subject=counting_and_probability,level=1,use_chain_of_thought=True", priority: 2} + {description: "math:model=text_code,subject=precalculus,level=1,use_chain_of_thought=True", priority: 2} + + {description: "math:model=text_code,subject=number_theory,level=2,use_chain_of_thought=True", priority: 4} + {description: "math:model=text_code,subject=intermediate_algebra,level=2,use_chain_of_thought=True", priority: 4} + {description: "math:model=text_code,subject=algebra,level=2,use_chain_of_thought=True", priority: 4} + {description: "math:model=text_code,subject=prealgebra,level=2,use_chain_of_thought=True", priority: 4} + {description: "math:model=text_code,subject=geometry,level=2,use_chain_of_thought=True", priority: 4} + {description: "math:model=text_code,subject=counting_and_probability,level=2,use_chain_of_thought=True", priority: 4} + {description: "math:model=text_code,subject=precalculus,level=2,use_chain_of_thought=True", priority: 4} + + {description: "math:model=text_code,subject=number_theory,level=3,use_chain_of_thought=True", priority: 3} + {description: "math:model=text_code,subject=intermediate_algebra,level=3,use_chain_of_thought=True", priority: 3} + {description: "math:model=text_code,subject=algebra,level=3,use_chain_of_thought=True", priority: 3} + {description: "math:model=text_code,subject=prealgebra,level=3,use_chain_of_thought=True", priority: 3} + {description: "math:model=text_code,subject=geometry,level=3,use_chain_of_thought=True", priority: 3} + {description: "math:model=text_code,subject=counting_and_probability,level=3,use_chain_of_thought=True", priority: 3} + {description: "math:model=text_code,subject=precalculus,level=3,use_chain_of_thought=True", priority: 3} + + {description: "math:model=text_code,subject=number_theory,level=4,use_chain_of_thought=True", priority: 4} + {description: "math:model=text_code,subject=intermediate_algebra,level=4,use_chain_of_thought=True", priority: 4} + {description: "math:model=text_code,subject=algebra,level=4,use_chain_of_thought=True", priority: 4} + {description: "math:model=text_code,subject=prealgebra,level=4,use_chain_of_thought=True", priority: 4} + {description: "math:model=text_code,subject=geometry,level=4,use_chain_of_thought=True", priority: 4} + {description: "math:model=text_code,subject=counting_and_probability,level=4,use_chain_of_thought=True", priority: 4} + {description: "math:model=text_code,subject=precalculus,level=4,use_chain_of_thought=True", priority: 4} + + {description: "math:model=text_code,subject=number_theory,level=5,use_chain_of_thought=True", priority: 3} + {description: "math:model=text_code,subject=intermediate_algebra,level=5,use_chain_of_thought=True", priority: 3} + {description: "math:model=text_code,subject=algebra,level=5,use_chain_of_thought=True", priority: 3} + {description: "math:model=text_code,subject=prealgebra,level=5,use_chain_of_thought=True", priority: 3} + {description: "math:model=text_code,subject=geometry,level=5,use_chain_of_thought=True", priority: 3} + {description: "math:model=text_code,subject=counting_and_probability,level=5,use_chain_of_thought=True", priority: 3} + {description: "math:model=text_code,subject=precalculus,level=5,use_chain_of_thought=True", priority: 3} + + {description: "gsm:model=text_code", priority: 2} + + ### Legal reasoning ### + # Scenarios: LegalSupport, LSAT + {description: "legal_support:model=text_code", priority: 2} + + {description: "lsat_qa:model=text_code,task=all", priority: 2} + {description: "lsat_qa:model=text_code,task=grouping", priority: 3} + {description: "lsat_qa:model=text_code,task=ordering", priority: 3} + {description: "lsat_qa:model=text_code,task=assignment", priority: 3} + {description: "lsat_qa:model=text_code,task=miscellaneous", priority: 3} + + ### Data processing ### + # Scenarios: Entity matching, Data imputation + {description: "entity_matching:model=text,dataset=Beer", priority: 2} + {description: "entity_matching:model=text,dataset=Abt_Buy", priority: 2} + {description: "entity_matching:model=text,dataset=Dirty_iTunes_Amazon", priority: 2} + + {description: "entity_data_imputation:model=text,dataset=Buy", priority: 2} + {description: "entity_data_imputation:model=text,dataset=Restaurant", priority: 2} +] \ No newline at end of file diff --git a/src/helm/clients/openai_client.py b/src/helm/clients/openai_client.py index c671f07273f..49e8d0b0b1f 100644 --- a/src/helm/clients/openai_client.py +++ b/src/helm/clients/openai_client.py @@ -12,8 +12,8 @@ TokenizationRequest, TokenizationRequestResult, ) -from .client import CachingClient, truncate_sequence, generate_uid_for_multimodal_prompt from helm.tokenizers.tokenizer import Tokenizer +from .client import CachingClient, truncate_sequence, generate_uid_for_multimodal_prompt try: import openai @@ -42,11 +42,13 @@ def __init__( api_key: Optional[str] = None, org_id: Optional[str] = None, base_url: Optional[str] = None, + model_name: Optional[str] = None, ): super().__init__(cache_config=cache_config) self.tokenizer = tokenizer self.tokenizer_name = tokenizer_name self.client = OpenAI(api_key=api_key, organization=org_id, base_url=base_url) + self.model_name = model_name def _is_chat_model_engine(self, model_engine: str) -> bool: if model_engine == "gpt-3.5-turbo-instruct": @@ -56,7 +58,7 @@ def _is_chat_model_engine(self, model_engine: str) -> bool: return False def _get_model_for_request(self, request: Request) -> str: - return request.model_engine + return self.model_name def _get_cache_key(self, raw_request: Dict, request: Request): cache_key = CachingClient.make_cache_key(raw_request, request) @@ -234,19 +236,20 @@ def _to_raw_completion_request(self, request: Request) -> Dict[str, Any]: "temperature": request.temperature, "n": request.num_completions, "max_tokens": request.max_tokens, - "best_of": request.top_k_per_token, - "logprobs": request.top_k_per_token, + # "best_of" request.top_k_per_token, + "logprobs": 1, + # request.top_k_per_token, "stop": request.stop_sequences or None, # API doesn't like empty list "top_p": request.top_p, "presence_penalty": request.presence_penalty, "frequency_penalty": request.frequency_penalty, "echo": request.echo_prompt, } - + # OpenAI doesn't let you ask for more completions than the number of # per-token candidates. - raw_request["best_of"] = max(raw_request["best_of"], raw_request["n"]) - raw_request["logprobs"] = max(raw_request["logprobs"], raw_request["n"]) + # raw_request["best_of"] = max(raw_request["best_of"], raw_request["n"]) + # raw_request["logprobs"] = max(raw_request["logprobs"], raw_request["n"]) return raw_request diff --git a/src/helm/config/model_deployments.yaml b/src/helm/config/model_deployments.yaml index 8898864f2ce..8759d6c7e1d 100644 --- a/src/helm/config/model_deployments.yaml +++ b/src/helm/config/model_deployments.yaml @@ -2363,3 +2363,63 @@ model_deployments: max_sequence_length: 64000 client_spec: class_name: "helm.clients.reka_client.RekaClient" + + - name: fireworks/llama3-405fp16 + model_name: fireworks/llama3-405fp16 + tokenizer_name: meta/llama-3.1-8b + max_sequence_length: 8192 + client_spec: + class_name: "helm.clients.openai_client.OpenAIClient" + args: + base_url: "https://api.fireworks.ai/inference/v1/" + model_name: "accounts/fireworks/models/llama-v3p1-405b-instruct-fp16" + + - name: fireworks/llama3-405fp8 + model_name: fireworks/llama3-405fp8 + tokenizer_name: meta/llama-3.1-8b + max_sequence_length: 8192 + client_spec: + class_name: "helm.clients.openai_client.OpenAIClient" + args: + base_url: "https://api.fireworks.ai/inference/v1/" + model_name: "accounts/fireworks/models/llama-v3p1-405b-instruct" + + - name: fireworks/llama3-405fp82 + model_name: fireworks/llama3-405fp82 + tokenizer_name: meta/llama-3.1-8b + max_sequence_length: 8192 + client_spec: + class_name: "helm.clients.openai_client.OpenAIClient" + args: + base_url: "https://api.fireworks.ai/inference/v1/" + model_name: "accounts/fireworks/models/llama-v3p1-405b-instruct" + + - name: together/llama3-405fp8 + model_name: together/llama3-405fp8 + tokenizer_name: meta/llama-3.1-8b + max_sequence_length: 4096 + client_spec: + class_name: "helm.clients.openai_client.OpenAIClient" + args: + base_url: "https://api.together.xyz/v1" + model_name: "meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo" + + - name: fireworks/llama3-405fp8-kv + model_name: fireworks/llama3-405fp8-kv + tokenizer_name: meta/llama-3.1-8b + max_sequence_length: 8192 + client_spec: + class_name: "helm.clients.openai_client.OpenAIClient" + args: + base_url: "http://0.0.0.0:80/v1/" + model_name: "accounts/models/models/meta-llama-v3p1-405b-instruct-fp8-405b-kv-had" + + - name: vllm/llama3-405fp8 + model_name: vllm/llama3-405fp8 + tokenizer_name: meta/llama-3.1-8b + max_sequence_length: 8192 + client_spec: + class_name: "helm.clients.openai_client.OpenAIClient" + args: + base_url: "http://0.0.0.0:8000/v1/" + model_name: "meta-llama/Meta-Llama-3.1-405B-Instruct-FP8" \ No newline at end of file