diff --git a/local_evaluation.py b/local_evaluation.py index 1723e2a..db751ec 100644 --- a/local_evaluation.py +++ b/local_evaluation.py @@ -182,7 +182,7 @@ def generate_predictions(dataset_path, participant_model): return queries, ground_truths, predictions -def evaluate_predictions(queries, ground_truths_list, predictions, evaluation_model_name): +def evaluate_predictions(queries, ground_truths_list, predictions, evaluation_model_name, openai_client): """ Evaluates the predictions generated by a model against ground truth answers. @@ -199,7 +199,7 @@ def evaluate_predictions(queries, ground_truths_list, predictions, evaluation_mo if "chat" in evaluation_model_name.lower(): # now we are using chatgpt - openai_client = OpenAI() + # openai_client = OpenAI() n_miss, n_correct = 0, 0 system_message = get_system_message() @@ -211,6 +211,7 @@ def evaluate_predictions(queries, ground_truths_list, predictions, evaluation_mo # trim prediction to 75 tokens using Llama2 tokenizer prediction = trim_predictions_to_max_token_length(prediction) prediction = prediction.strip() + prediction_lowercase = prediction.lower() if "i don't know" in prediction_lowercase: n_miss += 1 @@ -220,7 +221,6 @@ def evaluate_predictions(queries, ground_truths_list, predictions, evaluation_mo for ground_truth in ground_truths: ground_truth_lowercase = ground_truth.lower() - prediction_lowercase = prediction.lower() messages = [ {"role": "system", "content": system_message}, {