1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
|
# ------------------------------------
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
# ------------------------------------
import pytest
from azure.ai.evaluation import F1ScoreEvaluator
from azure.ai.evaluation import (
AzureOpenAIGrader,
)
from azure.ai.evaluation import AzureOpenAIModelConfiguration
@pytest.fixture
def mock_aoai_model_config():
return AzureOpenAIModelConfiguration(
azure_deployment="...",
azure_endpoint="...",
api_key="...",
api_version="...",
)
@pytest.fixture
def mock_grader_config():
return {}
def _get_file(name):
"""Get the file from the unittest data folder."""
import os, pathlib
data_path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data")
return os.path.join(data_path, name)
@pytest.fixture
def questions_file():
return _get_file("questions.jsonl")
def simple_eval_function():
return "123"
@pytest.mark.unittest
class TestRemoteEvaluationFeatures:
def test_eval_name_mapping(self, mock_aoai_model_config, mock_grader_config):
"""
Test that the name mapping works properly.
TODO upgrade test to test EVERY KNOWN EVALUATOR.
"""
# create f1 score eval
f1_score_eval = F1ScoreEvaluator()
# create aoai grader
aoai_grader = AzureOpenAIGrader(model_config=mock_aoai_model_config, grader_config=mock_grader_config)
from azure.ai.evaluation._evaluate._evaluate import _map_names_to_builtins
from azure.ai.evaluation._eval_mapping import EVAL_CLASS_MAP
evaluators = {
"my_f1": f1_score_eval,
}
graders = {
"my_aoai": aoai_grader,
}
result = _map_names_to_builtins(evaluators, graders)
assert result is not None
assert result["my_f1"] == EVAL_CLASS_MAP[F1ScoreEvaluator]
assert result["my_aoai"] == AzureOpenAIGrader.id
|
