1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
|
import pytest
from azure.ai.evaluation import evaluate, ToolCallAccuracyEvaluator
from azure.ai.evaluation._exceptions import EvaluationException
@pytest.mark.usefixtures("mock_model_config")
@pytest.mark.unittest
class TestEvaluate:
def test_tool_call_accuracy_evaluator_missing_inputs(self, mock_model_config):
tool_call_accuracy = ToolCallAccuracyEvaluator(model_config=mock_model_config)
# Test tool_calls provided but missing response
with pytest.raises(EvaluationException) as exc_info:
tool_call_accuracy(
query="Where is the Eiffel Tower?",
response="The Eiffel Tower is in Paris.",
tool_calls="Test",
tool_definitions={
"name": "fetch_weather",
"description": "Fetches the weather information for the specified location.",
"parameters": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The location to fetch weather for."
}
}
}
}
)
# Test with missing tool_definitions
with pytest.raises(EvaluationException) as exc_info:
tool_call_accuracy(
query="Where is the Eiffel Tower?",
tool_calls={
"type": "tool_call",
"tool_call": {
"id": "call_K21dwOxgCN2syn4qjutMVV7Z",
"type": "function",
"function": {
"name": "fetch_weather",
"arguments": {
"location": "Tokyo"
}
}
}
}
)
assert "Tool definitions must be provided." in str(exc_info.value)
# Test with missing tool_cools
with pytest.raises(EvaluationException) as exc_info:
tool_call_accuracy(
query="Where is the Eiffel Tower?",
tool_definitions={
"name": "fetch_weather",
"description": "Fetches the weather information for the specified location.",
"parameters": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The location to fetch weather for."
}
}
}
}
)
assert "Either response or tool_calls must be provided." in str(exc_info.value)
# Test response provided but missing tool_calls
with pytest.raises(EvaluationException) as exc_info:
tool_call_accuracy(
query="Where is the Eiffel Tower?",
response="The Eiffel Tower is in Paris.",
tool_definitions={
"name": "fetch_weather",
"description": "Fetches the weather information for the specified location.",
"parameters": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The location to fetch weather for."
}
}
}
}
)
assert "response does not have tool calls. Either provide tool_calls or response with tool calls." in str(exc_info.value)
# Test tool_calls provided but missing response
with pytest.raises(EvaluationException) as exc_info:
tool_call_accuracy(
query="Where is the Eiffel Tower?",
response="The Eiffel Tower is in Paris.",
tool_calls="Test",
tool_definitions={
"name": "fetch_weather",
"description": "Fetches the weather information for the specified location.",
"parameters": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The location to fetch weather for."
}
}
}
}
)
assert "Tool definition not found" in str(exc_info.value)
|
