1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
|
<html><body>
<style>
body, h1, h2, h3, div, span, p, pre, a {
margin: 0;
padding: 0;
border: 0;
font-weight: inherit;
font-style: inherit;
font-size: 100%;
font-family: inherit;
vertical-align: baseline;
}
body {
font-size: 13px;
padding: 1em;
}
h1 {
font-size: 26px;
margin-bottom: 1em;
}
h2 {
font-size: 24px;
margin-bottom: 1em;
}
h3 {
font-size: 20px;
margin-bottom: 1em;
margin-top: 1em;
}
pre, code {
line-height: 1.5;
font-family: Monaco, 'DejaVu Sans Mono', 'Bitstream Vera Sans Mono', 'Lucida Console', monospace;
}
pre {
margin-top: 0.5em;
}
h1, h2, h3, p {
font-family: Arial, sans serif;
}
h1, h2, h3 {
border-bottom: solid #CCC 1px;
}
.toc_element {
margin-top: 0.5em;
}
.firstline {
margin-left: 2 em;
}
.method {
margin-top: 1em;
border: solid 1px #CCC;
padding: 1em;
background: #EEE;
}
.details {
font-weight: bold;
font-size: 14px;
}
</style>
<h1><a href="dialogflow_v2.html">Dialogflow API</a> . <a href="dialogflow_v2.projects.html">projects</a> . <a href="dialogflow_v2.projects.locations.html">locations</a> . <a href="dialogflow_v2.projects.locations.conversationModels.html">conversationModels</a> . <a href="dialogflow_v2.projects.locations.conversationModels.evaluations.html">evaluations</a></h1>
<h2>Instance Methods</h2>
<p class="toc_element">
<code><a href="#close">close()</a></code></p>
<p class="firstline">Close httplib2 connections.</p>
<p class="toc_element">
<code><a href="#create">create(parent, body=None, x__xgafv=None)</a></code></p>
<p class="firstline">Creates evaluation of a conversation model.</p>
<p class="toc_element">
<code><a href="#get">get(name, x__xgafv=None)</a></code></p>
<p class="firstline">Gets an evaluation of conversation model.</p>
<p class="toc_element">
<code><a href="#list">list(parent, pageSize=None, pageToken=None, x__xgafv=None)</a></code></p>
<p class="firstline">Lists evaluations of a conversation model.</p>
<p class="toc_element">
<code><a href="#list_next">list_next()</a></code></p>
<p class="firstline">Retrieves the next page of results.</p>
<h3>Method Details</h3>
<div class="method">
<code class="details" id="close">close()</code>
<pre>Close httplib2 connections.</pre>
</div>
<div class="method">
<code class="details" id="create">create(parent, body=None, x__xgafv=None)</code>
<pre>Creates evaluation of a conversation model.
Args:
parent: string, Required. The conversation model resource name. Format: `projects//locations//conversationModels/` (required)
body: object, The request body.
The object takes the form of:
{ # The request message for ConversationModels.CreateConversationModelEvaluation
"conversationModelEvaluation": { # Represents evaluation result of a conversation model. # Required. The conversation model evaluation to be created.
"createTime": "A String", # Output only. Creation time of this model.
"displayName": "A String", # Optional. The display name of the model evaluation. At most 64 bytes long.
"evaluationConfig": { # The configuration for model evaluation. # Optional. The configuration of the evaluation task.
"datasets": [ # Required. Datasets used for evaluation.
{ # InputDataset used to create model or do evaluation. NextID:5
"dataset": "A String", # Required. ConversationDataset resource name. Format: `projects//locations//conversationDatasets/`
},
],
"smartComposeConfig": { # Smart compose specific configuration for evaluation job. # Configuration for smart compose model evaluation.
"allowlistDocument": "A String", # The allowlist document resource name. Format: `projects//knowledgeBases//documents/`. Only used for smart compose model.
"maxResultCount": 42, # Required. The model to be evaluated can return multiple results with confidence score on each query. These results will be sorted by the descending order of the scores and we only keep the first max_result_count results as the final results to evaluate.
},
"smartReplyConfig": { # Smart reply specific configuration for evaluation job. # Configuration for smart reply model evaluation.
"allowlistDocument": "A String", # The allowlist document resource name. Format: `projects//knowledgeBases//documents/`. Only used for smart reply model.
"maxResultCount": 42, # Required. The model to be evaluated can return multiple results with confidence score on each query. These results will be sorted by the descending order of the scores and we only keep the first max_result_count results as the final results to evaluate.
},
},
"name": "A String", # The resource name of the evaluation. Format: `projects//conversationModels//evaluations/`
"rawHumanEvalTemplateCsv": "A String", # Output only. Human eval template in csv format. It takes real-world conversations provided through input dataset, generates example suggestions for customer to verify quality of the model. For Smart Reply, the generated csv file contains columns of Context, (Suggestions,Q1,Q2)*3, Actual reply. Context contains at most 10 latest messages in the conversation prior to the current suggestion. Q1: "Would you send it as the next message of agent?" Evaluated based on whether the suggest is appropriate to be sent by agent in current context. Q2: "Does the suggestion move the conversation closer to resolution?" Evaluated based on whether the suggestion provide solutions, or answers customer's question or collect information from customer to resolve the customer's issue. Actual reply column contains the actual agent reply sent in the context.
"smartReplyMetrics": { # The evaluation metrics for smart reply model. # Output only. Only available when model is for smart reply.
"allowlistCoverage": 3.14, # Percentage of target participant messages in the evaluation dataset for which similar messages have appeared at least once in the allowlist. Should be [0, 1].
"conversationCount": "A String", # Total number of conversations used to generate this metric.
"topNMetrics": [ # Metrics of top n smart replies, sorted by TopNMetric.n.
{ # Evaluation metrics when retrieving `n` smart replies with the model.
"n": 42, # Number of retrieved smart replies. For example, when `n` is 3, this evaluation contains metrics for when Dialogflow retrieves 3 smart replies with the model.
"recall": 3.14, # Defined as `number of queries whose top n smart replies have at least one similar (token match similarity above the defined threshold) reply as the real reply` divided by `number of queries with at least one smart reply`. Value ranges from 0.0 to 1.0 inclusive.
},
],
},
},
}
x__xgafv: string, V1 error format.
Allowed values
1 - v1 error format
2 - v2 error format
Returns:
An object of the form:
{ # This resource represents a long-running operation that is the result of a network API call.
"done": True or False, # If the value is `false`, it means the operation is still in progress. If `true`, the operation is completed, and either `error` or `response` is available.
"error": { # The `Status` type defines a logical error model that is suitable for different programming environments, including REST APIs and RPC APIs. It is used by [gRPC](https://github.com/grpc). Each `Status` message contains three pieces of data: error code, error message, and error details. You can find out more about this error model and how to work with it in the [API Design Guide](https://cloud.google.com/apis/design/errors). # The error result of the operation in case of failure or cancellation.
"code": 42, # The status code, which should be an enum value of google.rpc.Code.
"details": [ # A list of messages that carry the error details. There is a common set of message types for APIs to use.
{
"a_key": "", # Properties of the object. Contains field @type with type URL.
},
],
"message": "A String", # A developer-facing error message, which should be in English. Any user-facing error message should be localized and sent in the google.rpc.Status.details field, or localized by the client.
},
"metadata": { # Service-specific metadata associated with the operation. It typically contains progress information and common metadata such as create time. Some services might not provide such metadata. Any method that returns a long-running operation should document the metadata type, if any.
"a_key": "", # Properties of the object. Contains field @type with type URL.
},
"name": "A String", # The server-assigned name, which is only unique within the same service that originally returns it. If you use the default HTTP mapping, the `name` should be a resource name ending with `operations/{unique_id}`.
"response": { # The normal, successful response of the operation. If the original method returns no data on success, such as `Delete`, the response is `google.protobuf.Empty`. If the original method is standard `Get`/`Create`/`Update`, the response should be the resource. For other methods, the response should have the type `XxxResponse`, where `Xxx` is the original method name. For example, if the original method name is `TakeSnapshot()`, the inferred response type is `TakeSnapshotResponse`.
"a_key": "", # Properties of the object. Contains field @type with type URL.
},
}</pre>
</div>
<div class="method">
<code class="details" id="get">get(name, x__xgafv=None)</code>
<pre>Gets an evaluation of conversation model.
Args:
name: string, Required. The conversation model evaluation resource name. Format: `projects//conversationModels//evaluations/` (required)
x__xgafv: string, V1 error format.
Allowed values
1 - v1 error format
2 - v2 error format
Returns:
An object of the form:
{ # Represents evaluation result of a conversation model.
"createTime": "A String", # Output only. Creation time of this model.
"displayName": "A String", # Optional. The display name of the model evaluation. At most 64 bytes long.
"evaluationConfig": { # The configuration for model evaluation. # Optional. The configuration of the evaluation task.
"datasets": [ # Required. Datasets used for evaluation.
{ # InputDataset used to create model or do evaluation. NextID:5
"dataset": "A String", # Required. ConversationDataset resource name. Format: `projects//locations//conversationDatasets/`
},
],
"smartComposeConfig": { # Smart compose specific configuration for evaluation job. # Configuration for smart compose model evaluation.
"allowlistDocument": "A String", # The allowlist document resource name. Format: `projects//knowledgeBases//documents/`. Only used for smart compose model.
"maxResultCount": 42, # Required. The model to be evaluated can return multiple results with confidence score on each query. These results will be sorted by the descending order of the scores and we only keep the first max_result_count results as the final results to evaluate.
},
"smartReplyConfig": { # Smart reply specific configuration for evaluation job. # Configuration for smart reply model evaluation.
"allowlistDocument": "A String", # The allowlist document resource name. Format: `projects//knowledgeBases//documents/`. Only used for smart reply model.
"maxResultCount": 42, # Required. The model to be evaluated can return multiple results with confidence score on each query. These results will be sorted by the descending order of the scores and we only keep the first max_result_count results as the final results to evaluate.
},
},
"name": "A String", # The resource name of the evaluation. Format: `projects//conversationModels//evaluations/`
"rawHumanEvalTemplateCsv": "A String", # Output only. Human eval template in csv format. It takes real-world conversations provided through input dataset, generates example suggestions for customer to verify quality of the model. For Smart Reply, the generated csv file contains columns of Context, (Suggestions,Q1,Q2)*3, Actual reply. Context contains at most 10 latest messages in the conversation prior to the current suggestion. Q1: "Would you send it as the next message of agent?" Evaluated based on whether the suggest is appropriate to be sent by agent in current context. Q2: "Does the suggestion move the conversation closer to resolution?" Evaluated based on whether the suggestion provide solutions, or answers customer's question or collect information from customer to resolve the customer's issue. Actual reply column contains the actual agent reply sent in the context.
"smartReplyMetrics": { # The evaluation metrics for smart reply model. # Output only. Only available when model is for smart reply.
"allowlistCoverage": 3.14, # Percentage of target participant messages in the evaluation dataset for which similar messages have appeared at least once in the allowlist. Should be [0, 1].
"conversationCount": "A String", # Total number of conversations used to generate this metric.
"topNMetrics": [ # Metrics of top n smart replies, sorted by TopNMetric.n.
{ # Evaluation metrics when retrieving `n` smart replies with the model.
"n": 42, # Number of retrieved smart replies. For example, when `n` is 3, this evaluation contains metrics for when Dialogflow retrieves 3 smart replies with the model.
"recall": 3.14, # Defined as `number of queries whose top n smart replies have at least one similar (token match similarity above the defined threshold) reply as the real reply` divided by `number of queries with at least one smart reply`. Value ranges from 0.0 to 1.0 inclusive.
},
],
},
}</pre>
</div>
<div class="method">
<code class="details" id="list">list(parent, pageSize=None, pageToken=None, x__xgafv=None)</code>
<pre>Lists evaluations of a conversation model.
Args:
parent: string, Required. The conversation model resource name. Format: `projects//conversationModels/` (required)
pageSize: integer, Optional. Maximum number of evaluations to return in a single page. By default 100 and at most 1000.
pageToken: string, Optional. The next_page_token value returned from a previous list request.
x__xgafv: string, V1 error format.
Allowed values
1 - v1 error format
2 - v2 error format
Returns:
An object of the form:
{ # The response message for ConversationModels.ListConversationModelEvaluations
"conversationModelEvaluations": [ # The list of evaluations to return.
{ # Represents evaluation result of a conversation model.
"createTime": "A String", # Output only. Creation time of this model.
"displayName": "A String", # Optional. The display name of the model evaluation. At most 64 bytes long.
"evaluationConfig": { # The configuration for model evaluation. # Optional. The configuration of the evaluation task.
"datasets": [ # Required. Datasets used for evaluation.
{ # InputDataset used to create model or do evaluation. NextID:5
"dataset": "A String", # Required. ConversationDataset resource name. Format: `projects//locations//conversationDatasets/`
},
],
"smartComposeConfig": { # Smart compose specific configuration for evaluation job. # Configuration for smart compose model evaluation.
"allowlistDocument": "A String", # The allowlist document resource name. Format: `projects//knowledgeBases//documents/`. Only used for smart compose model.
"maxResultCount": 42, # Required. The model to be evaluated can return multiple results with confidence score on each query. These results will be sorted by the descending order of the scores and we only keep the first max_result_count results as the final results to evaluate.
},
"smartReplyConfig": { # Smart reply specific configuration for evaluation job. # Configuration for smart reply model evaluation.
"allowlistDocument": "A String", # The allowlist document resource name. Format: `projects//knowledgeBases//documents/`. Only used for smart reply model.
"maxResultCount": 42, # Required. The model to be evaluated can return multiple results with confidence score on each query. These results will be sorted by the descending order of the scores and we only keep the first max_result_count results as the final results to evaluate.
},
},
"name": "A String", # The resource name of the evaluation. Format: `projects//conversationModels//evaluations/`
"rawHumanEvalTemplateCsv": "A String", # Output only. Human eval template in csv format. It takes real-world conversations provided through input dataset, generates example suggestions for customer to verify quality of the model. For Smart Reply, the generated csv file contains columns of Context, (Suggestions,Q1,Q2)*3, Actual reply. Context contains at most 10 latest messages in the conversation prior to the current suggestion. Q1: "Would you send it as the next message of agent?" Evaluated based on whether the suggest is appropriate to be sent by agent in current context. Q2: "Does the suggestion move the conversation closer to resolution?" Evaluated based on whether the suggestion provide solutions, or answers customer's question or collect information from customer to resolve the customer's issue. Actual reply column contains the actual agent reply sent in the context.
"smartReplyMetrics": { # The evaluation metrics for smart reply model. # Output only. Only available when model is for smart reply.
"allowlistCoverage": 3.14, # Percentage of target participant messages in the evaluation dataset for which similar messages have appeared at least once in the allowlist. Should be [0, 1].
"conversationCount": "A String", # Total number of conversations used to generate this metric.
"topNMetrics": [ # Metrics of top n smart replies, sorted by TopNMetric.n.
{ # Evaluation metrics when retrieving `n` smart replies with the model.
"n": 42, # Number of retrieved smart replies. For example, when `n` is 3, this evaluation contains metrics for when Dialogflow retrieves 3 smart replies with the model.
"recall": 3.14, # Defined as `number of queries whose top n smart replies have at least one similar (token match similarity above the defined threshold) reply as the real reply` divided by `number of queries with at least one smart reply`. Value ranges from 0.0 to 1.0 inclusive.
},
],
},
},
],
"nextPageToken": "A String", # Token to retrieve the next page of results, or empty if there are no more results in the list.
}</pre>
</div>
<div class="method">
<code class="details" id="list_next">list_next()</code>
<pre>Retrieves the next page of results.
Args:
previous_request: The request for the previous page. (required)
previous_response: The response from the request for the previous page. (required)
Returns:
A request object that you can call 'execute()' on to request the next
page. Returns None if there are no more items in the collection.
</pre>
</div>
</body></html>
|
