File: composebox_query_controller.h

package info (click to toggle)
chromium 139.0.7258.138-1
  • links: PTS, VCS
  • area: main
  • in suites: sid
  • size: 6,120,676 kB
  • sloc: cpp: 35,100,869; ansic: 7,163,530; javascript: 4,103,002; python: 1,436,920; asm: 946,517; xml: 746,709; pascal: 187,653; perl: 88,691; sh: 88,436; objc: 79,953; sql: 51,488; cs: 44,583; fortran: 24,137; makefile: 22,147; tcl: 15,277; php: 13,980; yacc: 8,984; ruby: 7,485; awk: 3,720; lisp: 3,096; lex: 1,327; ada: 727; jsp: 228; sed: 36
file content (381 lines) | stat: -rw-r--r-- 15,209 bytes parent folder | download | duplicates (3)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
// Copyright 2025 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#ifndef COMPONENTS_OMNIBOX_COMPOSEBOX_COMPOSEBOX_QUERY_CONTROLLER_H_
#define COMPONENTS_OMNIBOX_COMPOSEBOX_COMPOSEBOX_QUERY_CONTROLLER_H_
#include <memory>
#include <optional>
#include <string>
#include <vector>

#include "base/memory/ref_counted_memory.h"
#include "base/memory/scoped_refptr.h"
#include "base/time/time.h"
#include "base/unguessable_token.h"
#include "components/endpoint_fetcher/endpoint_fetcher.h"
#include "components/lens/lens_overlay_mime_type.h"
#include "components/lens/lens_overlay_request_id_generator.h"
#include "components/omnibox/composebox/composebox_query.mojom.h"
#include "components/search_engines/util.h"
#include "components/variations/variations_client.h"
#include "services/network/public/cpp/shared_url_loader_factory.h"
#include "third_party/lens_server_proto/lens_overlay_client_context.pb.h"
#include "third_party/lens_server_proto/lens_overlay_cluster_info.pb.h"
#include "third_party/lens_server_proto/lens_overlay_server.pb.h"
#include "third_party/lens_server_proto/lens_overlay_surface.pb.h"
#include "url/gurl.h"

class TemplateURLService;

#if !BUILDFLAG(IS_IOS)
#include "third_party/skia/include/core/SkBitmap.h"
#endif  // !BUILDFLAG(IS_IOS)

enum class QueryControllerState {
  // The initial state, before NotifySessionStarted() is called.
  kOff = 0,
  // The cluster info request is in flight.
  kAwaitingClusterInfoResponse = 1,
  // The cluster info response has been received and is valid.
  kClusterInfoReceived = 2,
  // The cluster info response was not received, or the cluster info has
  // expired.
  kClusterInfoInvalid = 3,
};

namespace version_info {
enum class Channel;
}  // namespace version_info

namespace signin {
class IdentityManager;
}  // namespace signin

namespace composebox {
// Image encoding options for an uploaded image.
struct ImageEncodingOptions {
  bool enable_webp_encoding;
  int max_size;
  int max_height;
  int max_width;
  int compression_quality;
};
}  // namespace composebox

// Callback type alias for the OAuth headers created.
using OAuthHeadersCreatedCallback =
    base::OnceCallback<void(std::vector<std::string>)>;
// Callback type alias for the request body proto created.
using FileUploadErrorType = composebox_query::mojom::FileUploadErrorType;
using RequestBodyProtoCreatedCallback =
    base::OnceCallback<void(lens::LensOverlayServerRequest,
                            std::optional<FileUploadErrorType>)>;
// Callback type alias for the upload progress.
using UploadProgressCallback =
    base::RepeatingCallback<void(uint64_t position, uint64_t total)>;
// Callback for when the query controller state changes.
using QueryControllerStateChangedCallback =
    base::RepeatingCallback<void(QueryControllerState state)>;
// Callback for when the file upload status changes.
using FileUploadStatus = composebox_query::mojom::FileUploadStatus;
using FileUploadStatusChangedCallback =
    base::RepeatingCallback<void(std::string file_token,
                                 FileUploadStatus status)>;

class ComposeboxQueryController {
 public:
  // Observer interface for the Page Handler to get updates on file upload
  class FileUploadStatusObserver : public base::CheckedObserver {
   public:
    virtual void OnFileUploadStatusChanged(
        const base::UnguessableToken& file_token,
        lens::MimeType mime_type,
        FileUploadStatus file_upload_status,
        const std::optional<FileUploadErrorType>& error_type) = 0;

   protected:
    ~FileUploadStatusObserver() override = default;
  };

  // Struct containing file information for a file upload.
  struct FileInfo {
   public:
    FileInfo();
    ~FileInfo();

    // Name of the selected file.
    std::string file_name;

    // Size in bytes of the file.
    uint64_t file_size_bytes = 0;

    // The time the file was selected.
    base::Time webui_selection_time;

    // Client-side unique identifier generated by UI. Used as the key in the
    // `active_files_` map.
    base::UnguessableToken file_token_;

    // The mime type of the file.
    lens::MimeType mime_type_;

    // Gets the file upload status.
    FileUploadStatus GetFileUploadStatus() const { return upload_status_; }

    // Gets the file upload error type.
    FileUploadErrorType GetFileUploadErrorType() const {
      return upload_error_type_;
    }

    // Gets the server response code.
    int GetResponseCode() const { return response_code_; }

    // Gets a pointer to the request ID for this request for testing.
    lens::LensOverlayRequestId* GetRequestIdForTesting() {
      return request_id_.get();
    }

   private:
    friend class ComposeboxQueryController;

    // Default to kNotUploaded, until UploadFile() is called.
    // Do not modify this field directly, use UpdateFileUploadStatus() instead.
    FileUploadStatus upload_status_ = FileUploadStatus::kNotUploaded;

    // The error type if the upload failed.
    FileUploadErrorType upload_error_type_ = FileUploadErrorType::kUnknown;

    // The request ID for this request. Set by StartFileUploadFlow().
    std::unique_ptr<lens::LensOverlayRequestId> request_id_;

    // When browser started the network request for the file upload.
    base::Time upload_network_request_start_time_;

    // When Lens server response was received.
    base::Time server_response_time_;

    // The network response code.
    int response_code_ = 0;

    // The request to be sent to the server. Will be set asynchronously after
    // StartFileUploadFlow() is called.
    std::unique_ptr<lens::LensOverlayServerRequest> request_body_;

    // The headers to attach to the request. Will be set asynchronously after
    // StartFileUploadFlow() is called.
    std::unique_ptr<std::vector<std::string>> request_headers_;

    // The access token fetcher used for getting OAuth for the file upload
    // request. Will be discarded after the OAuth headers are created.
    std::unique_ptr<signin::PrimaryAccountAccessTokenFetcher>
        file_upload_access_token_fetcher_;

    // The endpoint fetcher used for the file upload request.
    std::unique_ptr<endpoint_fetcher::EndpointFetcher>
        file_upload_endpoint_fetcher_;
  };

  ComposeboxQueryController(
      signin::IdentityManager* identity_manager,
      scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory,
      version_info::Channel channel,
      std::string locale,
      TemplateURLService* template_url_service,
      variations::VariationsClient* variations_client,
      bool send_lns_surface);
  virtual ~ComposeboxQueryController();

  // Session management. Virtual for testing.
  virtual void NotifySessionStarted();
  virtual void NotifySessionAbandoned();
  // Called when a query has been submitted. `query_start_time` is the time
  // that the user clicked the submit button.
  GURL CreateAimUrl(const std::string& query_text, base::Time query_start_time);

  // Observer management.
  void AddObserver(FileUploadStatusObserver* obs);
  void RemoveObserver(FileUploadStatusObserver* obs);

  // Triggers upload of the file with data and stores the file info in the
  // internal map. Call after setting the file info fields. Virtual for testing.
  virtual void StartFileUploadFlow(
      std::unique_ptr<FileInfo> file_info,
      scoped_refptr<base::RefCountedBytes> file_data,
      std::optional<composebox::ImageEncodingOptions> image_options);
  // Removes file from file cache.
  virtual bool DeleteFile(const base::UnguessableToken& file_token);

  // Clear entire file cache.
  virtual void ClearFiles();

  int num_files_in_request() { return num_files_in_request_; }

  // Return the file from `active_files_` map or nullptr if not found.
  virtual FileInfo* GetFileInfo(const base::UnguessableToken& file_token);

 protected:
  // Returns the EndpointFetcher to use with the given params. Protected to
  // allow overriding in tests to mock server responses.
  virtual std::unique_ptr<endpoint_fetcher::EndpointFetcher>
  CreateEndpointFetcher(std::string request_string,
                        const GURL& fetch_url,
                        endpoint_fetcher::HttpMethod http_method,
                        base::TimeDelta timeout,
                        const std::vector<std::string>& request_headers,
                        const std::vector<std::string>& cors_exempt_headers,
                        UploadProgressCallback upload_progress_callback);

  // Creates the client context for Lens requests. Protected to allow access
  // from tests.
  lens::LensOverlayClientContext CreateClientContext() const;

  // Clears all cluster info.
  void ClearClusterInfo();

  // Resets the request cluster info state. Protected to allow tests to
  // override. `session_id` is used to determine if the async timer is
  // from an old, invalid session.
  virtual void ResetRequestClusterInfoState(int session_id);

  // The internal state of the query controller. Protected to allow tests to
  // access the state. Do not modify this state directly, use
  // SetQueryControllerState() instead.
  QueryControllerState query_controller_state_ = QueryControllerState::kOff;

  // Callback for when the query controller state changes. Protected to allow
  // tests to set the callback.
  QueryControllerStateChangedCallback
      on_query_controller_state_changed_callback_;

  // The map of active files, keyed by the file token.
  // Protected to allow tests to access the files.
  std::map<base::UnguessableToken, std::unique_ptr<FileInfo>> active_files_;

 private:
  // Fetches the OAuth headers and calls the callback with the headers. If the
  // OAuth cannot be retrieved (like if the user is not logged in), the callback
  // will be called with an empty vector. Returns the access token fetcher
  // making the request so it can be kept alive.
  std::unique_ptr<signin::PrimaryAccountAccessTokenFetcher>
  CreateOAuthHeadersAndContinue(OAuthHeadersCreatedCallback callback);

  // Gets an OAuth token for the cluster info request and proceeds with sending
  // a LensOverlayServerClusterInfoRequest to get the cluster info.
  void FetchClusterInfo();

  // Asynchronous handler for when the fetch cluster info request headers are
  // ready. Creates the endpoint fetcher and sends the cluster info request.
  void SendClusterInfoNetworkRequest(std::vector<std::string> request_headers);

  // Handles the response from the cluster info request.
  void HandleClusterInfoResponse(
      std::unique_ptr<endpoint_fetcher::EndpointResponse> response);

  // Sets the query controller state and notifies the query controller state
  // changed callback if it has changed.
  void SetQueryControllerState(QueryControllerState new_state);

  // Updates the file upload status and notifies the file upload status
  // observers with an optional error type if the upload failed.
  void UpdateFileUploadStatus(const base::UnguessableToken& file_token,
                              FileUploadStatus status,
                              std::optional<FileUploadErrorType> error_type);

#if !BUILDFLAG(IS_IOS)
  // Handler for when the image from an image file upload is decoded. Creates
  // the request body proto and calls the callback with the request.
  void ProcessDecodedImageAndContinue(
      lens::LensOverlayRequestId request_id,
      const composebox::ImageEncodingOptions& options,
      RequestBodyProtoCreatedCallback callback,
      const SkBitmap& bitmap);
#endif  // !BUILDFLAG(IS_IOS)

  // Creates the request body proto and calls the callback with the request.
  void CreateFileUploadRequestBodyAndContinue(
      const base::UnguessableToken& file_token,
      scoped_refptr<base::RefCountedBytes> file_data,
      std::optional<composebox::ImageEncodingOptions> options,
      RequestBodyProtoCreatedCallback callback);

  // Asynchronous handler for when the file upload request body is ready.
  void OnUploadFileRequestBodyReady(
      const base::UnguessableToken& file_token,
      lens::LensOverlayServerRequest request,
      std::optional<FileUploadErrorType> error_type);

  // Asynchronous handler for when the file upload request headers are ready.
  void OnUploadFileRequestHeadersReady(const base::UnguessableToken& file_token,
                                       std::vector<std::string> headers);

  // Sends the file upload request if the request body, headers, and cluster
  // info are ready.
  void MaybeSendFileUploadNetworkRequest(
      const base::UnguessableToken& file_token);

  // Creates the endpoint fetcher and sends the file upload network request.
  void SendFileUploadNetworkRequest(FileInfo* file_infon);

  // Handles the response from the file upload request.
  void HandleFileUploadResponse(
      const base::UnguessableToken& file_token,
      std::unique_ptr<endpoint_fetcher::EndpointResponse> response);

  // The last received cluster info.
  std::optional<lens::LensOverlayClusterInfo> cluster_info_ = std::nullopt;

  // The endpoint fetcher used for the cluster info request.
  std::unique_ptr<endpoint_fetcher::EndpointFetcher>
      cluster_info_endpoint_fetcher_;

  // The access token fetcher used for getting OAuth for the cluster info
  // request. Will be discarded after the OAuth headers are created.
  std::unique_ptr<signin::PrimaryAccountAccessTokenFetcher>
      cluster_info_access_token_fetcher_;

  // Unowned IdentityManager for fetching access tokens. Could be null for
  // incognito profiles.
  const raw_ptr<signin::IdentityManager> identity_manager_;

  // The url loader factory to use for Lens network requests.
  scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory_;

  // The channel to use for Lens network requests.
  version_info::Channel channel_;

  // The locale used for creating the client context.
  std::string locale_;

  // The request id generator for this query flow instance.
  lens::LensOverlayRequestIdGenerator request_id_generator_;

  // The observer list, managed via AddObserver() and RemoveObserver().
  base::ObserverList<FileUploadStatusObserver> observers_;

  // Task runner used to create the file upload request proto asynchronously.
  scoped_refptr<base::TaskRunner> create_request_task_runner_;

  // Owned by the Profile, and thus guaranteed to outlive this instance.
  const raw_ptr<TemplateURLService> template_url_service_;

  // Owned by the Profile, and thus guaranteed to outlive this instance.
  const raw_ptr<variations::VariationsClient> variations_client_;

  // Whether or not to send the lns_surface parameter.
  // TODO(crbug.com/430070871): Remove this once the server supports the
  // `lns_surface` parameter.
  bool send_lns_surface_ = false;

  // The session counter, incremented when the session is stopped. This is used
  // to determine if the session is active when handling cluster info
  // expiration.
  int session_id_ = 0;

  // The number of files that are sent in the AIM request.
  int num_files_in_request_ = 0;

  base::WeakPtrFactory<ComposeboxQueryController> weak_ptr_factory_{this};
};

#endif  // COMPONENTS_OMNIBOX_COMPOSEBOX_COMPOSEBOX_QUERY_CONTROLLER_H_