1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444
|
// Copyright 2025 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef CHROME_BROWSER_UI_LENS_LENS_SEARCH_CONTROLLER_H_
#define CHROME_BROWSER_UI_LENS_LENS_SEARCH_CONTROLLER_H_
#include <memory>
#include "base/memory/raw_ptr.h"
#include "base/memory/weak_ptr.h"
#include "chrome/browser/lens/core/mojom/geometry.mojom.h"
#include "chrome/browser/ui/lens/lens_overlay_controller.h"
#include "chrome/browser/ui/lens/lens_overlay_query_controller.h"
#include "components/lens/lens_overlay_dismissal_source.h"
#include "components/lens/lens_overlay_invocation_source.h"
#include "components/omnibox/browser/autocomplete_match_type.h"
#include "components/tabs/public/tab_interface.h"
#include "third_party/skia/include/core/SkBitmap.h"
#include "ui/gfx/geometry/rect.h"
class LensOverlayController;
class GURL;
namespace lens {
class LensSessionMetricsLogger;
class LensOverlayEventHandler;
class LensOverlayGen204Controller;
class LensOverlaySidePanelCoordinator;
class LensPermissionBubbleController;
class LensSearchboxController;
class LensSearchContextualizationController;
} // namespace lens
namespace variations {
class VariationsClient;
} // namespace variations
namespace signin {
class IdentityManager;
} // namespace signin
namespace syncer {
class SyncService;
} // namespace syncer
class PrefService;
class ThemeService;
// Controller for all Lens Search features in Chrome. All external entry points
// should go through this controller.
// This migration is still in progress. Follow progress via crbug.com/404941800.
class LensSearchController {
public:
explicit LensSearchController(tabs::TabInterface* tab);
virtual ~LensSearchController();
// Initializes all the necessary dependencies for the LensSearchController.
void Initialize(variations::VariationsClient* variations_client,
signin::IdentityManager* identity_manager,
PrefService* pref_service,
syncer::SyncService* sync_service,
ThemeService* theme_service);
// A simple utility that gets the the LensSearchController TabFeature set by
// the embedding tab of a lens WebUI hosted in `webui_web_contents`.
// May return nullptr if no LensSearchController TabFeature is associated
// with `webui_web_contents`.
static LensSearchController* FromWebUIWebContents(
content::WebContents* webui_web_contents);
// A simple utility that gets the the LensSearchController TabFeature set by
// the instances of WebContents associated with a tab.
// May return nullptr if no LensSearchController TabFeature is associated
// with `tab_web_contents`.
static LensSearchController* FromTabWebContents(
content::WebContents* tab_web_contents);
// This is an entry point for showing the overlay UI. This has no effect if
// the overlay is not currently `kOff`. This has no effect if the tab is not
// in the foreground. If the overlay is successfully invoked, then the value
// of `invocation_source` will be recorded in the relevant metrics. Virtual
// for testing.
virtual void OpenLensOverlay(
lens::LensOverlayInvocationSource invocation_source);
// Sets a region to search after the overlay loads, then calls ShowUI().
// All units are in device pixels. region_bitmap contains the high definition
// image bytes to use for the search instead of cropping the region from the
// viewport. Virtual for testing.
virtual void OpenLensOverlayWithPendingRegion(
lens::LensOverlayInvocationSource invocation_source,
lens::mojom::CenterRotatedBoxPtr region,
const SkBitmap& region_bitmap);
// Convenience method for calling OpenLensOverlayWithPendingRegion, that will
// convert the bounds into a CenterRotated Box to pass to the overlay.
void OpenLensOverlayWithPendingRegionFromBounds(
lens::LensOverlayInvocationSource invocation_source,
const gfx::Rect& tab_bounds,
const gfx::Rect& view_bounds,
const gfx::Rect& region_bounds,
const SkBitmap& region_bitmap);
// Starts the contextualization flow without the overlay being shown to the
// user. Virtual for testing.
virtual void StartContextualization(
lens::LensOverlayInvocationSource invocation_source);
// Issues a contextual search request for Lens to fulfill. Starts
// contextualization flow if its not already in progress. If the Lens Overlay
// is in the process of opening, the request will be queued until the overlay
// is fully opened.
// TODO(crbug.com/403629222): Revisit if it makes sense to pass the
// destination URL instead of the query text directly.
void IssueContextualSearchRequest(
lens::LensOverlayInvocationSource invocation_source,
const GURL& destination_url,
AutocompleteMatchType::Type match_type,
bool is_zero_prefix_suggestion);
// Starts the closing process of the overlay. This is an asynchronous process
// with the following sequence:
// (1) Close the side panel
// (2) Close the overlay.
// Step (1) is asynchronous.
virtual void CloseLensAsync(
lens::LensOverlayDismissalSource dismissal_source);
// Instantly closes all Lens components currently opened.This may not look
// nice if the overlay is visible when this is called.
virtual void CloseLensSync(lens::LensOverlayDismissalSource dismissal_source);
// Hides the Lens overlay. This does not close the side panel. If the overlay
// is open without the side panel, this will end the Lens session.
void HideOverlay(lens::LensOverlayDismissalSource dismissal_source);
// Launches the survey if the user has not already seen it.
void MaybeLaunchSurvey();
// Returns true if Lens is currently active on this tab.
bool IsActive();
// Returns true if Lens is currently off on this tab.
bool IsOff();
// Returns true if the overlay is in the process of closing. If true, Lens on
// this tab will soon be off.
bool IsClosing();
// Returns whether the handshake with the Lens backend is complete.
bool IsHandshakeComplete();
// Returns the tab interface that owns this controller.
tabs::TabInterface* GetTabInterface();
// Returns the page URL of the tab if Lens has access to it.
const GURL& GetPageURL() const;
// Gets the page title.
std::optional<std::string> GetPageTitle();
// Returns the weak pointer to this class.
base::WeakPtr<LensSearchController> GetWeakPtr();
// Returns the LensOverlayController.
LensOverlayController* lens_overlay_controller();
const LensOverlayController* lens_overlay_controller() const;
// Returns the LensOverlayQueryController.
lens::LensOverlayQueryController* lens_overlay_query_controller();
// Returns the LensOverlaySidePanelCoordinator.
lens::LensOverlaySidePanelCoordinator* lens_overlay_side_panel_coordinator();
// Returns the LensSearchboxController.
lens::LensSearchboxController* lens_searchbox_controller();
// Returns the event handler for this instance of the Lens Overlay.
lens::LensOverlayEventHandler* lens_overlay_event_handler();
// Returns the LensSearchContextualizationController.
lens::LensSearchContextualizationController*
lens_search_contextualization_controller();
// Returns the LensSessionMetricsLogger.
lens::LensSessionMetricsLogger* lens_session_metrics_logger();
lens::LensPermissionBubbleController*
get_lens_permission_bubble_controller_for_testing() {
return lens_permission_bubble_controller_.get();
}
protected:
friend class LensOverlayController;
friend class lens::LensOverlaySidePanelCoordinator;
// Override these methods to stub out individual feature controllers for
// testing.
virtual std::unique_ptr<LensOverlayController> CreateLensOverlayController(
tabs::TabInterface* tab,
LensSearchController* lens_search_controller,
variations::VariationsClient* variations_client,
signin::IdentityManager* identity_manager,
PrefService* pref_service,
syncer::SyncService* sync_service,
ThemeService* theme_service);
// Override these methods to stub out network requests for testing.
virtual std::unique_ptr<lens::LensOverlayQueryController>
CreateLensQueryController(
lens::LensOverlayFullImageResponseCallback full_image_callback,
lens::LensOverlayUrlResponseCallback url_callback,
lens::LensOverlayInteractionResponseCallback interaction_callback,
lens::LensOverlaySuggestInputsCallback suggest_inputs_callback,
lens::LensOverlayThumbnailCreatedCallback thumbnail_created_callback,
lens::UploadProgressCallback page_content_upload_progress_callback,
variations::VariationsClient* variations_client,
signin::IdentityManager* identity_manager,
Profile* profile,
lens::LensOverlayInvocationSource invocation_source,
bool use_dark_mode,
lens::LensOverlayGen204Controller* gen204_controller);
// Override these methods to be able to track calls made to the side panel
// coordinator.
virtual std::unique_ptr<lens::LensOverlaySidePanelCoordinator>
CreateLensOverlaySidePanelCoordinator();
// Override these methods to be able to track calls made to the searchbox
// controller.
virtual std::unique_ptr<lens::LensSearchboxController>
CreateLensSearchboxController();
// Override these methods to be able to track calls made to the
// contextualization controller.
virtual std::unique_ptr<lens::LensSearchContextualizationController>
CreateLensSearchContextualizationController();
// Called by the Lens overlay when it has finished opening and has moved to
// the kOverlay state. This is how this class knows it can move into kActive
// state.
// TODO(crbug.com/404941800): Make this more generic to allow for multiple
// features to initialize at the same time.
void NotifyOverlayOpened();
// Shared logic for cleanup that is called after all features have finished
// cleaning up.
void CloseLensPart2(lens::LensOverlayDismissalSource dismissal_source);
// The final step for closing the overlay. This is called after the lens
// overlay has faded out.
void OnOverlayHidden(lens::LensOverlayDismissalSource dismissal_source);
// Called before the lens results panel begins hiding. This is called before
// any side panel closing animations begin.
void OnSidePanelWillHide(SidePanelEntryHideReason reason);
// Called when the lens side panel has been hidden.
void OnSidePanelHidden();
// Internal state machine. States are mutually exclusive. Exposed for testing.
enum class State {
// This is the default state. No feature is currently active or soon to be
// active.
kOff,
// The controller is in the process of starting up. Soon to be kActive.
kInitializing,
// One or more Lens features are active on this tab.
kActive,
// The UI has been made inactive / backgrounded and is hidden. This differs
// from kSuspended as the overlay and web view are not freed and could be
// immediately reshown.
kBackground,
// The side panel is in the process of closing. Soon will move to kClosing.
kClosingSidePanel,
// The controller is in the process of closing all dependencies and cleaning
// up. Will soon be kOff.
kClosing,
// TODO(crbug.com/335516480): Implement suspended state.
kSuspended,
};
State state() { return state_; }
private:
// Passes the correct callbacks and dependencies to the protected
// CreateLensQueryController method.
std::unique_ptr<lens::LensOverlayQueryController> CreateLensQueryController(
lens::LensOverlayInvocationSource invocation_source);
// Creates all state necessary to start a Lens session. This method contains
// shared state that is used no matter the entrypoint.
void StartLensSession(lens::LensOverlayInvocationSource invocation_source);
// Runs the eligibility checks necessary for Lens to open on this tab. If the
// user has not granted permission to use Lens on this tab, the permission
// request will be shown and callback will be called after the user accepts.
// Returns true if the checks pass and its safe to open Lens, false otherwise.
bool RunLensEligibilityChecks(
lens::LensOverlayInvocationSource invocation_source,
base::RepeatingClosure permission_granted_callback);
// Callback used by the query controller to notify the search controller of
// the response to the initial image upload request.
void HandleStartQueryResponse(
std::vector<lens::mojom::OverlayObjectPtr> objects,
lens::mojom::TextPtr text,
bool is_error);
// Callback used by the query controller to notify the search controller of
// the URL response to the interaction request, aka, the URL that should be
// opened in the results frame.
void HandleInteractionURLResponse(
lens::proto::LensOverlayUrlResponse response);
// Callback used by the query controller to notify the search controller of
// the response of an interaction request. If this is a visual interaction
// request, the response will contain the text container within that image.
void HandleInteractionResponse(lens::mojom::TextPtr text);
// Callback used by the query controller to notify the search controller of
// the suggest inputs response. This is used to update the searchbox with
// the most recent suggest inputs.
void HandleSuggestInputsResponse(
lens::proto::LensOverlaySuggestInputs suggest_inputs);
// Callback used by the query controller to pass the thumbnail bytes of a
// visual interaction request to the searchbox.
void HandleThumbnailCreated(const std::string& thumbnail_bytes,
const SkBitmap& region_bitmap);
// Callback used by the query controller to notify the search controller of
// the progress of the page content upload.
void HandlePageContentUploadProgress(uint64_t position, uint64_t total);
// Called when the associated tab enters the foreground.
void TabForegrounded(tabs::TabInterface* tab);
// Called when the associated tab will enter the background.
void TabWillEnterBackground(tabs::TabInterface* tab);
// Called when the tab's WebContents is discarded.
void WillDiscardContents(tabs::TabInterface* tab,
content::WebContents* old_contents,
content::WebContents* new_contents);
// Called when the tab will be removed from the window.
void WillDetach(tabs::TabInterface* tab,
tabs::TabInterface::DetachReason reason);
// Whether the LensSearchController has been initialized. Meaning, all the
// dependencies have been initialized and the controller is ready to use.
bool initialized_ = false;
// Tracks the internal state machine.
State state_ = State::kOff;
// Tracks the state of the Lens Search feature when the tab is backgrounded.
// This state is used to restore the Lens Search feature to the same state
// when the tab is foregrounded.
State backgrounded_state_ = State::kOff;
// Indicates whether a trigger for the HaTS survey has occurred in the current
// session. Note that a trigger does not mean the survey will actually be
// shown.
bool hats_triggered_in_session_ = false;
// If the side panel needed to be closed before dismissing Lens, this
// stores the original dismissal_source so it is properly recorded when the
// side panel is done closing and the callback is invoked.
std::optional<lens::LensOverlayDismissalSource> last_dismissal_source_;
// The query controller for the Lens Search feature on this tab. Lives for the
// duration of a Lens feature being active on this tab.
std::unique_ptr<lens::LensOverlayQueryController>
lens_overlay_query_controller_;
std::unique_ptr<lens::LensPermissionBubbleController>
lens_permission_bubble_controller_;
// The overlay controller for the Lens Search feature on this tab.
std::unique_ptr<LensOverlayController> lens_overlay_controller_;
// The controller for sending gen204 pings. Owned by this class so it can
// outlive the query controller, allowing gen204 requests to be sent upon
// query end.
std::unique_ptr<lens::LensOverlayGen204Controller> gen204_controller_;
// The side panel coordinator for the Lens Search feature on this tab.
std::unique_ptr<lens::LensOverlaySidePanelCoordinator>
lens_overlay_side_panel_coordinator_;
// The searchbox controller for the Lens Search feature on this tab.
// TODO(crbug.com/413138792): Hook up this controller to handle searchbox
// interactions, without a dependency on the overlay controller.
std::unique_ptr<lens::LensSearchboxController> lens_searchbox_controller_;
// The contextualization controller for the Lens Search feature on this tab.
std::unique_ptr<lens::LensSearchContextualizationController>
lens_contextualization_controller_;
std::unique_ptr<lens::LensSessionMetricsLogger> lens_session_metrics_logger_;
// Class for handling key events from the renderer that were not handled. This
// is used by both the overlay and the WebUI to share common event handling
// logic.
std::unique_ptr<lens::LensOverlayEventHandler> lens_overlay_event_handler_;
// Holds subscriptions for TabInterface callbacks.
std::vector<base::CallbackListSubscription> tab_subscriptions_;
// Owned by Profile, and thus guaranteed to outlive this instance.
raw_ptr<variations::VariationsClient> variations_client_;
// Unowned IdentityManager for fetching access tokens. Could be null for
// incognito profiles.
raw_ptr<signin::IdentityManager> identity_manager_;
// The pref service associated with the current profile. Owned by Profile,
// and thus guaranteed to outlive this instance.
raw_ptr<PrefService> pref_service_;
// The sync service associated with the current profile.
raw_ptr<syncer::SyncService> sync_service_;
// The theme service associated with the current profile. Owned by Profile,
// and thus guaranteed to outlive this instance.
raw_ptr<ThemeService> theme_service_;
// Owns this class.
raw_ptr<tabs::TabInterface> tab_;
// Must be the last member.
base::WeakPtrFactory<LensSearchController> weak_ptr_factory_{this};
};
#endif // CHROME_BROWSER_UI_LENS_LENS_SEARCH_CONTROLLER_H_
|