1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263
|
// Copyright 2017 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef CHROME_BROWSER_ASH_POWER_ML_USER_ACTIVITY_MANAGER_H_
#define CHROME_BROWSER_ASH_POWER_ML_USER_ACTIVITY_MANAGER_H_
#include <optional>
#include "base/cancelable_callback.h"
#include "base/memory/raw_ptr.h"
#include "base/scoped_observation.h"
#include "base/task/sequenced_task_runner.h"
#include "base/time/time.h"
#include "chrome/browser/ash/power/ml/boot_clock.h"
#include "chrome/browser/ash/power/ml/idle_event_notifier.h"
#include "chrome/browser/ash/power/ml/smart_dim/ml_agent.h"
#include "chrome/browser/ash/power/ml/user_activity_event.pb.h"
#include "chrome/browser/ash/power/ml/user_activity_ukm_logger.h"
#include "chromeos/dbus/power/power_manager_client.h"
#include "chromeos/dbus/power_manager/idle.pb.h"
#include "chromeos/dbus/power_manager/policy.pb.h"
#include "chromeos/dbus/power_manager/suspend.pb.h"
#include "components/session_manager/core/session_manager.h"
#include "components/session_manager/core/session_manager_observer.h"
#include "mojo/public/cpp/bindings/pending_receiver.h"
#include "mojo/public/cpp/bindings/receiver.h"
#include "mojo/public/cpp/bindings/remote_set.h"
#include "services/metrics/public/cpp/ukm_source_id.h"
#include "services/viz/public/mojom/compositing/video_detector_observer.mojom-forward.h"
#include "ui/aura/window.h"
#include "ui/base/user_activity/user_activity_detector.h"
#include "ui/base/user_activity/user_activity_observer.h"
namespace ash {
namespace power {
namespace ml {
// These values are persisted to logs. Entries should not be renumbered and
// numeric values should never be reused.
// The values below are not mutually exclusive. kError is any error which could
// be any of the other kErrors.
enum class PreviousEventLoggingResult {
kSuccess = 0,
kError = 1,
kErrorModelPredictionMissing = 2,
kErrorModelDisabled = 3,
kErrorMultiplePreviousEvents = 4,
kErrorIdleStartMissing = 5,
kMaxValue = kErrorIdleStartMissing
};
struct TabProperty {
ukm::SourceId source_id = -1;
std::string domain;
// Tab URL's engagement score. -1 if engagement service is disabled.
int engagement_score = -1;
// Whether user has form entry, i.e. text input.
bool has_form_entry = false;
};
// These values are persisted to logs. Entries should not be renumbered and
// numeric values should never be reused.
// What happens after a screen dim imminent is received.
enum class DimImminentAction {
kModelIgnored = 0,
kModelDim = 1,
kModelNoDim = 2,
kMaxValue = kModelNoDim
};
// These values are persisted to logs. Entries should not be renumbered and
// numeric values should never be reused.
enum class FinalResult { kReactivation = 0, kOff = 1, kMaxValue = kOff };
// Logs user activity after an idle event is observed.
// TODO(renjieliu): Add power-related activity as well.
class UserActivityManager : public ui::UserActivityObserver,
public chromeos::PowerManagerClient::Observer,
public viz::mojom::VideoDetectorObserver,
public session_manager::SessionManagerObserver {
public:
UserActivityManager(
UserActivityUkmLogger* ukm_logger,
ui::UserActivityDetector* detector,
chromeos::PowerManagerClient* power_manager_client,
session_manager::SessionManager* session_manager,
mojo::PendingReceiver<viz::mojom::VideoDetectorObserver> receiver);
UserActivityManager(const UserActivityManager&) = delete;
UserActivityManager& operator=(const UserActivityManager&) = delete;
~UserActivityManager() override;
// ui::UserActivityObserver overrides.
void OnUserActivity(const ui::Event* event) override;
// chromeos::PowerManagerClient::Observer overrides:
void LidEventReceived(chromeos::PowerManagerClient::LidState state,
base::TimeTicks timestamp) override;
void PowerChanged(const power_manager::PowerSupplyProperties& proto) override;
void TabletModeEventReceived(chromeos::PowerManagerClient::TabletMode mode,
base::TimeTicks timestamp) override;
void ScreenIdleStateChanged(
const power_manager::ScreenIdleState& proto) override;
void SuspendImminent(power_manager::SuspendImminent::Reason reason) override;
void InactivityDelaysChanged(
const power_manager::PowerManagementPolicy::Delays& delays) override;
// viz::mojom::VideoDetectorObserver overrides:
void OnVideoActivityStarted() override;
void OnVideoActivityEnded() override {}
// Called in UserActivityController::ShouldDeferScreenDim to make smart dim
// decision and response via |callback|.
void UpdateAndGetSmartDimDecision(const IdleEventNotifier::ActivityData& data,
base::OnceCallback<void(bool)> callback);
// Converts a Smart Dim model |prediction| into a yes/no decision about
// whether to defer the screen dim and provides the result via |callback|.
// |prediction| should be empty if Smart Dim made no decision (e.g. call was
// canceled). In this case, |callback| is invoked with default decision of
// |false| (i.e. allow screen dim).
void HandleSmartDimDecision(
base::OnceCallback<void(bool)> callback,
std::optional<UserActivityEvent::ModelPrediction> prediction);
// session_manager::SessionManagerObserver overrides:
void OnSessionStateChanged() override;
private:
friend class UserActivityManagerTest;
// Data structure associated with the 1st ScreenDimImminent event. See
// PopulatePreviousEventData function below.
struct PreviousIdleEventData;
// Updates lid state and tablet mode from received switch states.
void OnReceiveSwitchStates(
std::optional<chromeos::PowerManagerClient::SwitchStates> switch_states);
void OnReceiveInactivityDelays(
std::optional<power_manager::PowerManagementPolicy::Delays> delays);
// Gets properties of active tab from visible focused/topmost browser.
TabProperty UpdateOpenTabURL();
// Extracts features from last known activity data, device states and topmost
// browser window.
void ExtractFeatures(const IdleEventNotifier::ActivityData& activity_data);
// Log event only when an idle event is observed.
void MaybeLogEvent(UserActivityEvent::Event::Type type,
UserActivityEvent::Event::Reason reason);
// We could have two consecutive idle events (i.e. two ScreenDimImminent)
// without a final event logged in between. This could happen when the 1st
// screen dim is deferred and after another idle period, powerd decides to
// dim the screen again. We want to log both events. Hence we record the
// event data associated with the 1st ScreenDimImminent using the method
// below.
void PopulatePreviousEventData(const base::TimeDelta& now);
void ResetAfterLogging();
// Cancel any pending request to `SmartDimMlAgent` to get a dim decision.
void CancelDimDecisionRequest();
// Time when an idle event is received and we start logging. Null if an idle
// event hasn't been observed.
std::optional<base::TimeDelta> idle_event_start_since_boot_;
chromeos::PowerManagerClient::LidState lid_state_ =
chromeos::PowerManagerClient::LidState::NOT_PRESENT;
chromeos::PowerManagerClient::TabletMode tablet_mode_ =
chromeos::PowerManagerClient::TabletMode::UNSUPPORTED;
UserActivityEvent::Features::DeviceType device_type_ =
UserActivityEvent::Features::UNKNOWN_DEVICE;
std::optional<power_manager::PowerSupplyProperties::ExternalPower>
external_power_;
// Battery percent. This is in the range [0.0, 100.0].
std::optional<float> battery_percent_;
// Indicates whether the screen is locked.
bool screen_is_locked_ = false;
// Features extracted when receives an idle event.
UserActivityEvent::Features features_;
BootClock boot_clock_;
const raw_ptr<UserActivityUkmLogger> ukm_logger_;
base::ScopedObservation<ui::UserActivityDetector, ui::UserActivityObserver>
user_activity_observation_{this};
base::ScopedObservation<chromeos::PowerManagerClient,
chromeos::PowerManagerClient::Observer>
power_manager_client_observation_{this};
base::ScopedObservation<session_manager::SessionManager,
session_manager::SessionManagerObserver>
session_manager_observation_{this};
const raw_ptr<session_manager::SessionManager> session_manager_;
mojo::Receiver<viz::mojom::VideoDetectorObserver> receiver_;
const raw_ptr<chromeos::PowerManagerClient> power_manager_client_;
// Delays to dim and turn off the screen. Zero means disabled.
base::TimeDelta screen_dim_delay_;
base::TimeDelta screen_off_delay_;
// Whether screen is currently dimmed/off.
bool screen_dimmed_ = false;
bool screen_off_ = false;
// Whether screen dim/off occurred before final event was logged. They are
// reset to false at the start of each idle event.
bool screen_dim_occurred_ = false;
bool screen_off_occurred_ = false;
bool screen_lock_occurred_ = false;
// Number of positive/negative actions up to but excluding the current event.
// REACTIVATE is a negative action, all other event types (OFF, TIMEOUT) are
// positive actions.
int previous_negative_actions_count_ = 0;
int previous_positive_actions_count_ = 0;
// Whether screen-dim was deferred by the model when the previous
// ScreenDimImminent event arrived.
bool dim_deferred_ = false;
// Whether we are waiting for the final action after an idle event. It's only
// set to true after we've received an idle event, but haven't received final
// action to log the event.
bool waiting_for_final_action_ = false;
// Whether we are waiting for a decision from the `SmartDimMlAgent`
// regarding whether to proceed with a dim or not. It is only set
// to true in OnIdleEventObserved() when we request a dim decision.
bool waiting_for_model_decision_ = false;
// Represents the time when a dim decision request was made. It is used to
// calculate time deltas while logging ML service dim decision request
// results.
base::TimeTicks time_dim_decision_requested_;
// Model prediction for the current ScreenDimImminent event. Unset if
// model prediction is disabled by an experiment.
std::optional<UserActivityEvent::ModelPrediction> model_prediction_;
std::unique_ptr<PreviousIdleEventData> previous_idle_event_data_;
SEQUENCE_CHECKER(sequence_checker_);
base::WeakPtrFactory<UserActivityManager> weak_ptr_factory_{this};
};
} // namespace ml
} // namespace power
} // namespace ash
#endif // CHROME_BROWSER_ASH_POWER_ML_USER_ACTIVITY_MANAGER_H_
|