1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
|
# frozen_string_literal: true
module Projects
class UpdateRemoteMirrorService < BaseService
include Gitlab::Utils::StrongMemoize
MAX_TRIES = 3
def execute(remote_mirror, tries)
return success unless remote_mirror.enabled?
# Blocked URLs are a hard failure, no need to attempt to retry
if Gitlab::HTTP_V2::UrlBlocker.blocked_url?(
normalized_url(remote_mirror.url),
schemes: Project::VALID_MIRROR_PROTOCOLS,
allow_localhost: Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services?,
allow_local_network: Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services?,
deny_all_requests_except_allowed: Gitlab::CurrentSettings.deny_all_requests_except_allowed?,
outbound_local_requests_allowlist: Gitlab::CurrentSettings.outbound_local_requests_whitelist # rubocop:disable Naming/InclusiveLanguage -- existing setting
)
hard_retry_or_fail(remote_mirror, _('The remote mirror URL is invalid.'), tries)
return error(remote_mirror.last_error)
end
update_mirror(remote_mirror)
success
rescue Gitlab::Git::CommandError => e
# This happens if one of the gitaly calls above fail, for example when
# branches have diverged, or the pre-receive hook fails.
hard_retry_or_fail(remote_mirror, e.message, tries)
error(e.message)
rescue StandardError => e
remote_mirror.hard_fail!(e.message)
raise e
end
private
def normalized_url(url)
strong_memoize_with(:normalized_url, url) do
CGI.unescape(Gitlab::UrlSanitizer.sanitize(url))
end
end
def update_mirror(remote_mirror)
remote_mirror.update_start!
# LFS objects must be sent first, or the push has dangling pointers
lfs_status = send_lfs_objects!(remote_mirror)
response = remote_mirror.update_repository
failed, failure_message = failure_status(lfs_status, response, remote_mirror)
# When the issue https://gitlab.com/gitlab-org/gitlab/-/issues/349262 is closed,
# we can move this block within failure_status.
if failed
remote_mirror.mark_as_failed!(failure_message)
else
remote_mirror.update_finish!
end
end
def failure_status(lfs_status, response, remote_mirror)
message = ''
failed = false
lfs_sync_failed = false
if lfs_status&.dig(:status) == :error
lfs_sync_failed = true
message += "Error synchronizing LFS files:"
message += "\n\n#{lfs_status[:message]}\n\n"
failed = Feature.enabled?(:remote_mirror_fail_on_lfs, project)
end
if response.divergent_refs.any?
message += "Some refs have diverged and have not been updated on the remote:"
message += "\n\n#{response.divergent_refs.join("\n")}"
failed = true
end
if message.present?
Gitlab::AppJsonLogger.info(
message: "Error synching remote mirror",
project_id: project.id,
project_path: project.full_path,
remote_mirror_id: remote_mirror.id,
lfs_sync_failed: lfs_sync_failed,
divergent_ref_list: response.divergent_refs
)
end
[failed, message]
end
def send_lfs_objects!(remote_mirror)
return unless project.lfs_enabled?
# TODO: Support LFS sync over SSH
# https://gitlab.com/gitlab-org/gitlab/-/issues/249587
return unless %r{\Ahttps?://}i.match?(remote_mirror.url)
return unless remote_mirror.password_auth?
Lfs::PushService.new(
project,
current_user,
url: remote_mirror.bare_url,
credentials: remote_mirror.credentials
).execute
end
def hard_retry_or_fail(mirror, message, tries)
if tries < MAX_TRIES
mirror.hard_retry!(message)
else
# It's not likely we'll be able to recover from this ourselves, so we'll
# notify the users of the problem, and don't trigger any sidekiq retries
# Instead, we'll wait for the next change to try the push again, or until
# a user manually retries.
mirror.hard_fail!(message)
end
end
end
end
|
