File: container_repository.rb

package info (click to toggle)
gitlab 17.6.5-19
  • links: PTS, VCS
  • area: main
  • in suites: sid
  • size: 629,368 kB
  • sloc: ruby: 1,915,304; javascript: 557,307; sql: 60,639; xml: 6,509; sh: 4,567; makefile: 1,239; python: 406
file content (345 lines) | stat: -rw-r--r-- 10,870 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
# frozen_string_literal: true

class ContainerRepository < ApplicationRecord
  include Gitlab::Utils::StrongMemoize
  include Gitlab::SQL::Pattern
  include EachBatch
  include Sortable
  include AfterCommitQueue
  include Packages::Destructible

  WAITING_CLEANUP_STATUSES = %i[cleanup_scheduled cleanup_unfinished].freeze
  REQUIRING_CLEANUP_STATUSES = %i[cleanup_unscheduled cleanup_scheduled].freeze

  MAX_TAGS_PAGES = 2000
  MAX_DELETION_FAILURES = 10

  # The Registry client uses JWT token to authenticate to Registry. We cache the client using expiration
  # time of JWT token. However it's possible that the token is valid but by the time the request is made to
  # Regsitry, it's already expired. To prevent this case, we are subtracting a few seconds, defined by this constant
  # from the cache expiration time.
  AUTH_TOKEN_USAGE_RESERVED_TIME_IN_SECS = 5

  belongs_to :project

  validates :name, length: { minimum: 0, allow_nil: false }
  validates :name, uniqueness: { scope: :project_id }
  validates :failed_deletion_count, presence: true
  validates :failed_deletion_count, numericality: { greater_than_or_equal_to: 0, less_than_or_equal_to: MAX_DELETION_FAILURES }

  enum status: { delete_scheduled: 0, delete_failed: 1, delete_ongoing: 2 }
  enum expiration_policy_cleanup_status: { cleanup_unscheduled: 0, cleanup_scheduled: 1, cleanup_unfinished: 2, cleanup_ongoing: 3 }

  delegate :client, :gitlab_api_client, to: :registry

  scope :ordered, -> { order(:name) }
  scope :with_api_entity_associations, -> { preload(project: [:route, { namespace: :route }]) }
  scope :for_group_and_its_subgroups, ->(group) do
    project_scope = Project
      .for_group_and_its_subgroups(group)
      .with_feature_enabled(:container_registry)
      .select(:id)

    joins("INNER JOIN (#{project_scope.to_sql}) projects on projects.id=container_repositories.project_id")
  end
  scope :for_project_id, ->(project_id) { where(project_id: project_id) }
  scope :search_by_name, ->(query) { fuzzy_search(query, [:name], use_minimum_char_limit: false) }
  scope :waiting_for_cleanup, -> { where(expiration_policy_cleanup_status: WAITING_CLEANUP_STATUSES) }
  scope :expiration_policy_started_at_nil_or_before, ->(timestamp) { where('expiration_policy_started_at < ? OR expiration_policy_started_at IS NULL', timestamp) }
  scope :with_stale_ongoing_cleanup, ->(threshold) { cleanup_ongoing.expiration_policy_started_at_nil_or_before(threshold) }
  scope :with_stale_delete_at, ->(threshold) { where('delete_started_at < ?', threshold) }

  before_update :set_status_updated_at_to_now, if: :status_changed?

  # Container Repository model and the code that makes API calls
  # are tied. Sometimes (mainly in Geo) we need to work with Registry
  # when Container Repository record doesn't even exist.
  # The ability to create a not-persisted record with a certain "path" parameter
  # is very useful
  attr_writer :path

  def self.exists_by_path?(path)
    where(
      project: path.repository_project,
      name: path.repository_name
    ).exists?
  end

  def self.with_enabled_policy
    joins('INNER JOIN container_expiration_policies ON container_repositories.project_id = container_expiration_policies.project_id')
      .where(container_expiration_policies: { enabled: true })
  end

  def self.requiring_cleanup
    with_enabled_policy
      .where(container_repositories: { expiration_policy_cleanup_status: REQUIRING_CLEANUP_STATUSES })
      .where('container_repositories.expiration_policy_started_at IS NULL OR container_repositories.expiration_policy_started_at < container_expiration_policies.next_run_at')
      .where('container_expiration_policies.next_run_at < ?', Time.zone.now)
  end

  def self.with_unfinished_cleanup
    with_enabled_policy.cleanup_unfinished
  end

  def self.registry_client_expiration_time
    (Gitlab::CurrentSettings.container_registry_token_expire_delay * 60) - AUTH_TOKEN_USAGE_RESERVED_TIME_IN_SECS
  end

  # needed by Packages::Destructible
  def self.pending_destruction
    delete_scheduled.where('next_delete_attempt_at IS NULL OR next_delete_attempt_at < ?', Time.zone.now)
  end

  # rubocop: disable CodeReuse/ServiceClass
  def registry
    strong_memoize_with_expiration(:registry, self.class.registry_client_expiration_time) do
      token = Auth::ContainerRegistryAuthenticationService.full_access_token(path)

      url = Gitlab.config.registry.api_url
      host_port = Gitlab.config.registry.host_port

      ContainerRegistry::Registry.new(url, token: token, path: host_port)
    end
  end
  # rubocop: enable CodeReuse/ServiceClass

  def path
    @path ||= [project.full_path, name]
      .select(&:present?).join('/').downcase
  end

  def location
    File.join(registry.path, path)
  end

  # If the container registry GitLab API is available, the API
  # does a search of tags containing the name and we filter them
  # to find the exact match. Otherwise, we instantiate a tag.
  def tag(tag)
    if gitlab_api_client.supports_gitlab_api?
      page = tags_page(name: tag)
      return if page[:tags].blank?

      page[:tags].find { |result_tag| result_tag.name == tag }
    else
      ContainerRegistry::Tag.new(self, tag)
    end
  end

  def image_manifest(reference)
    client.repository_manifest(path, reference)
  end

  def manifest
    @manifest ||= client.repository_tags(path)
  end

  def tags
    strong_memoize(:tags) do
      if gitlab_api_client.supports_gitlab_api?
        result = []
        each_tags_page do |array_of_tags|
          result << array_of_tags
        end

        result.flatten
      else
        next [] unless manifest && manifest['tags']

        manifest['tags'].sort.map do |tag|
          ContainerRegistry::Tag.new(self, tag)
        end
      end
    end
  end

  def each_tags_page(page_size: 100, &block)
    raise ArgumentError, _('GitLab container registry API not supported') unless gitlab_api_client.supports_gitlab_api?
    raise ArgumentError, 'block not given' unless block

    # dummy uri to initialize the loop
    next_page_uri = URI('')
    page_count = 0

    while next_page_uri && page_count < MAX_TAGS_PAGES
      last = Rack::Utils.parse_nested_query(next_page_uri.query)['last']
      current_page = gitlab_api_client.tags(self.path, page_size: page_size, last: last)

      if current_page&.key?(:response_body)
        yield transform_tags_page(current_page[:response_body])
        next_page_uri = current_page.dig(:pagination, :next, :uri)
      else
        # no current page. Break the loop
        next_page_uri = nil
      end

      page_count += 1
    end

    raise 'too many pages requested' if page_count >= MAX_TAGS_PAGES
  end

  def tags_page(before: nil, last: nil, sort: nil, name: nil, page_size: 100, referrers: nil, referrer_type: nil)
    raise ArgumentError, _('GitLab container registry API not supported') unless gitlab_api_client.supports_gitlab_api?

    page = gitlab_api_client.tags(
      self.path,
      page_size: page_size,
      before: before,
      last: last,
      sort: sort,
      name: name,
      referrers: referrers,
      referrer_type: referrer_type
    )

    {
      tags: transform_tags_page(page[:response_body]),
      pagination: page[:pagination]
    }
  end

  def tags_count
    return 0 unless manifest && manifest['tags']

    manifest['tags'].size
  end

  def blob(config)
    ContainerRegistry::Blob.new(self, config)
  end

  def has_tags?
    tags.any?
  end

  def root_repository?
    name.empty?
  end

  def delete_tags!
    return unless has_tags?

    digests = tags.map { |tag| tag.digest }.compact.to_set

    digests.map { |digest| delete_tag(digest) }.all?
  end

  def delete_tag(name_or_digest)
    client.delete_repository_tag_by_digest(self.path, name_or_digest)
  end

  def start_expiration_policy!
    update!(
      expiration_policy_started_at: Time.zone.now,
      last_cleanup_deleted_tags_count: nil,
      expiration_policy_cleanup_status: :cleanup_ongoing
    )
  end

  def size
    strong_memoize(:size) do
      next unless gitlab_api_client.supports_gitlab_api?

      gitlab_api_client_repository_details['size_bytes']
    end
  end

  def last_published_at
    return unless gitlab_api_client.supports_gitlab_api?

    timestamp_string = gitlab_api_client_repository_details['last_published_at']
    DateTime.iso8601(timestamp_string)
  rescue ArgumentError
    nil
  end
  strong_memoize_attr :last_published_at

  def set_delete_ongoing_status
    now = Time.zone.now

    update_columns(
      status: :delete_ongoing,
      delete_started_at: now,
      status_updated_at: now,
      next_delete_attempt_at: nil
    )
  end

  def set_delete_scheduled_status
    update_columns(
      status: :delete_scheduled,
      delete_started_at: nil,
      status_updated_at: Time.zone.now,
      failed_deletion_count: failed_deletion_count + 1,
      next_delete_attempt_at: next_delete_attempt_with_delay
    )
  end

  def set_delete_failed_status
    update_columns(
      status: :delete_failed,
      delete_started_at: nil,
      status_updated_at: Time.zone.now
    )
  end

  def self.build_from_path(path)
    self.new(project: path.repository_project, name: path.repository_name)
  end

  def self.find_or_create_from_path!(path)
    ContainerRepository.upsert({
      project_id: path.repository_project.id,
      name: path.repository_name
    }, unique_by: %i[project_id name])

    find_by_path!(path)
  end

  def self.build_root_repository(project)
    self.new(project: project, name: '')
  end

  def self.find_by_path!(path)
    self.find_by!(project: path.repository_project, name: path.repository_name)
  end

  def self.find_by_path(path)
    self.find_by(project: path.repository_project, name: path.repository_name)
  end

  private

  def transform_tags_page(tags_response_body)
    return [] unless tags_response_body

    tags_response_body.map do |raw_tag|
      tag = ContainerRegistry::Tag.new(self, raw_tag['name'], from_api: true)
      tag.force_created_at_from_iso8601(raw_tag['created_at'])
      tag.updated_at = raw_tag['updated_at']
      tag.total_size = raw_tag['size_bytes']
      tag.manifest_digest = raw_tag['digest']
      tag.revision = raw_tag['config_digest'].to_s.split(':')[1] || ''
      tag.referrers = raw_tag['referrers']
      tag.published_at = raw_tag['published_at']
      tag.media_type = raw_tag['media_type']
      tag
    end
  end

  def set_status_updated_at_to_now
    self.status_updated_at = Time.zone.now
  end

  def gitlab_api_client_repository_details
    gitlab_api_client.repository_details(self.path, sizing: :self)
  end
  strong_memoize_attr :gitlab_api_client_repository_details

  def next_delete_attempt_with_delay(now = Time.zone.now)
    now + (2**failed_deletion_count).minutes
  end
end

ContainerRepository.prepend_mod_with('ContainerRepository')