1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
|
# frozen_string_literal: true
class Upload < ApplicationRecord
include Checksummable
include EachBatch
# Upper limit for foreground checksum processing
CHECKSUM_THRESHOLD = 100.megabytes
belongs_to :model, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations
belongs_to :uploaded_by_user, class_name: 'User', optional: true
validates :size, presence: true
validates :path, presence: true
validates :model, presence: true
validates :uploader, presence: true
scope :with_files_stored_locally, -> { where(store: ObjectStorage::Store::LOCAL) }
scope :with_files_stored_remotely, -> { where(store: ObjectStorage::Store::REMOTE) }
scope :for_model_type_and_id, ->(type, id) { where(model_type: type, model_id: id) }
scope :for_uploader, ->(uploader_class) { where(uploader: uploader_class.to_s) }
scope :order_by_created_at_desc, -> { reorder(created_at: :desc) }
scope :preload_uploaded_by_user, -> { preload(:uploaded_by_user) }
before_save :calculate_checksum!, if: :foreground_checksummable?
# as the FileUploader is not mounted, the default CarrierWave ActiveRecord
# hooks are not executed and the file will not be deleted
after_destroy :delete_file!, if: -> { uploader_class <= FileUploader }
after_commit :schedule_checksum, if: :needs_checksum?
after_commit :update_project_statistics, on: [:create, :destroy], if: :project?
class << self
def inner_join_local_uploads_projects
upload_table = Upload.arel_table
project_table = Project.arel_table
join_statement = upload_table.project(upload_table[Arel.star])
.join(project_table)
.on(
upload_table[:model_type].eq('Project')
.and(upload_table[:model_id].eq(project_table[:id]))
.and(upload_table[:store].eq(ObjectStorage::Store::LOCAL))
)
joins(join_statement.join_sources)
end
##
# FastDestroyAll concerns
def begin_fast_destroy
{
Uploads::Local => Uploads::Local.new.keys(with_files_stored_locally),
Uploads::Fog => Uploads::Fog.new.keys(with_files_stored_remotely)
}
end
##
# FastDestroyAll concerns
def finalize_fast_destroy(items_to_remove)
items_to_remove.each do |store_class, keys|
store_class.new.delete_keys_async(keys)
end
end
end
def absolute_path
raise ObjectStorage::RemoteStoreError, _("Remote object has no absolute path.") unless local?
return path unless relative_path?
uploader_class.absolute_path(self)
end
def relative_path
uploader_class.relative_path(self)
end
def calculate_checksum!
self.checksum = nil
return unless needs_checksum?
self.checksum = self.class.sha256_hexdigest(absolute_path)
end
# Initialize the associated Uploader class with current model
#
# @param [String] mounted_as
# @return [GitlabUploader] one of the subclasses, defined at the model's uploader attribute
def build_uploader(mounted_as = nil)
uploader_class.new(model, mounted_as || mount_point).tap do |uploader|
uploader.upload = self
end
end
# Initialize the associated Uploader class with current model and
# retrieve existing file from the store to a local cache
#
# @param [String] mounted_as
# @return [GitlabUploader] one of the subclasses, defined at the model's uploader attribute
def retrieve_uploader(mounted_as = nil)
build_uploader(mounted_as).tap do |uploader|
uploader.retrieve_from_store!(filename)
end
end
# This checks for existence of the upload on storage
#
# @return [Boolean] whether upload exists on storage
def exist?
exist = if local?
File.exist?(absolute_path)
else
retrieve_uploader.exists?
end
# Help sysadmins find missing upload files
if persisted? && !exist
exception = RuntimeError.new("Uploaded file does not exist")
Gitlab::ErrorTracking.track_exception(exception, self.attributes)
Gitlab::Metrics.counter(:upload_file_does_not_exist_total, _('The number of times an upload record could not find its file')).increment
end
exist
end
def uploader_context
{
identifier: filename,
secret: secret,
uploaded_by_user_id: uploaded_by_user_id
}.compact
end
def local?
store == ObjectStorage::Store::LOCAL
end
# Returns whether generating checksum is needed
#
# This takes into account whether file exists, if any checksum exists
# or if the storage has checksum generation code implemented
#
# @return [Boolean] whether generating a checksum is needed
def needs_checksum?
checksum.nil? && local? && exist?
end
def filename
File.basename(path)
end
private
def delete_file!
retrieve_uploader.remove!
end
def foreground_checksummable?
needs_checksum? && size <= CHECKSUM_THRESHOLD
end
def schedule_checksum
UploadChecksumWorker.perform_async(id)
end
def relative_path?
!path.start_with?('/')
end
def uploader_class
Object.const_get(uploader, false)
end
def mount_point
super&.to_sym
end
def project?
model_type == "Project"
end
def update_project_statistics
ProjectCacheWorker.perform_async(model_id, [], ['uploads_size'])
end
end
Upload.prepend_mod_with('Upload')
|
