1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
|
# -*- coding: utf-8 -*-
require 'moneta'
Plugin.create :image_file_cache do
UserConfig[:image_file_cache_expire] ||= 32
UserConfig[:image_file_cache_size_limit] ||= 128
@queue = Delayer.generate_class(priority: %i[none check_subdirs check_dirs],
default: :none,
expire: 0.02)
@cache_directory = File.join(Environment::CACHE, 'image_file_cache').freeze
@cache_journal_directory = File.join(Environment::CACHE, 'image_file_cache', 'journal').freeze
@db = ::Moneta.build(&->(dir){ ->(this){
this.use :Transformer, key: %i[md5 spread]
this.adapter(:File, dir: dir)
} }.(@cache_directory))
@journal_db = ::Moneta.build(&->(dir){ ->(this){
this.use :Transformer, key: %i[md5 spread], value: :marshal
this.adapter(:File, dir: dir)
} }.(@cache_journal_directory))
@urls = nil
on_image_file_cache_cache do |url|
photos = Enumerator.new{|y|
Plugin.filtering(:photo_filter, url, y)
}
Plugin.call(:image_file_cache_photo, photos.first)
end
on_image_file_cache_photo do |photo|
cache_it(photo)
end
# キャッシュがあれば画像を返す
filter_image_cache do |url, image, &stop|
body = @db[url]
if body
@journal_db.increment("#{url}:read_count")
stop.call([url, body])
end
[url, image]
rescue => e
error e
[url, image]
end
settings _('画像キャッシュ') do
adjustment(_('画像をキャッシュする最大日数(日)'), :image_file_cache_expire, 1, 365)
.tooltip(_('キャッシュされた画像は、この日数が経てば、無条件に削除されます。もしもう一度使われることがあれば、ネット上から再取得されるでしょう。'))
adjustment(_('画像キャッシュを保存する最大容量(MB)'), :image_file_cache_size_limit, 1, 1024)
.tooltip(_('この容量に達するまで、アイコンなどの、また使われる可能性のある画像をローカルに保存しておきます。例えば1024を指定すると、最大で1GBのストレージを消費する可能性があります。'))
end
# キャッシュの有効期限を秒単位で返す
def cache_expire
[1, UserConfig[:image_file_cache_expire].to_i].max * 24 * 60 * 60
end
# キャッシュの容量制限を返す(Bytes)
def cache_size_limit
[1, UserConfig[:image_file_cache_size_limit].to_i].max << 20
end
# 容量オーバーの時、一度に開放する画像の最小点数
def size_exceeded_minimum_photo_count_atonce
128
end
def cache_it(photo)
unless urls.include?(photo.uri.to_s)
if photo.blob
cache_blob(photo.uri.to_s, photo.blob)
else
photo.download.next{|downloaded|
cache_blob(photo.uri.to_s, downloaded.blob)
}
end
end
end
def cache_blob(uri, blob)
return if blob.bytesize >= cache_size_limit
SerialThread.new do
unless urls.include?(uri)
all_size = @journal_db.increment("all:size", blob.bytesize)
if all_size >= cache_size_limit
free_unused_cache
end
urls << uri
@db[uri.to_s] = blob
@journal_db["#{uri}:created"] = Time.now
@journal_db["#{uri}:size"] = blob.bytesize
@journal_db["all:urls"] = urls
notice "image file cache added #{uri} (#{blob.bytesize}B, all_size: #{all_size}B)"
end
end
end
def urls
@urls ||= Set.new(@journal_db.fetch("all:urls", []))
end
def free_unused_cache
before_size = @journal_db.raw.fetch("all:size", 0).to_i
notice "there are exists #{@urls.size} cache(s). it will delete #{[@urls.size/10, size_exceeded_minimum_photo_count_atonce].max.to_i} cache(s)."
target_urls = @urls.to_a.sample([@urls.size/10, size_exceeded_minimum_photo_count_atonce].max.to_i)
target_bytesize_sum = 0
params = target_urls.map{|uri|
count = @journal_db.raw.fetch("#{uri}:read_count", 0).to_i
target_bytesize_sum += count
{ uri: uri,
size: @journal_db.fetch("#{uri}:size"){ @db.fetch(uri, ''.freeze).bytesize },
count: count }
}
target_bytesize_average = target_bytesize_sum.to_f / params.size
delete_items = params.sort_by{|param| (param[:count] - target_bytesize_average) * param[:size] }.first(params.size/2)
deleted_size = 0
delete_items.each do |item|
uri = item[:uri]
notice "delete ((#{item[:count]} - #{target_bytesize_average}) * #{item[:size]} = #{(item[:count] - target_bytesize_average) * item[:size]}pts) #{uri}"
urls.delete(uri)
@db.delete(uri.to_s)
@journal_db.delete("#{uri}:created")
@journal_db.delete("#{uri}:size")
@journal_db.raw.delete("#{uri}:read_count")
deleted_size += item[:size]
end
@journal_db.decrement("all:size", deleted_size)
@journal_db["all:urls"] = urls
after_size = @journal_db.raw["all:size"].to_i
notice "image file cache free. #{before_size} -> #{after_size} (#{before_size - after_size}B free)"
activity :system, "image file cache free. #{before_size} -> #{after_size} (#{before_size - after_size}B free)"
end
def check_subdirs(dir)
@queue.new(:check_subdirs) do
Dir.foreach(dir)
.map{|x| File.join(dir, x) }
.select{|x| FileTest.file?(x) }
.each{|x|
Delayer.new(:destroy_cache, delay: (File.atime(x) rescue File.mtime(x)) + cache_expire) do
notice "cache deleted #{x}"
File.delete(x) if FileTest.file?(x)
if Dir.foreach(dir).select{|y| File.file? File.join(dir, y) }.empty?
Dir.delete(dir) rescue nil end end
}
end
end
def check_dirs(target_dir)
@queue.new(:check_dirs) do
Dir.foreach(target_dir)
.select{|x| x =~ %r<\A(?:[a-fA-F0-9]{2})\Z> }
.shuffle
.each{|subdir|
check_subdirs(File.join(target_dir, subdir))
}
Delayer.new(:destroy_cache, delay: cache_expire) do
check_dirs(target_dir) end
end
end
def _loop
Delayer.new(:destroy_cache, delay: 60) do
if @queue
@queue.run
_loop
end
end
end
on_unload do
@db.close
@journal_db.close
@journal_db = @db = @queue = nil
end
check_dirs(@cache_directory)
check_dirs(@cache_journal_directory)
_loop
end
|
