1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
|
#
# Copyright (C) 2002 Manuel Estrada Sainz <ranty@debian.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of version 2.1 of the GNU Lesser General Public
# License as published by the Free Software Foundation.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import os, time
from twisted.internet import reactor, defer
from twisted import python
class DomainLogger:
"""
This class should help us classify messages into domains and levels.
This way we can select messages by kind and level.
You just have to set in the configuration file something like:
debug = db:3 import:8
Which means that we only want to see messages of domain 'db' and
level <= 3 and domain 'import' and level <= 8
There are three special domains:
all: if enabled all messages will be shown.
log: is on by default and only the level can be changed
it is meant for production logging.
debug: aptProxyConfig will define it if you select any loging
domains.
Pretended meaning of levels:
0: nothing or maybe critical information
1: important information
...
9: useless information
"""
def __init__(self, enabled={'all':9}):
self.enabled = enabled
def setDomains(self, domains):
self.enabled = domains
def addDomains(self, domains):
self.enabled.update(domains)
#print "enabled: ", self.enabled
def isEnabled(self, domain, level=9):
domains = self.enabled.keys()
if domain in domains and level > self.enabled[domain]:
return 0
if(('all' in domains and level <= self.enabled['all'])
or (domain in domains and level <= self.enabled[domain])):
return 1
else:
return 0
def msg(self, msg, domain='log', level=4):
"Logs 'msg' if domain and level are appropriate"
#print 'domain:', domain, 'level:', level
if self.isEnabled(domain, level):
try:
python.log.msg("[%s] %s"%(domain, msg))
except IOError:
pass
def debug(self, msg, domain='debug', level=9):
"Useful to save typing on new debuging messages"
if self.isEnabled(domain, level):
try:
python.log.msg("[%s] %s"%(domain, msg), debug=True)
except IOError:
pass
def err(self, msg, domain='error', level=9):
"Log an error message"
try:
python.log.err("[%s] %s"%(domain, msg))
except IOError:
pass
# Prevent log being replace on reload. This only works in cpython.
try:
log
except NameError:
log = DomainLogger()
class MirrorRecycler:
"""
Reads the mirror tree looking for 'forgotten' files and adds them to
factory.access_times so they can age and be removed like the others.
It processes one directory entry per 'timer' seconds, which unless
set to 0 is very slow, but it is also very light weight. And files
which get recuested are recycled automatically anyway, so it is
not urgent to find forgotten files. If also uses the files oun
atime, so if the files has been there for a long time it will soon
be removed anyway.
"""
working = 0
def __init__(self, factory, timer):
self.timer = timer
self.factory = factory
self.callback = None
self.working = None # Deferred triggered when recycler finishes
def start(self):
"""
Starts the Recycler if it is not working, it will use
callLater to keep working until it finishes with the whole
tree.
"""
if not self.working:
self.working = defer.Deferred()
if self.factory.backends == []:
log.msg("NO BACKENDS FOUND",'recycle')
self.working.errback(python.failure.Failure())
return self.working
self.cur_uri = '/'
self.cur_dir = self.factory.config.cache_dir
self.pending = []
for backend in self.factory.backends.values():
self.pending.append(backend.base)
self.stack = []
self.callback = reactor.callLater(self.timer, self.process)
return self.working
def stop(self):
if self.callback is not None:
self.callback.cancel()
self.callback = None
if self.working:
self.working.callback(None)
self.working = None
def pop(self):
if self.stack:
(self.cur_dir, self.cur_uri, self.pending) = self.stack.pop()
else:
self.working.callback(None)
self.working = None
def push(self):
if self.pending:
self.stack.append((self.cur_dir, self.cur_uri, self.pending))
def process(self):
"""
Process the next entry, is called automatically via callLater.
"""
self.callback = None
entry = self.pending.pop()
uri = os.path.join(self.cur_uri, entry)
path = os.path.join(self.cur_dir, entry)
if not os.path.exists(path):
pass
elif os.path.isdir(path):
self.push()
self.cur_dir = path
self.cur_uri = uri
self.pending = os.listdir(self.cur_dir)
if not self.pending:
# Prune directory if it has not just been created
pathage = time.time() - os.path.getctime(path)
if pathage > 60:
log.msg("Pruning empty directory: "+path,'recycle')
os.removedirs(path)
elif os.path.isfile(path):
ext = os.path.splitext(path)[1]
if not ext == 'apDownload':
#print "PATH:", path
#print "URI: ", uri
if not self.factory.access_times.has_key(uri):
log.msg("Adopting new file: "+ uri,'recycle')
self.factory.access_times[uri] = os.path.getatime(path)
else:
log.msg("UNKNOWN:"+path,'recycle')
if not self.pending:
self.pop()
if self.working:
self.callback = reactor.callLater(self.timer, self.process)
if __name__ == '__main__':
#Just for testing purposes.
from apt_proxy_conf import aptProxyFactoryConfig
import shelve
class DummyFactory:
pass
factory = DummyFactory()
aptProxyFactoryConfig(factory)
factory.access_times=shelve.open("tmp.db")
recycle = MirrorRecycler(factory, 10)
recycle.start()
while recycle.working:
recycle.process()
factory.access_times.close()
|
