1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
|
#!/usr/bin/python3
# Copyright (C) 2023 J.F.Dockes
#
# License: GPL 2.1
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2.1 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the
# Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
"""Report the longest documents in a Recoll index (by term count), and the biggest
directories (by sum of term counts for contained documents).
This should give a pretty good estimate of what consumes space in your index.
You will need to install the Xapian and Recoll Python3 bindings (e.g. python3-xapian,
python3-recoll on Debian or Ubuntu)"""
import sys
import os
import getopt
import xapian
import recoll.rclconfig as rclconfig
# Default count of biggest files we show
nfilesprinted = 10
# Default depth of paths for which we compute occupation. e.g. /home/me/doc/science would be
# at depth 4
dirdepth = 4
# Default count of directories for which we print the totals
ndirsprinted = 20
def msg(s):
print(f"{s}", file=sys.stderr)
# Retrieve named value(s) from a recoll document data record.
# The record format is a sequence of nm=value lines
def get_attributes(xdb, xdocid, flds, decode=True):
# msg(f"get_attributes xdocid {xdocid} flds {flds} decode {decode}")
doc = xdb.get_document(xdocid)
data = doc.get_data()
res = []
for fld in flds:
s = data.find(fld + b"=")
if s == -1:
res.append(None)
else:
e = data.find(b"\n", s)
if decode:
res.append(data[s + len(fld) + 1 : e].decode("UTF-8"))
else:
res.append(data[s + len(fld) + 1 : e])
return res
# Sample code. Not called and does nothing at the moment. Sample code for looking at doc terms in
# more detail
def doc_details(xdb, xdocid):
# xdoc = xdb.get_document(xdocid)
# for term in xdoc.termlist():
# msg(f"TERM {term.term}")
# for position in xdb.positionlist(xdocid, term.term):
# poscount +=1
# msg(f"Positions count {poscount}")
pass
if __name__ == "__main__":
# Note: the -c option is only for command line tests, the recoll process always sets the config
# in RECOLL_CONFDIR
def usage(f=sys.stderr):
prog = os.path.basename(sys.argv[0])
print(f"Usage: {prog} [OPTION]...", file=f)
print(f" -h, --help show this help", file=f)
print(f" -c, --config select Recoll configuration directory", file=f)
print(
f" -n, --files-count set the size of the file list"
f" (default {nfilesprinted})",
file=f,
)
print(
f" -d, --depth set the depth of the selected directories (default {dirdepth})",
file=f,
)
print(
f" -N, --dirs-count set the size of the directory list (default {ndirsprinted})",
file=f,
)
sys.exit(1)
confdir = None
try:
options, args = getopt.getopt(
sys.argv[1:],
"hc:d:N:n:",
["help", "config=", "files-count=", "depth=", "dirs-count="],
)
except Exception as err:
print(err, file=sys.stderr)
usage()
for o, a in options:
if o in ("-h", "--help"):
usage(sys.stdout)
elif o in ("-c", "--config"):
confdir = a
elif o in ("-d", "--depth"):
dirdepth = int(a)
elif o in ("-N", "--dirs-count"):
ndirsprinted = int(a)
elif o in ("-n", "--files-count"):
nfilesprinted = int(a)
if len(args) != 0:
usage()
rclconfig = rclconfig.RclConfig(argcnf=confdir)
confdir = rclconfig.getConfDir()
dbdir = rclconfig.getDbDir()
xdb = xapian.Database(dbdir)
alldoclengths = []
# Walk the Xapian index whole document list (postlist for empty term). Take note of the document
# term counts.
lastxdocid = xdb.get_lastdocid()
for doc in xdb.postlist(""):
xdocid = int(doc.docid)
if xdocid % 10000 == 0:
msg(f"Walking whole Xapian term list: docid {xdocid}/{lastxdocid}")
try:
url = get_attributes(
xdb,
xdocid,
[
b"url",
],
decode=False,
)[0]
except Exception as ex:
msg(f"get_attributes failed: {ex}")
continue
alldoclengths.append((doc.doclength, url))
# Sort by document term count, highest first
alldoclengths.sort(reverse=True)
# Walk the sorted list, print the "nfilesprinted" longest documents sizes and paths, accumulate
# the sizes for directories at depth "dirdepth"
dirsizes = {}
i = 0
for e in alldoclengths:
path = e[1][7:]
if i < nfilesprinted:
print(f"{e[0]} {path}")
i += 1
path = os.path.dirname(path)
# print(f"PATH {path}")
l = path.split(b"/")
if len(l) < dirdepth:
dir = path
else:
dir = b"/".join(l[0 : dirdepth + 1])
# print(f"DIR {dir}")
if dir in dirsizes:
dirsizes[dir] += e[0]
else:
dirsizes[dir] = e[0]
# Print the biggest directories
print("\nDirectory sizes:")
dirsizes = sorted(dirsizes.items(), key=lambda entry: entry[1], reverse=True)
for dir, sz in dirsizes[0:ndirsprinted]:
print(f"{sz} {dir}")
|
