File: meta.py

package info (click to toggle)
thawab 4.1-2
  • links: PTS, VCS
  • area: non-free
  • in suites: bullseye, sid
  • size: 1,912 kB
  • sloc: python: 4,041; makefile: 134; sh: 33
file content (262 lines) | stat: -rw-r--r-- 8,782 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
# -*- coding: UTF-8 -*-
"""
The meta handling classes of thawab
Copyright © 2008, Muayyad Alsadi <alsadi@ojuba.org>

        Released under terms of Waqf Public License.
        This program is free software; you can redistribute it and/or modify
        it under the terms of the latest version Waqf Public License as
        published by Ojuba.org.

        This program is distributed in the hope that it will be useful,
        but WITHOUT ANY WARRANTY; without even the implied warranty of
        MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.

        The Latest version of the license can be found on
        "http://waqf.ojuba.org/license"

"""
import os
import os.path
import sqlite3
import threading
import time
import hashlib
from itertools import groupby
from .dataModel import *
from okasha.utils import fromFs, toFs, strverscmp
import re

def prettyId(i, empty_for_special = True):
    """convert the id into a more human form"""
    if empty_for_special and i.startswith('_'):
        return ''
    return i.replace('_',' ')

def makeId(i):
    """convert the id into a canonical form"""
    return i.strip().replace(' ','_').replace('/','_')

def metaVr(m):
    return m["version"] + "-" + str(m["releaseMajor"])

def metaVrr(m):
    return "-".join((m["version"],
                      str(m["releaseMajor"]),
                      str(m["releaseMinor"])))

def metaDict2Hash(meta, suffix = None):
    k = [i for i in list(meta.keys()) if i != 'cache_hash']
    k.sort()
    l = []
    for i in k:
        l.append("%s:%s" % (i,meta[i]))
    l.append("timestamp:%d" % int(time.time()))
    if suffix:
        l.append(suffix)
    return hashlib.sha256(("-".join(l)).encode('utf-8')).digest().encode('base64').strip()[:-1]

class MCache(object):
    """a class holding metadata cache"""
    def __init__(self, mcache_db, uri_list, smart = -1):
        self.db_fn = mcache_db
        if not os.path.exists(mcache_db):
            create_new = True
        else:
            create_new = False
        self._cn = {}
        cn = self._getConnection()
        if create_new:
            cn.executescript(SQL_MCACHE_DATA_MODEL)
            cn.commit()
        self.__reload()
        if self.__create_cache(uri_list, smart) > 0:
            self.__reload()

    def _getConnection(self):
        n = threading.current_thread().name
        if n in self._cn:
            r = self._cn[n]
        else:
            r = sqlite3.connect(self.db_fn)
            r.row_factory = sqlite3.Row
            self._cn[n] = r
        return r


    def __reload(self):
        self.__meta = [dict(i) for i in self._getConnection().execute(SQL_MCACHE_GET_BY_KITAB)]
        self.__meta_by_uri = (dict([(a[1]['uri'], a[0]) for a in enumerate(self.__meta)]))
        self.__meta_uri_list = list(self.__meta_by_uri.keys())
        self.__meta_by_kitab = {}
        for k,G in groupby(enumerate(self.__meta), lambda a: a[1]['kitab']):
            g = list(G)
            self.__meta_by_kitab[k] = [i[0] for i in g]

    def load_from_uri(self, uri):
        """extract meta object from kitab's uri and return it"""
        cn = sqlite3.connect(uri)
        cn.row_factory=sqlite3.Row
        c = cn.cursor()
        try:
            r = c.execute(SQL_MCACHE_GET).fetchone()
        except sqlite3.OperationalError:
            return None
        if not r:
            return None
        return dict(r)

    def __cache(self, c, uri, meta = None):
        if not meta:
            meta = self.load_from_uri(uri)
        if not meta:
            return 0
        #if drop_old_needed: 
        meta['uri'] = uri
        meta['mtime'] = os.path.getmtime(toFs(uri))
        meta['flags'] = 0
        c.execute(SQL_MCACHE_ADD, meta)
        return 1

    def __create_cache(self, uri_list, smart = -1):
        """
        create cache and return the number of newly created meta caches
        
        smart is how fast you want to do that:
            * 0 force regeneration of entire meta cache
            * 1 regenerate cache when hash differs (it would need to open every kitab)
            * 2 regenerate when mtime differs
            * -1 do not update cache for exiting meta (even if the file is changed)
        """
        cn = self._getConnection()
        c = cn.cursor()
        r = 0
        uri_set = set(uri_list)
        #c.execute('BEGIN TRANSACTION')
        # remove meta for kitab that no longer exists
        deleted = [i for i in self.__meta_uri_list if i not in uri_set]
        for uri in deleted:
            c.execute(SQL_MCACHE_DROP, (uri,))
            r += 1
        # update meta for the rest (in a smart way)
        for uri in uri_list:
            if not os.access(toFs(uri), os.R_OK): continue
            if smart == 0:
                # force recreation of cache, drop all, then create all
                r+=self.__cache(c, uri, uri in self.__meta_uri_list)
                continue
            meta = None
            drop_old_needed = False
            cache_needed = False
            if uri not in self.__meta_uri_list:
                cache_needed = True
            else:
                drop_old_needed = True
                cache_needed = True
                if smart == -1: continue # don't replace existing cache
                elif smart == 2: # rely of mtime
                    if abs(os.path.getmtime(toFs(uri)) - self.getByUri(uri)['mtime']) < 1e-5:
                        continue
                elif smart == 1: # rely on a hash saved inside the database
                    old_meta = self.getByUri(uri)
                    meta = self.load_from_uri(uri)
                    if not meta or old_meta['hash'] == meta['hash']:
                        continue
            if cache_needed:
                r += self.__cache(c, uri, meta)
        #c.execute('END TRANSACTION')
        cn.commit()
        return r

    def getKitabList(self):
        return list(self.__meta_by_kitab.keys())

    def getUriList(self):
        return list(self.__meta_by_uri.keys())

    def getByUri(self, uri):
        """return meta object for uri"""
        i = self.__meta_by_uri.get(uri,None)
        if i == None: return None
        return self.__meta[i]

    def getByKitab(self, kitab):
        """return a list of meta objects for a kitab"""
        a = self.__meta_by_kitab.get(kitab,None)
        if not a:
            return None
        return [self.__meta[i] for i in a]

    def _latest(self, a):
        lm = a[0]
        l = metaVrr(lm)
        for m in a[1:]:
            v = metaVrr(m)
            if strverscmp(v, l) > 0:
                lm = m
                l = v
        return lm

    def getLatestKitab(self, kitab):
        """return a meta object for latest kitab (based on version)"""
        a = self.__meta_by_kitab.get(kitab, None)
        if not a:
            return None
        return self._latest([self.__meta[i] for i in a])

    def getLatestKitabV(self, kitab, v):
        """
        given kitab name and version
        return a meta object for latest kitab (based on version)
        """
        a = self.__meta_by_kitab.get(kitab, None)
        if not a:
            return None
        ma = [m for m in [self.__meta[i] for i in a] if m['version'] == v]
        if not ma:
            return None
        return self._latest(ma)

    def getLatestKitabVr(self, kitab, v, r):
        """
        given kitab name and version and major release
        return a meta object for latest kitab (based on version)
        """
        if type(r) != int:
            r = int(r)
        a = self.__meta_by_kitab.get(kitab, None)
        ma = [m for m in [self.__meta[i] for i in a] if m['version'] == v and m['releaseMajor'] == r]
        if not ma:
            return None
        return self._latest(ma)

    def setIndexedFlags(self, uri, flags=2):
        cn = self._getConnection()
        cn.execute(SQL_MCACHE_SET_INDEXED, (flags, uri,))
        cn.commit()

    def setAllIndexedFlags(self, flags=0):
        cn = self._getConnection()
        cn.execute(SQL_MCACHE_SET_ALL_INDEXED, (flags,))
        cn.commit()

    def getUnindexedList(self):
        """
        return a list of meta dicts for Kutub that are likely to be unindexed
        """
        
        return [dict(i) for i in self._getConnection().execute(SQL_MCACHE_GET_UNINDEXED)]

    def getDirtyIndexList(self):
        """
        return a list of meta dicts for Kutub that are likely to have broken index
        """
        return [dict(i) for i in self._getConnection().execute(SQL_MCACHE_GET_DIRTY_INDEX)]

    def getIndexedList(self):
        """
        return a list of meta dicts for Kutub that are already in index.
        """
        return [dict(i) for i in self._getConnection().execute(SQL_MCACHE_GET_INDEXED)]