File: __init__.py

package info (click to toggle)
kopanocore 8.7.0-3
  • links: PTS, VCS
  • area: main
  • in suites: bullseye, buster, sid
  • size: 15,400 kB
  • sloc: cpp: 175,422; python: 24,623; perl: 7,319; php: 6,056; sh: 2,172; makefile: 1,294; xml: 45; ansic: 1
file content (469 lines) | stat: -rw-r--r-- 20,710 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
#!/usr/bin/python3
# SPDX-License-Identifier: AGPL-3.0-or-later
from .version import __version__

import codecs
import csv
import datetime
import json
import time
import sys

from collections import defaultdict

from MAPI.Util import *
from MAPI.Tags import (PS_COMMON, PS_ARCHIVE,
                       PR_SENDER_EMAIL_ADDRESS_W, PR_SENDER_ENTRYID,
                       PR_SENDER_NAME_W, PR_SENDER_ADDRTYPE_W,
                       PR_SENDER_SEARCH_KEY,
                       PR_SENT_REPRESENTING_ENTRYID, PR_SENT_REPRESENTING_NAME_W,
                       PR_SENT_REPRESENTING_SEARCH_KEY, PR_SENT_REPRESENTING_EMAIL_ADDRESS_W,
                       PR_SENT_REPRESENTING_ADDRTYPE_W,
                       PR_RECEIVED_BY_ENTRYID, PR_RECEIVED_BY_NAME_W, PR_RECEIVED_BY_SEARCH_KEY,
                       PR_RECEIVED_BY_EMAIL_ADDRESS_W, PR_RECEIVED_BY_ADDRTYPE_W,
                       PR_RCVD_REPRESENTING_ENTRYID, PR_RCVD_REPRESENTING_NAME_W,
                       PR_RCVD_REPRESENTING_SEARCH_KEY, PR_RCVD_REPRESENTING_EMAIL_ADDRESS_W,
                       PR_RCVD_REPRESENTING_ADDRTYPE_W)
import MAPI.Time
import kopano
from kopano import log_exc

from kopano.pidlid import (
    PidLidDistributionListMembers,
    PidLidDistributionListOneOffMembers
)

from . import pst

if sys.hexversion >= 0x03000000:
    def _encode(s):
        return s

    def bval(x):
        return x

else: # pragma: no cover
    def _encode(s):
        return s.encode(sys.stdout.encoding or 'utf8')

    def bval(x):
        return ord(x)

WRAPPED_ENTRYID_PREFIX = codecs.decode('00000000C091ADD3519DCF11A4A900AA0047FAA4', 'hex')
WRAPPED_EID_TYPE_MASK = 0xf
WRAPPED_EID_TYPE_CONTACT = 3
WRAPPED_EID_TYPE_PERSONAL_DISTLIST = 4

REMINDER_SIGNALTIME_ID = 0x8560
REMINDER_SET_ID = 0x8503

NOW = datetime.datetime.utcnow()

SENDER_PROPS = {
    'entryid': PR_SENDER_ENTRYID,
    'name': PR_SENDER_NAME_W,
    'searchkey': PR_SENDER_SEARCH_KEY,
    'email': PR_SENDER_EMAIL_ADDRESS_W, # PidTagSenderSmtpAddress
    'addrtype': PR_SENDER_ADDRTYPE_W,
    'smtp': pst.PropIdEnum.PidTagSentRepresentingSmtpAddress,
}

SENT_REPRESENTING_PROPS = {
    'entryid': PR_SENT_REPRESENTING_ENTRYID,
    'name': PR_SENT_REPRESENTING_NAME_W,
    'searchkey': PR_SENT_REPRESENTING_SEARCH_KEY,
    'email': PR_SENT_REPRESENTING_EMAIL_ADDRESS_W, # PidTagSentRepresentingSmtpAddress
    'addrtype': PR_SENT_REPRESENTING_ADDRTYPE_W,
    'smtp': pst.PropIdEnum.PidTagSenderSmtpAddress,
}

RECEIVED_PROPS = {
    'entryid': PR_RECEIVED_BY_ENTRYID,
    'name': PR_RECEIVED_BY_NAME_W,
    'searchkey': PR_RECEIVED_BY_SEARCH_KEY,
    'email': PR_RECEIVED_BY_EMAIL_ADDRESS_W, # PidTagReceivedBySmtpAddress
    'addrtype': PR_RECEIVED_BY_ADDRTYPE_W,
    'smtp': pst.PropIdEnum.PidTagReceivedBySmtpAddress,
}

RECEIVED_REPR_PROPS = {
    'entryid': PR_RCVD_REPRESENTING_ENTRYID,
    'name': PR_RCVD_REPRESENTING_NAME_W,
    'searchkey': PR_RCVD_REPRESENTING_SEARCH_KEY,
    'email': PR_RCVD_REPRESENTING_EMAIL_ADDRESS_W, # PidTagReceivedRepresentingSmtpAddress
    'addrtype': PR_RCVD_REPRESENTING_ADDRTYPE_W,
    'smtp': pst.PropIdEnum.PidTagReceivedRepresentingSmtpAddress,
}

EMAIL_RECIP_PROPS = [SENDER_PROPS, SENT_REPRESENTING_PROPS, RECEIVED_PROPS, RECEIVED_REPR_PROPS]

def recip_prop(propid, value):
    # PST not generated with full unicode?
    if isinstance(value, bytes):
        return SPropValue(PROP_TAG(PT_STRING8, propid), value)
    else:
        return SPropValue(PROP_TAG(PT_UNICODE, propid), value)

def rev_cp1252(value):
    # PST not generated with full unicode flag?
    if isinstance(value, bytes):
        return value.decode('cp1252')
    else:
        return value

class Service(kopano.Service):
    def import_props(self, parent, mapiobj, embedded=False):
        props2 = {}
        attach_method = subnode_nid = None
        reminder_in_past = False
        reminder_set_id = None
        for k, v in parent.pc.props.items():
            propid, proptype, value = k, v.wPropType, v.value

            if propid == (PR_MESSAGE_CLASS_W>>16):
                if value == 'IPM.DistList':
                    self.distlist_entryids.append(parent.EntryId)

            if proptype in (PT_CURRENCY, PT_MV_CURRENCY, PT_ACTIONS, PT_SRESTRICT, PT_SVREID):
                continue # unsupported by parser

            nameid = self.propid_nameid.get(propid)
            if nameid:
                propid = PROP_ID(mapiobj.GetIDsFromNames([MAPINAMEID(*nameid)], MAPI_CREATE)[0])
                if nameid[0] == PS_ARCHIVE:
                    continue

                elif nameid[0] == PS_COMMON:
                    if nameid[2] == REMINDER_SIGNALTIME_ID:
                        if MAPI.Time.FileTime(value).datetime() < NOW:
                            reminder_in_past = True
                    elif nameid[2] == REMINDER_SET_ID:
                        reminder_set_id = propid

            if propid == (PR_SUBJECT_W>>16) and value and ord(value[0]) == 0x01:
                value = value[2:] # \x01 plus another char indicates normalized-subject-prefix-length
            elif proptype == PT_SYSTIME:
                value = MAPI.Time.FileTime(value)
            elif propid == PR_ATTACH_METHOD >> 16:
                attach_method = value
            elif propid == PR_ATTACH_DATA_OBJ >> 16 and len(value) == 4: # XXX why not 4?
                subnode_nid = struct.unpack('I', value)[0]

            if isinstance(parent, pst.Attachment) or embedded or PROP_TAG(proptype, propid) != PR_ATTACH_NUM: # work around webapp bug? KC-390
                props2[propid] = SPropValue(PROP_TAG(proptype, propid), value)

        if attach_method == ATTACH_EMBEDDED_MSG and subnode_nid is not None:
            submessage = pst.Message(subnode_nid, self.ltp, self.nbd, parent)
            submapiobj = mapiobj.OpenProperty(PR_ATTACH_DATA_OBJ, IID_IMessage, 0, MAPI_CREATE | MAPI_MODIFY)
            self.import_attachments(submessage, submapiobj)
            self.import_recipients(submessage, submapiobj)
            self.import_props(submessage, submapiobj, embedded=True)

        # Optionally dismiss reminders for events in the past
        if (self.options.dismiss_reminders and \
            reminder_in_past and \
            reminder_set_id is not None):
            props2[reminder_set_id].Value = False

        proplist = list(props2.values()) # TODO move down and overwrite instead of append

        # Check if any recipient property on an item has an EX address and try to convert it.
        for prop_dict in EMAIL_RECIP_PROPS:
            addrtype = parent.pc.getval(PROP_ID(prop_dict['addrtype']))
            if addrtype and addrtype == 'EX':
                proplist.extend(self.convert_exchange_recipient(parent, prop_dict))

        mapiobj.SetProps(proplist)
        mapiobj.SaveChanges(KEEP_OPEN_READWRITE)

    def convert_exchange_recipient(self, parent, convertprops):
        props = []
        email = parent.pc.getval(convertprops['smtp'])

        # When mapping is provided and no email address was found
        if not email and self.options.mapping:
            legacyExchangeDN = parent.pc.getval(PROP_ID(convertprops['email']))
            if legacyExchangeDN:
                legacyExchangeDN = legacyExchangeDN.lower()
                try:
                    email = self.mapping[legacyExchangeDN]
                except KeyError:
                    if legacyExchangeDN not in self.nomapping:
                        self.log.warning('legacyExchangeDN: \'%s\' was not found in mapping', legacyExchangeDN)
                        self.nomapping.add(legacyExchangeDN)
            else:
                self.log.debug('item has no legacyExchangeDN, skipping importing recipient data')

        # Still no email, logs erros and add it to stats
        if not email:
            self.stats['noemail'] += 1
            recip = 'sender' if convertprops['email'] in [PR_SENDER_EMAIL_ADDRESS_W, PR_SENT_REPRESENTING_EMAIL_ADDRESS_W] else 'receiver'
            self.log.warn('no email address found for %s property, skipping', recip)
            return []

        try:
            user = self.server.user(email=email)
        except kopano.NotFoundError:
            user = None

        if user:
            props.append(SPropValue(convertprops['addrtype'], u'ZARAFA'))
            props.append(SPropValue(convertprops['email'], user.email))
            props.append(SPropValue(convertprops['name'], user.fullname))
            props.append(SPropValue(convertprops['entryid'], codecs.decode(user.userid, 'hex')))
            props.append(SPropValue(convertprops['searchkey'], user[PR_SEARCH_KEY]))  # TODO: add to Pyko.
        else:
            name = parent.pc.getval(PROP_ID(convertprops['name']))
            if not name:
                name = email

            props.append(SPropValue(convertprops['addrtype'], u'SMTP'))
            props.append(SPropValue(convertprops['email'], email))
            props.append(SPropValue(convertprops['name'], name))
            props.append(SPropValue(convertprops['entryid'], self.server.ab.CreateOneOff(name, u'SMTP', email, MAPI_UNICODE)))
            props.append(SPropValue(convertprops['searchkey'], b'SMTP:' + email.encode('UTF-8') + b'\x00'))

        return props

    def import_attachments(self, message, mapiobj):
        for attachment in message.subattachments:
            attachment = message.get_attachment(attachment)
            (id_, attachment2) = mapiobj.CreateAttach(None, 0)
            self.import_props(attachment, attachment2)

    def import_recipients(self, message, mapiobj):
        recipients = [] # XXX groups etc?
        for r in message.subrecipients:
            user = None
            key = None
            if r.ObjectType == 6 and r.DisplayType == 0:
                if r.AddressType == 'EX':
                    key = r.SmtpAddress
                if not r.AddressType: # missing addr type observed in the wild
                    key = r.DisplayName or r.EmailAddress
                elif r.AddressType == 'ZARAFA':
                    key = r.EmailAddress or r.DisplayName
            if key:
                try:
                    user = self.server.user(email=key) # XXX using email arg for name/fullname/email
                except kopano.NotFoundError:
                    if key not in self.unresolved:
                        self.log.warning("could not resolve user '%s'" % key)
                        self.unresolved.add(key)
            props = []
            if r.RecipientType is not None:
                props.append(SPropValue(PR_RECIPIENT_TYPE, r.RecipientType))
            if user or r.AddressType is not None:
                value = u'ZARAFA' if user else r.AddressType
                value = u'SMTP' if not user and r.AddressType == 'EX' else value
                props.append(recip_prop(PROP_ID(PR_ADDRTYPE), value))
            if user or r.DisplayName is not None:
                value = user.fullname if user else r.DisplayName
                props.append(recip_prop(PROP_ID(PR_DISPLAY_NAME), value))
            if r.DisplayType is not None:
                props.append(SPropValue(PR_DISPLAY_TYPE, r.DisplayType))
            if user:
                props.append(SPropValue(PR_ENTRYID, codecs.decode(user.userid, 'hex')))
                props.append(recip_prop(PROP_ID(PR_EMAIL_ADDRESS), user.name))
            else:
                email = None
                if r.AddressType == 'EX':
                    if r.SmtpAddress:
                        email = r.SmtpAddress
                    else:
                        email = self.mapping.get(r.EmailAddress.lower())
                elif r.AddressType == 'SMTP':
                    email = r.EmailAddress
                elif r.AddressType == 'ZARAFA':
                    email = r.SmtpAddress
                if email:
                    props.append(SPropValue(PR_ENTRYID, self.server.ab.CreateOneOff(r.DisplayName, u'SMTP', email, MAPI_UNICODE)))
                    props.append(recip_prop(PROP_ID(PR_EMAIL_ADDRESS), email))
                else:
                    self.log.warning("no email address for recipient '%s'" % r.DisplayName or '')
            recipients.append(props)
        mapiobj.ModifyRecipients(0, recipients)

    def import_pst(self, p, store):
        folders = p.folder_generator()
        root_path = rev_cp1252(next(folders).path) # skip root
        self.distlist_entryids = []
        self.entryid_map = {}

        type_map = [
            ('IPM.Note', 'mail'),
            ('IPM.Schedule', 'mail'),
            ('IPM.Contact', 'contact'),
            ('IPM.DistList', 'distlist'),
            ('IPM.Appointment', 'appointment'),
        ]

        for folder in folders:
            with log_exc(self.log, self.stats):
                path = rev_cp1252(folder.path[len(root_path)+1:])
                if self.options.folders and \
                   path.lower() not in [f.lower() for f in self.options.folders]:
                    continue
                self.log.info("importing folder '%s'" % path)
                if self.options.import_root:
                    path = self.options.import_root + '/' + path
                folder2 = store.folder(path, create=True)
                if self.options.clean_folders:
                    folder2.empty()
                if folder.ContainerClass:
                    folder2.container_class = folder.ContainerClass
                for message in p.message_generator(folder):
                    with log_exc(self.log, self.stats):
                        for (class_, type_) in type_map:
                            if message.MessageClass and message.MessageClass.startswith(class_):
                                break
                        else:
                            type_ = 'item'
                        self.log.debug("importing %s '%s'" % (type_, rev_cp1252(message.Subject or '')))
                        message2 = folder2.create_item(save=False)
                        self.entryid_map[message.EntryId] = message2.entryid
                        self.import_attachments(message, message2.mapiobj)
                        self.import_recipients(message, message2.mapiobj)
                        self.import_props(message, message2.mapiobj)
                        self.stats['messages'] += 1
        self.rewrite_entryids(store)

    def rewrite_entryids(self, store):
        # distribution lists
        for eid in self.distlist_entryids:
            with log_exc(self.log, self.stats):
                item = store.item(self.entryid_map[eid])
                members = item.get(PidLidDistributionListMembers)
                oneoffs = item.get(PidLidDistributionListOneOffMembers)
                if members:
                    for i, (member, oneoff) in enumerate(zip(members, oneoffs)):
                        pos = len(WRAPPED_ENTRYID_PREFIX)
                        prefix, flags, rest = member[:pos], bval(member[pos]), member[pos+1:]
                        if (prefix == WRAPPED_ENTRYID_PREFIX and \
                            (flags & WRAPPED_EID_TYPE_MASK) in (WRAPPED_EID_TYPE_CONTACT, WRAPPED_EID_TYPE_PERSONAL_DISTLIST) and \
                            rest in self.entryid_map):
                                members[i] = member[:pos+1] + codecs.decode(self.entryid_map[rest], 'hex')
                        else: # fall-back to oneoff if we can't resolve
                            members[i] = oneoffs[i]
                    item[PidLidDistributionListMembers] = members

    def get_named_property_map(self, p):
        propid_nameid = {}
        for nameid in p.messaging.nameid_entries:
            propid_nameid[nameid.NPID] = (
                nameid.guid,
                MNID_STRING if nameid.N==1 else MNID_ID,
                nameid.name if nameid.N==1 else nameid.dwPropertyID
            )
        return propid_nameid

    @property
    def mapping(self):
        if not self._mapping and self.options.mapping:
            with open(self.options.mapping, 'r') as fp:
                self._mapping = json.load(fp)
        return self._mapping

    def main(self):
        self._mapping = {}
        self.nomapping = set()
        self.stats = {'messages': 0, 'errors': 0, 'noemail': 0}

        pst.set_log(self.log, self.stats)
        self.unresolved = set()
        t0 = time.time()
        for arg in self.args:
            self.log.info("importing file '%s'" % arg)
            try:
                p = pst.PST(arg)
            except pst.PSTException:
                self.log.error("'%s' is not a valid PST file", arg)
                continue
            self.nbd, self.ltp = p.nbd, p.ltp
            self.propid_nameid = self.get_named_property_map(p)
            for name in self.options.users:
                self.log.info("importing to user '%s'" % name)
                store = self.server.user(name).store
                if store:
                    self.import_pst(p, store)
                else:
                    self.log.error("user '%s' has no store", name)
            for guid in self.options.stores:
                store = self.server.store(guid)
                self.log.info("importing to store '%s'" % guid)
                if store:
                    self.import_pst(p, store)
                else:
                    self.log.error("guid '%s' has no store", name)

        self.log.info('imported %d items in %.2f seconds (%.2f/sec, %d errors) (no resolved email address: %d)' %
            (self.stats['messages'], time.time()-t0, self.stats['messages']/(time.time()-t0), self.stats['errors'], self.stats['noemail']))


def show_contents(args, options):
    writer = csv.writer(sys.stdout)
    for arg in args:
        p = pst.PST(arg)
        folders = p.folder_generator()
        root_path = rev_cp1252(next(folders).path) # skip root
        for folder in folders:
            path = rev_cp1252(folder.path[len(root_path)+1:])
            if options.folders and path.lower() not in [f.lower() for f in options.folders]:
                continue
            if options.stats:
                writer.writerow([_encode(path), folder.ContentCount])
            elif options.index:
                for message in p.message_generator(folder):
                    writer.writerow([_encode(path), _encode(rev_cp1252(message.Subject or ''))])


def create_mapping(args, options):
    mapping = defaultdict(dict)
    for arg in args:
        print("Creating mapping of PST '%s'" % arg)
        try:
            p = pst.PST(arg)
        except pst.PSTException:
            print("'%s' is not a valid PST file" % arg)
            continue

        for folder in p.folder_generator():
            for message in p.message_generator(folder):
                for rec in message.subrecipients:
                    if rec.AddressType != 'EX':
                        continue
                    if not rec.EmailAddress or not rec.SmtpAddress:
                        continue
                    mapping[rec.EmailAddress.lower()] = rec.SmtpAddress

    with open(options.create_mapping, 'w') as fp:
        json.dump(mapping, fp)
        print('Written mapping file to \'%s\'.' % options.create_mapping)


def main():
    parser = kopano.parser('cflskpUPuS', usage='kopano-migration-pst PATH [-u NAME]')
    parser.add_option('', '--stats', dest='stats', action='store_true', help='list folders for PATH')
    parser.add_option('', '--index', dest='index', action='store_true', help='list items for PATH')
    parser.add_option('', '--import-root', dest='import_root', action='store', help='import under specific folder', metavar='PATH')
    parser.add_option('', '--clean-folders', dest='clean_folders', action='store_true', default=False, help='empty folders before import (dangerous!)', metavar='PATH')
    parser.add_option('', '--create-ex-mapping', dest='create_mapping', help='create legacyExchangeDN mapping file', metavar='FILE')
    parser.add_option('', '--ex-mapping', dest='mapping', help='use legacyExchangeDN mapping file', metavar='FILE')
    parser.add_option('', '--dismiss-reminders', dest='dismiss_reminders', action='store_true', default=False, help='dismiss reminders for events in the past')

    options, args = parser.parse_args()
    options.service = False

    if not args or (bool(options.stats or options.index or options.create_mapping) == bool(options.users or options.stores)):
        parser.print_help()
        sys.exit(1)

    if options.stats or options.index:
        show_contents(args, options)
    elif options.create_mapping:
        create_mapping(args, options)
    else:
        Service('migration-pst', options=options, args=args).start()

if __name__ == '__main__':
    main() # pragma: no cover