1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
|
<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Notes for backup implementation — Cyrus IMAP 3.6.1 documentation</title>
<link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
<link rel="stylesheet" href="../../../_static/graphviz.css" type="text/css" />
<link rel="stylesheet" href="../../../_static/cyrus.css" type="text/css" />
<link rel="index" title="Index"
href="../../../genindex.html"/>
<link rel="search" title="Search" href="../../../search.html"/>
<link rel="top" title="Cyrus IMAP 3.6.1 documentation" href="../../../index.html"/>
<link rel="up" title="Developer Thoughts & Notes" href="../thoughts.html"/>
<link rel="next" title="Cyrus IMAP Server: Sieve Bytecode" href="bytecode.html"/>
<link rel="prev" title="Developer Thoughts & Notes" href="../thoughts.html"/>
</head>
<body class="wy-body-for-nav" role="document">
<div class="pageheader">
<ul>
<li><a href="../../../index.html">Home</a></li>
<li><a href="../../../download.html">Download</a></li>
<li><a href="../../../contribute.html">Contribute</a></li>
<li><a href="../../../support.html">Support</a></li>
<li><a href="http://www.cyrusimap.org/sasl">Cyrus SASL</a></li>
</ul>
<div>
<a href="../../../index.html">
<h1>Cyrus IMAP</h1>
<!-- <img src="../../../_static/logo.gif" alt="CYRUS" /> -->
</a>
</div>
</div>
<div style="clear: both;"></div>
<div class="wy-grid-for-nav">
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
<div class="wy-side-nav-search">
<a href="../../../index.html">
<h1>Cyrus IMAP</h1>
</a>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
<input type="text" name="q" placeholder="Search docs" />
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
</div>
</div>
<div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
<p class="caption"><span class="caption-text">Cyrus IMAP</span></p>
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../../../download.html">Download</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../../download/getcyrus.html">Get Cyrus</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../../download/getcyrus.html#distribution-package">Distribution Package</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../download/installation/distributions/centos.html">CentOS</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../download/installation/distributions/debian.html">Debian</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../download/installation/distributions/fedora.html">Fedora</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../download/installation/distributions/opensuse.html">openSUSE</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../download/installation/distributions/rhel.html">Red Hat Enterprise Linux</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../download/installation/distributions/ubuntu.html">Ubuntu</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../download/getcyrus.html#build-and-install-yourself">Build and Install Yourself</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../download/getcyrus.html#use-a-release-packaged-tarball">Use a release packaged tarball</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../download/getcyrus.html#use-the-source-from-git">Use the source from Git</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../download/getcyrus.html#external-tools">External Tools</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../download/getcyrus.html#licensing">Licensing</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../download/release-notes/index.html">Release Notes</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../../download/release-notes/index.html#stable-version">Stable Version</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../download/release-notes/index.html#development-version">Development Version</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../download/release-notes/index.html#supported-product-series">Supported Product Series</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../download/release-notes/index.html#series-3-6">Series 3.6</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../download/release-notes/index.html#series-3-4">Series 3.4</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../download/release-notes/index.html#series-3-2">Series 3.2</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../download/release-notes/index.html#series-3-0">Series 3.0</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../download/release-notes/index.html#series-2-5">Series 2.5</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../download/release-notes/index.html#development-snapshots">Development snapshots</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../download/release-notes/3.7/index.html">Cyrus IMAP 3.7 Tags</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../download/release-notes/3.5/index.html">Cyrus IMAP 3.5 Tags</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../download/release-notes/3.3/index.html">Cyrus IMAP 3.3 Tags</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../download/release-notes/3.1/index.html">Cyrus IMAP 3.1 Tags</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../download/release-notes/index.html#older-versions">Older Versions</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../download/release-notes/index.html#series-1">Series 1</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../download/release-notes/index.html#series-2-2-0-2-4">Series 2: 2.0 - 2.4</a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../download/packagers.html">Notes for Packagers</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../../download/packagers.html#binary-naming">Binary naming</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../download/packagers.html#sample-configuration-files">Sample configuration files</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../download/packagers.html#predefined-configurations">Predefined configurations</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../download/packagers.html#the-configuration-file-for-master-cyrus-conf">The configuration file for master: cyrus.conf</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../download/packagers.html#the-configuration-file-for-the-various-programs-imapd-conf">The configuration file for the various programs: imapd.conf</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../download/packagers.html#services-in-etc-services">Services in <code class="docutils literal notranslate"><span class="pre">/etc/services</span></code></a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../../quickstart.html">Quickstart Guide</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../../quickstart/introduction.html">Introduction to Cyrus IMAP</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../../quickstart/introduction.html#what-is-imap">What is IMAP?</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../../quickstart.html#quick-install">Quick install</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../../../quickstart.html#install-cyrus-package-s">1. Install Cyrus package(s)</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../quickstart.html#setup-the-cyrus-mail-user-and-group">2. Setup the cyrus:mail user and group</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../quickstart.html#setting-up-authentication-with-sasl">3. Setting up authentication with SASL</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../quickstart.html#setup-mail-delivery-from-your-mta">4. Setup mail delivery from your MTA</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../quickstart.html#protocol-ports">5. Protocol ports</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../quickstart.html#configuring-cyrus">6. Configuring Cyrus</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../quickstart.html#launch-cyrus">7. Launch Cyrus</a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../../overview.html">Overview</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../../concepts/features.html">Features</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../../concepts/features.html#security-and-authentication">Security and Authentication</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/features/authentication-kerberos.html">Kerberos Authentication</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/features/authentication-ldap.html">LDAP Authentication</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/features/authentication-sql.html">SQL Authentication</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/features/access-control.html">Access Control</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/features/sealed-system.html">Sealed System Design</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../concepts/features.html#mailbox-management">Mailbox Management</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/features/automatic-creation-of-mailboxes.html">Automatic Creation of Mailboxes</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/features/namespaces.html">Mailbox Namespaces</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/features/virtual-domains.html">Virtual Domains</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/features/mailbox-annotations.html">Mailbox Annotations (METADATA)</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/features/mailbox-distribution.html">Mailbox Distribution</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../concepts/features.html#message-management">Message Management</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/features/delayed-delete.html">Delayed Delete</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/features/delayed-expunge.html">Delayed Expunge</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/features/message-annotations.html">Message Annotations (METADATA)</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/features/duplicate-message-delivery-suppression.html">Duplicate Message Delivery Suppression</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/features/shared-seen-state.html">Shared Seen State</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/features/server-side-filtering.html">Server Side Filtering (Sieve)</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/features/event-notifications.html">Event Notifications</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../concepts/features.html#calendar-and-contact-dav-collection-management">Calendar and Contact (DAV) Collection Management</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/features/caldav-collections.html">CalDAV Collections</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/features/dav-components.html">DAV Components</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/features/dav-collection-mgmt.html">DAV Collection Management</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/features/carddav.html">CardDAV Support</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../concepts/features.html#storage">Storage</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/features/mail-spool-partitions.html">Mail Spool Partitions</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/features/mailbox-metadata-partitions.html">Mailbox Metadata Partitions</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/features/archiving.html">Archiving</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/features/quota.html">Quota</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/features/single-instance-store.html">Single-Instance Store</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../concepts/features.html#load-management">Load Management</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/features/server-aggregation.html">Cyrus IMAP Murder (Server Aggregation)</a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../concepts/overview_and_concepts.html">Concepts</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../../concepts/overview_and_concepts.html#access-control-lists">Access Control Lists</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/overview_and_concepts.html#working-with-acls">Working with ACLs</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/overview_and_concepts.html#sample-acl">Sample ACL</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/overview_and_concepts.html#access-rights">Access Rights</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/overview_and_concepts.html#access-control-defaults">Access Control Defaults</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/overview_and_concepts.html#access-control-identifier-aci">Access Control Identifier (ACI)</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/overview_and_concepts.html#negative-rights">Negative Rights</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/overview_and_concepts.html#calculating-a-users-rights">Calculating a Users’ Rights</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../concepts/overview_and_concepts.html#login-authentication">Login Authentication</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/overview_and_concepts.html#anonymous-login">Anonymous Login</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/overview_and_concepts.html#plaintext-authentication">Plaintext Authentication</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/overview_and_concepts.html#kerberos-logins">Kerberos Logins</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/overview_and_concepts.html#shared-secrets-logins">Shared Secrets Logins</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../concepts/overview_and_concepts.html#quotas">Quotas</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/overview_and_concepts.html#working-with-quotas">Working with Quotas</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/overview_and_concepts.html#monitor-and-repair">Monitor and Repair</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/overview_and_concepts.html#supported-quota-types">Supported Quota Types</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/overview_and_concepts.html#quota-roots">Quota Roots</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/overview_and_concepts.html#controlling-quota-behavior">Controlling Quota Behavior</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/overview_and_concepts.html#mail-delivery-behavior">Mail Delivery Behavior</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/overview_and_concepts.html#quota-warnings-upon-select-when-user-has-d-rights">Quota Warnings Upon Select When User Has <code class="docutils literal notranslate"><span class="pre">d</span></code> Rights</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/overview_and_concepts.html#quotas-and-partitions">Quotas and Partitions</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../concepts/overview_and_concepts.html#new-mail-notification">New Mail Notification</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../concepts/overview_and_concepts.html#partitions">Partitions</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/overview_and_concepts.html#specifying-partitions-with-create">Specifying Partitions with “create”</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/overview_and_concepts.html#changing-partitions-with-rename">Changing Partitions with “rename”</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../concepts/overview_and_concepts.html#news">News</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../concepts/overview_and_concepts.html#pop3-server">POP3 Server</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../concepts/overview_and_concepts.html#the-syslog-facility">The syslog facility</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../concepts/overview_and_concepts.html#mail-directory-recovery">Mail Directory Recovery</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/overview_and_concepts.html#reconstructing-mailbox-directories">Reconstructing Mailbox Directories</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/overview_and_concepts.html#reconstructing-the-mailboxes-file">Reconstructing the Mailboxes File</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/overview_and_concepts.html#reconstructing-quota-roots">Reconstructing Quota Roots</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/overview_and_concepts.html#removing-quota-roots">Removing Quota Roots</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/overview_and_concepts.html#subscriptions">Subscriptions</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../concepts/overview_and_concepts.html#configuration-directory">Configuration Directory</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/overview_and_concepts.html#log-directory">Log Directory</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/overview_and_concepts.html#proc-directory">Proc Directory</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../concepts/overview_and_concepts.html#message-delivery">Message Delivery</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/overview_and_concepts.html#local-mail-transfer-protocol-lmtp">Local Mail Transfer Protocol (lmtp)</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/overview_and_concepts.html#single-instance-store">Single Instance Store</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/overview_and_concepts.html#duplicate-delivery-suppression">Duplicate Delivery Suppression</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../concepts/overview_and_concepts.html#sieve-a-mail-filtering-language">Sieve, a Mail Filtering Language</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../concepts/overview_and_concepts.html#cyrus-murder-the-imap-aggregator">Cyrus Murder, the IMAP Aggregator</a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../../setup.html">Setup</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../compiling.html">Compiling</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../compiling.html#setting-up-dependencies">Setting up dependencies</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../compiling.html#required-build-dependencies">Required Build Dependencies</a></li>
<li class="toctree-l4"><a class="reference internal" href="../compiling.html#build-dependencies-for-additional-functionality">Build dependencies for additional functionality</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../compiling.html#compile-cyrus">Compile Cyrus</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../compiling.html#default-build-mail-only">Default build: mail only</a></li>
<li class="toctree-l4"><a class="reference internal" href="../compiling.html#optional-dependencies">Optional dependencies</a></li>
<li class="toctree-l4"><a class="reference internal" href="../compiling.html#compile">Compile</a></li>
<li class="toctree-l4"><a class="reference internal" href="../compiling.html#check">Check</a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../installing.html">Installing Cyrus</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../../installing.html#install-cyrus">Install Cyrus</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../installing.html#optional-components">Optional Components</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../download/installation/manage-dav.html">HTTP modules</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../download/installation/virus.html">Virus Scanner</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../installing.html#setting-up-syslog">Setting up syslog</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../installing.html#create-cyrus-environment">Create Cyrus environment</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../installing.html#set-up-the-cyrus-mail-user-and-group">Set up the cyrus:mail user and group</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../installing.html#authentication-with-sasl">Authentication with SASL</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../installing.html#mail-delivery-from-your-mta">Mail delivery from your MTA</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../installing.html#protocol-ports">Protocol ports</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../installing.html#cyrus-config-files">Cyrus config files</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../installing.html#optional-setting-up-tls-certificates">Optional: Setting up TLS certificates</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../installing.html#prepare-ephemeral-run-time-storage-directories">Prepare ephemeral (run-time) storage directories</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../installing.html#launch-cyrus">Launch Cyrus</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../installing.html#send-a-test-email">Send a test email</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../installing.html#checking-carddav-and-caldav">Checking CardDAV and CalDAV</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../installing.html#troubleshooting">Troubleshooting</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../download/upgrade.html">Upgrading to 3.6</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../../download/upgrade.html#preparation">1. Preparation</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../download/upgrade.html#versions-to-upgrade-from">Versions to upgrade from</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../download/upgrade.html#installation-from-tarball">Installation from tarball</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../download/upgrade.html#storage-changes">Storage changes</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../download/upgrade.html#how-are-you-planning-on-upgrading">How are you planning on upgrading?</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../download/upgrade.html#do-what-as-who">Do What As Who?</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../download/upgrade.html#install-new-3-6-cyrus">2. Install new 3.6 Cyrus</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../download/upgrade.html#shut-down-existing-cyrus">3. Shut down existing Cyrus</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../download/upgrade.html#backup-and-copy-existing-data">4. Backup and Copy existing data</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../download/upgrade.html#copy-config-files-and-update">5. Copy config files and update</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../download/upgrade.html#upgrade-specific-items">6. Upgrade specific items</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../download/upgrade.html#start-new-3-6-cyrus-and-verify">7. Start new 3.6 Cyrus and verify</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../download/upgrade.html#reconstruct-databases-and-cache">8. Reconstruct databases and cache</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../download/upgrade.html#do-you-want-any-new-features">9. Do you want any new features?</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../download/upgrade.html#upgrade-complete">10. Upgrade complete</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../download/upgrade.html#special-note-for-murder-configurations">Special note for Murder configurations</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../concepts/deployment.html">Configuration Guide</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../../concepts/deployment/deployment_scenarios.html">Deployment Scenarios</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/deployment/deployment_scenarios.html#single-server-deployments">Single Server Deployments</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/deployment/deployment_scenarios.html#multi-server-deployments">Multi Server Deployments</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../concepts/deployment/deployment_scenarios.html#cyrus-murder-server-aggregation">Cyrus Murder: Server aggregation</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/deployment/deployment_scenarios.html#the-discrete-murder">The Discrete Murder</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/deployment/deployment_scenarios.html#the-unified-murder">The Unified Murder</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/deployment/deployment_scenarios.html#the-shared-murder">The Shared Murder</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../concepts/deployment/deployment_scenarios.html#cyrus-replication">Cyrus Replication</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../concepts/deployment/deployment_scenarios.html#hosted-environments">Hosted Environments</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../concepts/deployment/databases.html">Databases</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/deployment/databases.html#overview">Overview</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/deployment/databases.html#file-list">File list</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/deployment/databases.html#storage-types">Storage types</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../concepts/deployment/mailbox_creation_distribution.html">Mailbox Creation Distribution</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/deployment/mailbox_creation_distribution.html#selection-mode">Selection Mode</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/deployment/mailbox_creation_distribution.html#special-cases">Special cases</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/deployment/mailbox_creation_distribution.html#partitions-exclusion">Partitions Exclusion</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/deployment/mailbox_creation_distribution.html#partitions-usage-data-reset">Partitions Usage Data Reset</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/deployment/mailbox_creation_distribution.html#mailbox-creation-distribution-through-murder-frontend">Mailbox Creation Distribution Through <code class="docutils literal notranslate"><span class="pre">murder</span> <span class="pre">frontend</span></code></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/deployment/mailbox_creation_distribution.html#backends-exclusion">Backends Exclusion</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/deployment/mailbox_creation_distribution.html#backends-usage-data-reset">Backends Usage Data Reset</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../concepts/deployment/known_protocol_limitations.html">Known Protocol Limitations</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/deployment/known_protocol_limitations.html#pop3-and-mailbox-locking">POP3 and Mailbox Locking</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/deployment/known_protocol_limitations.html#cyrus-imap-pop3-implementation">Cyrus IMAP POP3 Implementation</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../concepts/deployment/authentication_and_authorization.html">Authentication and Authorization</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/deployment/authentication_and_authorization.html#client-authentication">Client Authentication</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/deployment/authentication_and_authorization.html#users-and-mailboxes">Users and Mailboxes</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../concepts/deployment/performance_recommendations.html">Performance Recommendations</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/deployment/performance_recommendations.html#databases-on-temporary-filesystems">Databases on Temporary Filesystems</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/deployment/performance_recommendations.html#certificates">Certificates</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../concepts/deployment/storage.html">Storage Considerations</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/deployment/storage.html#general-notes-on-storage">General Notes on Storage</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/deployment/storage.html#redundancy">Redundancy</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/deployment/storage.html#availability">Availability</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/deployment/storage.html#performance">Performance</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/deployment/storage.html#scalability">Scalability</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/deployment/storage.html#capacity">Capacity</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/deployment/storage.html#cost">Cost</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../concepts/deployment/supported-platforms.html">Supported Platforms and System Requirements</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/deployment/supported-platforms.html#building-cyrus-imap">Building Cyrus IMAP</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/deployment/supported-platforms.html#required-software-components">Required Software Components</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/deployment/supported-platforms.html#recommended-software-components">Recommended Software Components</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../concepts/deployment/supported-platforms.html#recommended-software-components-enabled-by-default">Recommended Software Components Enabled by Default</a></li>
</ul>
</li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../../operations.html">Operations</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../../reference/manpages/index.html">Man pages</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../../reference/manpages/index.html#configuration-files">(5) Configuration Files</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/configs/cyrus.conf.html"><strong>cyrus.conf</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/configs/imapd.conf.html"><strong>imapd.conf</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/configs/krb.equiv.html"><strong>krb.equiv</strong></a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../reference/manpages/index.html#system-commands">(8) System Commands</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/arbitron.html"><strong>arbitron</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/backupd.html"><strong>backupd</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/chk_cyrus.html"><strong>chk_cyrus</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/ctl_backups.html"><strong>ctl_backups</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/ctl_conversationsdb.html"><strong>ctl_conversationsdb</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/ctl_cyrusdb.html"><strong>ctl_cyrusdb</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/ctl_deliver.html"><strong>ctl_deliver</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/ctl_mboxlist.html"><strong>ctl_mboxlist</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/ctl_zoneinfo.html"><strong>ctl_zoneinfo</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/cvt_cyrusdb.html"><strong>cvt_cyrusdb</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/cvt_xlist_specialuse.html"><strong>cvt_xlist_specialuse</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/cyr_backup.html"><strong>cyr_backup</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/cyr_buildinfo.html"><strong>cyr_buildinfo</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/cyr_dbtool.html"><strong>cyr_dbtool</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/cyr_deny.html"><strong>cyr_deny</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/cyr_df.html"><strong>cyr_df</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/cyr_expire.html"><strong>cyr_expire</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/cyr_info.html"><strong>cyr_info</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/cyr_ls.html"><strong>cyr_ls</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/cyr_synclog.html"><strong>cyr_synclog</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/cyr_userseen.html"><strong>cyr_userseen</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/cyr_virusscan.html"><strong>cyr_virusscan</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/cyradm.html"><strong>cyradm</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/cyrdump.html"><strong>cyrdump</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/deliver.html"><strong>deliver</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/fetchnews.html"><strong>fetchnews</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/fud.html"><strong>fud</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/httpd.html"><strong>httpd</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/idled.html"><strong>idled</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/imapd.html"><strong>imapd</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/ipurge.html"><strong>ipurge</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/lmtpd.html"><strong>lmtpd</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/lmtpproxyd.html"><strong>lmtpproxyd</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/masssievec.html"><strong>masssievec</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/master.html"><strong>master</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/mbexamine.html"><strong>mbexamine</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/mbpath.html"><strong>mbpath</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/mbtool.html"><strong>mbtool</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/mkimap.html"><strong>mkimap</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/mknewsgroups.html"><strong>mknewsgroups</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/mupdate.html"><strong>mupdate</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/nntpd.html"><strong>nntpd</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/notifyd.html"><strong>notifyd</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/pop3d.html"><strong>pop3d</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/pop3proxyd.html"><strong>pop3proxyd</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/promstatsd.html"><strong>promstatsd</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/proxyd.html"><strong>proxyd</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/ptdump.html"><strong>ptdump</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/ptexpire.html"><strong>ptexpire</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/ptloader.html"><strong>ptloader</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/quota.html"><strong>quota</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/reconstruct.html"><strong>reconstruct</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/rehash.html"><strong>rehash</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/relocate_by_id.html"><strong>relocate_by_id</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/restore.html"><strong>restore</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/sievec.html"><strong>sievec</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/sieved.html"><strong>sieved</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/smmapd.html"><strong>smmapd</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/squatter.html"><strong>squatter</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/sync_client.html"><strong>sync_client</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/sync_reset.html"><strong>sync_reset</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/sync_server.html"><strong>sync_server</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/timsieved.html"><strong>timsieved</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/tls_prune.html"><strong>tls_prune</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/translatesieve.html"><strong>translatesieve</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/systemcommands/unexpunge.html"><strong>unexpunge</strong></a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../reference/manpages/index.html#user-commands">(1) User Commands</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/usercommands/arbitronsort.pl.html"><strong>arbitronsort.pl</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/usercommands/dav_reconstruct.html"><strong>dav_reconstruct</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/usercommands/httptest.html"><strong>httptest</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/usercommands/imtest.html"><strong>imtest</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/usercommands/installsieve.html"><strong>installsieve</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/usercommands/lmtptest.html"><strong>lmtptest</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/usercommands/mupdatetest.html"><strong>mupdatetest</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/usercommands/nntptest.html"><strong>nntptest</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/usercommands/pop3test.html"><strong>pop3test</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/usercommands/sieveshell.html"><strong>sieveshell</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/usercommands/sivtest.html"><strong>sivtest</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/usercommands/smtptest.html"><strong>smtptest</strong></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/manpages/usercommands/synctest.html"><strong>synctest</strong></a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../reference/admin.html">Administrator Guide</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../../reference/admin.html#architecture">Architecture</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../reference/architecture.html">System Architecture</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../reference/admin.html#management">Management</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../reference/admin/locations.html">File & Directory Locations</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/admin/ports-sockets.html">Ports and Sockets</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/admin/access-control.html">Access Control</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/admin/quotas.html">Quotas</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/admin/sieve.html">Cyrus Sieve</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/admin/backups.html">Cyrus Backups</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/admin/nntp.html">Cyrus NNTP</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/admin/protlayer.html">Cyrus Prot Layer</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/admin/sop.html">Standard Operating Procedures</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/admin/eventsource.html">Cyrus Event Source</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/admin/monitoring.html">Monitoring</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/admin/config-mailboxdistribution.html">Mailbox Distribution</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/admin/murder/murder.html">Cyrus Murder</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/admin/nginx-proxy.html">HOWTO: Using an NGINX IMAP Proxy</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/admin/tweaking.html">Tweaking Cyrus IMAP</a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../reference/faq.html">Frequently Asked Questions</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../../reference/faq.html#features">Features</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../reference/faqs/feature-database-backend.html">Which database backend should I use for which databases?</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/faqs/feature-duplicate-delivery.html">Duplicate Delivery Suppression</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../reference/faq.html#installation-problems">Installation Problems</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../reference/faqs/install-compilationerrors.html">Compilation errors about kssl.h and krb5.h on Red Hat Linux/Fedora</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/faqs/install-install-help.html">Help! There must be an easier way to get all this going…</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/faqs/install-linkerwarnings.html">OpenSSL Version Mismatches</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../reference/faq.html#common-feature-requests">Common Feature Requests</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../reference/faqs/r-murder-ha.html">Does the Cyrus Murder support High Availability configurations?</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/faqs/r-pop3-logging.html">Can I configure pop3d to log amount and size of messages fetched by user?</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/faqs/r-publicssharedfolders.html">How can I make CyrusSieve work with public shared folders?</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/faqs/r-subfolders.html">Can I have subfolders not appear under INBOX?</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../reference/faq.html#common-operational-questions">Common Operational Questions</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../reference/faqs/o-acls.html">How do I view ACLs on a mailbox?</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/faqs/o-annotations.html">What annotations are available?</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/faqs/o-configdir-tempfs.html">Is it safe to put <configdirectory>/proc and <configdirectory>/lock on a tmpfs filesystem?</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/faqs/o-coredump.html">How to enable core dumps</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/faqs/o-delete-mailbox.html">Why can I not delete a mailbox as an admin user?</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/faqs/o-deleted-expired-expunged-purged.html">When is What … Deleted, Expired, Expunged or Purged?</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/faqs/o-deliverdb-size.html">Why is deliver.db so large?</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/faqs/o-freezes.html">I have multiple imapd-SERVICES configured and experience occasional freezes when I try to log in!</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/faqs/o-gdb.html">How to run gdb on Cyrus components</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/faqs/o-mailbox-doesnotexist.html">Cyrus delivers claims that the mailbox does not exist</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/faqs/o-mixedcase.html">Why is mail being rejected with No Mailbox found due to MiXed CaSe incoming e-mail?</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/faqs/o-otherdatabases.html">Can I use MySQL (or another SQL database) as the primary mail store?</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/faqs/o-pop3slow.html">Why do POP3 connections take so long, but once the connection is established all is well?</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/faqs/o-reconstruct.html">Why does reconstruct -m not work?</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/faqs/o-sharedfilesystem-gpfs.html">Shared File Systems GPFS for high availability</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/faqs/o-telemetry.html">How to enable telemetry</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/faqs/o-toomanyprocesses.html">The process count keeps growing!</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/faqs/o-unable-join-environment.html">“unable to join environment” error</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/faqs/o-vacation-mailfrom.html">Why does Cyrus set the MAIL FROM address of the sender of vacation responses to ‘<>’?</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../reference/faq.html#common-interoperability-problems">Common Interoperability Problems</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../reference/faqs/interop-8bit.html">Why does Cyrus reject 8-bit characters in the headers of my messages?</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/faqs/interop-barenewlines.html">Why does Cyrus reject messages with “bare newlines”?</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/faqs/interop-sieve-exim.html">How do I get Cyrus Sieve to play nice with Exim?</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../reference/faqs/interop-slow-delivery.html">Why does mail delivery go slow or hang sometimes?</a></li>
</ul>
</li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l1 current"><a class="reference internal" href="../../../developers.html">Developers</a><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="../../../contribute.html">We need your help</a></li>
<li class="toctree-l2"><a class="reference internal" href="../documentation.html">Contribute docs</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../documentation.html#overview">Overview</a></li>
<li class="toctree-l3"><a class="reference internal" href="../documentation.html#documentation-tools">Documentation Tools</a></li>
<li class="toctree-l3"><a class="reference internal" href="../documentation.html#building-the-files">Building the files</a></li>
<li class="toctree-l3"><a class="reference internal" href="../documentation.html#submitting-updates">Submitting updates</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../documentation.html#using-github-pull-requests">Using GitHub pull requests</a></li>
<li class="toctree-l4"><a class="reference internal" href="../documentation.html#patches-through-the-mailing-list">Patches through the mailing list</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../documentation.html#special-tags">Special Tags</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../documentation.html#rfc">rfc</a></li>
<li class="toctree-l4"><a class="reference internal" href="../documentation.html#cyrusman">cyrusman</a></li>
<li class="toctree-l4"><a class="reference internal" href="../documentation.html#imap-current-stable-version">imap_current_stable_version</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../documentation.html#conventions-man-pages">Conventions: Man Pages</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../documentation.html#synopsis">Synopsis</a></li>
<li class="toctree-l4"><a class="reference internal" href="../documentation.html#examples">Examples</a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../developer.html">Contribute code and tests</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../../developer.html#getting-started">Getting Started</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../process.html">Development Process</a></li>
<li class="toctree-l4"><a class="reference internal" href="../overview.html">Overview of Cyrus development environment</a></li>
<li class="toctree-l4"><a class="reference internal" href="../github-guide.html">GitHub guide</a></li>
<li class="toctree-l4"><a class="reference internal" href="../compiling.html">Compiling</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../installing.html">Installing Cyrus</a></li>
<li class="toctree-l4"><a class="reference internal" href="../developer-testing.html">Developer Test Environment</a></li>
<li class="toctree-l4"><a class="reference internal" href="../jmap.html">JMAP support</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../developer.html#system-files-and-databases">System files and Databases</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../namespaces.html">Namespaces: a developer view</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../developer.html#resources">Resources</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../libraries.html">Developer Libraries</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../developer.html#releasing">Releasing</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../releasing.html">Releasing Cyrus IMAP</a></li>
<li class="toctree-l4"><a class="reference internal" href="../ancient-releasing.html">Releasing new builds of ancient Cyrus IMAP versions</a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../cyrusworks.html">Cyrus.Works</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../cyrusworks.html#about-cyrus-works">About Cyrus Works</a></li>
<li class="toctree-l3"><a class="reference internal" href="../cyrusworks.html#how-it-works">How it works</a></li>
</ul>
</li>
<li class="toctree-l2 current"><a class="reference internal" href="../../../developers.html#cyrus-internals">Cyrus Internals</a><ul class="current">
<li class="toctree-l3"><a class="reference internal" href="../API.html">Cyrus APIs</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../API/cyrusdb.html">CyrusDB API</a></li>
<li class="toctree-l4"><a class="reference internal" href="../API/cyrusdb2.html">cyrusdb API</a></li>
<li class="toctree-l4"><a class="reference internal" href="../API/index-api.html">Index API</a></li>
<li class="toctree-l4"><a class="reference internal" href="../API/mailbox-api.html">Mailbox API</a></li>
</ul>
</li>
<li class="toctree-l3 current"><a class="reference internal" href="../thoughts.html">Thoughts & Notes</a><ul class="current">
<li class="toctree-l4 current"><a class="current reference internal" href="#">Notes for backup implementation</a></li>
<li class="toctree-l4"><a class="reference internal" href="bytecode.html">Cyrus IMAP Server: Sieve Bytecode</a></li>
<li class="toctree-l4"><a class="reference internal" href="caldav_scheduling_flowchart.html">Cyrus CalDAV Scheduling Flowchart</a></li>
<li class="toctree-l4"><a class="reference internal" href="improved_mboxlist_sort.html">Enabling improved_mboxlist_sort</a></li>
<li class="toctree-l4"><a class="reference internal" href="notes.html">Cyrus IMAP Server: Notes</a></li>
<li class="toctree-l4"><a class="reference internal" href="prot-events.html">Cyrus IMAP Server: Prot Events</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../guidance.html">Guidance for Developers</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../guidance/hacking.html">Cyrus IMAP Server: Hacking</a></li>
<li class="toctree-l4"><a class="reference internal" href="../guidance/internationalization.html">Cyrus IMAP Server: Internationalization</a></li>
<li class="toctree-l4"><a class="reference internal" href="../guidance/locking.html">Cyrus IMAP Server: Locking</a></li>
<li class="toctree-l4"><a class="reference internal" href="../guidance/mailbox-format.html">Cyrus IMAP Server: Mailbox File Formats</a></li>
<li class="toctree-l4"><a class="reference internal" href="../guidance/namelocks.html">Cyrus IMAP Server: Namelocks</a></li>
<li class="toctree-l4"><a class="reference internal" href="../guidance/prot.html">Cyrus IMAP Server: prot layer</a></li>
<li class="toctree-l4"><a class="reference internal" href="../guidance/replication_examples.html">Cyrus IMAP Server: Replication Examples</a></li>
<li class="toctree-l4"><a class="reference internal" href="../guidance/replication_protocol.html">Cyrus IMAP Server: Replication Protocol v2.4+</a></li>
<li class="toctree-l4"><a class="reference internal" href="../guidance/special_chars.html">Cyrus IMAP Server: Special Characters</a></li>
<li class="toctree-l4"><a class="reference internal" href="../guidance/var_directory_structure.html">Cyrus IMAP Server: var directory structure</a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../../developers.html#unit-tests">Unit Tests</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../unit-tests.html">Unit Tests</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../unit-tests.html#table-of-contents">Table of Contents</a></li>
<li class="toctree-l4"><a class="reference internal" href="../unit-tests.html#introduction">1. Introduction</a></li>
<li class="toctree-l4"><a class="reference internal" href="../unit-tests.html#what-is-a-unit-test">2. What Is A Unit Test?</a></li>
<li class="toctree-l4"><a class="reference internal" href="../unit-tests.html#running-the-tests">3. Running The Tests</a></li>
<li class="toctree-l4"><a class="reference internal" href="../unit-tests.html#debugging-a-test">3.6 Debugging A Test</a></li>
<li class="toctree-l4"><a class="reference internal" href="../unit-tests.html#adding-your-own-tests">4. Adding Your Own Tests</a></li>
<li class="toctree-l4"><a class="reference internal" href="../unit-tests.html#where-to-put-your-tests">4.1 Where To Put Your Tests</a></li>
<li class="toctree-l4"><a class="reference internal" href="../unit-tests.html#adding-a-new-suite">4.1 Adding A New Suite</a></li>
<li class="toctree-l4"><a class="reference internal" href="../unit-tests.html#adding-a-test-to-a-suite">4.2 Adding A Test To A Suite</a></li>
<li class="toctree-l4"><a class="reference internal" href="../unit-tests.html#suite-setup-and-teardown">4.3 Suite Setup And Teardown</a></li>
</ul>
</li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../../support.html">Support/Community</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../../support/feedback-bugs.html">Found a bug?</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../support/feedback-mailing-lists.html">Mailing lists</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../support/feedback-meetings.html">Weekly meetings</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../overview/about_cyrus.html">About</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../../../overview/what_is_cyrus.html">What is Cyrus</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../../overview/what_is_cyrus.html#what-is-cyrus">What is Cyrus</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../../overview/what_is_cyrus.html#what-is-imap">What is IMAP?</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../../overview/what_is_cyrus.html#imap-version-4-imap4">IMAP Version 4 (IMAP4)</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../../overview/what_is_cyrus.html#mime">Mime</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../../overview/what_is_cyrus.html#smtp">SMTP</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../../overview/who_is_cyrus.html">Who Is Cyrus</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../../overview/who_is_cyrus.html#core-contributors">Core Contributors</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../../overview/who_is_cyrus.html#individual-contributors-and-past-contributors">Individual contributors and past contributors</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../../overview/cyrus_roadmap.html">Cyrus Roadmap</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../../overview/cyrus_roadmap.html#high-level-roadmap">High Level Roadmap</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../../overview/cyrus_history.html">Cyrus History</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../overview/cyrus_bylaws.html">Cyrus Bylaws</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../../overview/cyrus_bylaws.html#i-the-cyrus-governance-board">I. The Cyrus Governance Board</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../../overview/cyrus_bylaws.html#ii-the-cyrus-core-developers-group">II. The Cyrus Core Developers Group</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../../overview/cyrus_bylaws.html#iii-the-release-engineer">III. The Release Engineer</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../../overview/cyrus_bylaws.html#iv-the-cyrus-roadmap">IV. The Cyrus Roadmap</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../../overview/cyrus_bylaws.html#v-development-process">V. Development Process</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../../overview/cyrus_bylaws.html#vi-changes-to-the-bylaws">VI. Changes to the Bylaws</a></li>
</ul>
</li>
</ul>
</li>
</ul>
</li>
</ul>
<p class="caption"><span class="caption-text">Cyrus SASL</span></p>
<ul>
<li class="toctree-l1"><a class="reference external" href="http://www.cyrusimap.org/sasl">Cyrus SASL</a></li>
</ul>
</div>
<div class="buildstatus">
<a href="https://github.com/cyrusimap/cyrus-imapd/actions" target="_blank">cyrus-imapd-3.6: <img src="https://github.com/cyrusimap/cyrus-imapd/actions/workflows/main.yml/badge.svg?branch=cyrus-imapd-3.6"></a>
</div>
</nav>
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
<nav class="wy-nav-top" role="navigation" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../index.html">Cyrus IMAP</a>
</nav>
<div class="wy-nav-content">
<div class="rst-content">
<div role="navigation" aria-label="breadcrumbs navigation">
<ul class="wy-breadcrumbs">
<li><a href="../../../index.html">Docs v3.6.1</a> »</li>
<li><a href="../../../developers.html">Developers</a> »</li>
<li><a href="../thoughts.html">Developer Thoughts & Notes</a> »</li>
<li>Notes for backup implementation</li>
<li class="wy-breadcrumbs-aside">
<a href="https://github.com/cyrusimap/cyrus-imapd/blob/cyrus-imapd-3.6/docsrc/imap/developer/thoughts/backup.rst" class="fa fa-github"> Edit on GitHub</a>
</li>
</ul>
<hr/>
</div>
<div role="main" class="document">
<span class="target" id="imap-developer-thoughts-backup"></span><div class="section" id="notes-for-backup-implementation">
<h1>Notes for backup implementation<a class="headerlink" href="#notes-for-backup-implementation" title="Permalink to this headline">¶</a></h1>
<p>Backup index database (one per user):</p>
<p>chunk:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="nb">int</span> <span class="nb">id</span>
<span class="n">timestamp</span> <span class="n">ts</span>
<span class="nb">int</span> <span class="n">offset</span>
<span class="nb">int</span> <span class="n">length</span>
<span class="n">text</span> <span class="n">file_sha1</span> <span class="o">-></span> <span class="n">sha1</span> <span class="n">of</span> <span class="p">(</span><span class="n">compressed</span><span class="p">)</span> <span class="n">data</span> <span class="n">prior</span> <span class="n">to</span> <span class="n">this</span> <span class="n">chunk</span>
<span class="n">text</span> <span class="n">data_sha1</span> <span class="o">-></span> <span class="n">sha1</span> <span class="n">of</span> <span class="p">(</span><span class="n">uncompressed</span><span class="p">)</span> <span class="n">data</span> <span class="n">contained</span> <span class="ow">in</span> <span class="n">this</span> <span class="n">chunk</span>
</pre></div>
</div>
<p>mailbox:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="nb">int</span> <span class="nb">id</span>
<span class="nb">int</span> <span class="n">last_chunk_id</span> <span class="o">-></span> <span class="n">chunk</span> <span class="n">that</span> <span class="n">knows</span> <span class="n">the</span> <span class="n">current</span> <span class="n">state</span>
<span class="n">char</span> <span class="n">uniqueid</span> <span class="o">-></span> <span class="n">unique</span>
<span class="n">char</span> <span class="n">mboxname</span> <span class="o">-></span> <span class="n">altered</span> <span class="n">by</span> <span class="n">a</span> <span class="n">rename</span>
<span class="n">char</span> <span class="n">mboxtype</span>
<span class="nb">int</span> <span class="n">last_uid</span>
<span class="nb">int</span> <span class="n">highestmodseq</span>
<span class="nb">int</span> <span class="n">recentuid</span>
<span class="n">timestamp</span> <span class="n">recenttime</span>
<span class="n">timestamp</span> <span class="n">last_appenddate</span>
<span class="n">timestamp</span> <span class="n">pop3_last_login</span>
<span class="n">timestamp</span> <span class="n">pop3_show_after</span>
<span class="n">timestamp</span> <span class="n">uidvalidity</span>
<span class="n">char</span> <span class="n">partition</span>
<span class="n">char</span> <span class="n">acl</span>
<span class="n">char</span> <span class="n">options</span>
<span class="nb">int</span> <span class="n">sync_crc</span>
<span class="nb">int</span> <span class="n">sync_crc_annot</span>
<span class="n">char</span> <span class="n">quotaroot</span>
<span class="nb">int</span> <span class="n">xconvmodseq</span>
<span class="n">char</span> <span class="n">annotations</span>
<span class="n">timestamp</span> <span class="n">deleted</span> <span class="o">-></span> <span class="n">time</span> <span class="n">that</span> <span class="n">it</span> <span class="n">was</span> <span class="n">unmailbox</span><span class="s1">'d, or NULL if still alive</span>
</pre></div>
</div>
<p>message:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="nb">int</span> <span class="nb">id</span>
<span class="n">char</span> <span class="n">guid</span>
<span class="n">char</span> <span class="n">partition</span> <span class="o">-></span> <span class="n">this</span> <span class="ow">is</span> <span class="n">used</span> <span class="n">to</span> <span class="nb">set</span> <span class="n">the</span> <span class="n">spool</span> <span class="n">directory</span> <span class="k">for</span> <span class="n">the</span> <span class="n">temp</span> <span class="n">file</span> <span class="o">-</span> <span class="n">we</span> <span class="n">might</span> <span class="ow">not</span> <span class="n">need</span> <span class="n">it</span>
<span class="nb">int</span> <span class="n">chunk_id</span>
<span class="nb">int</span> <span class="n">offset</span> <span class="o">-></span> <span class="n">offset</span> <span class="n">within</span> <span class="n">chunk</span> <span class="n">of</span> <span class="n">dlist</span> <span class="n">containing</span> <span class="n">this</span> <span class="n">message</span>
<span class="nb">int</span> <span class="n">size</span> <span class="o">-></span> <span class="n">size</span> <span class="n">of</span> <span class="n">this</span> <span class="n">message</span> <span class="p">(</span><span class="n">n</span><span class="o">.</span><span class="n">b</span><span class="o">.</span> <span class="ow">not</span> <span class="n">length</span> <span class="n">of</span> <span class="n">dlist</span><span class="p">)</span>
</pre></div>
</div>
<p>mailbox_message:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="nb">int</span> <span class="n">mailbox_id</span>
<span class="nb">int</span> <span class="n">message_id</span>
<span class="nb">int</span> <span class="n">last_chunk_id</span> <span class="o">-></span> <span class="n">chunk</span> <span class="n">that</span> <span class="n">has</span> <span class="n">a</span> <span class="n">RECORD</span> <span class="ow">in</span> <span class="n">a</span> <span class="n">MAILBOX</span> <span class="k">for</span> <span class="n">this</span>
<span class="nb">int</span> <span class="n">uid</span>
<span class="nb">int</span> <span class="n">modseq</span>
<span class="n">timestamp</span> <span class="n">last_updated</span>
<span class="n">char</span> <span class="n">flags</span>
<span class="n">timestamp</span> <span class="n">internaldate</span>
<span class="nb">int</span> <span class="n">size</span>
<span class="n">char</span> <span class="n">annotations</span>
<span class="n">timestamp</span> <span class="n">expunged</span> <span class="o">-></span> <span class="n">time</span> <span class="n">that</span> <span class="n">it</span> <span class="n">was</span> <span class="n">expunged</span><span class="p">,</span> <span class="ow">or</span> <span class="n">NULL</span> <span class="k">if</span> <span class="n">still</span> <span class="n">alive</span>
</pre></div>
</div>
<p>subscription:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="nb">int</span> <span class="n">last_chunk_id</span> <span class="o">-></span> <span class="n">chunk</span> <span class="n">that</span> <span class="n">knows</span> <span class="n">the</span> <span class="n">current</span> <span class="n">state</span>
<span class="n">char</span> <span class="n">mboxname</span> <span class="o">-></span> <span class="n">no</span> <span class="n">linkage</span> <span class="n">to</span> <span class="n">mailbox</span> <span class="n">table</span><span class="p">,</span> <span class="n">users</span> <span class="n">can</span> <span class="n">be</span> <span class="n">sub</span><span class="s1">'d to nonexistent</span>
<span class="n">timestamp</span> <span class="n">unsubscribed</span> <span class="o">-></span> <span class="n">time</span> <span class="n">that</span> <span class="n">it</span> <span class="n">was</span> <span class="n">unsubscribed</span><span class="p">,</span> <span class="ow">or</span> <span class="n">NULL</span> <span class="k">if</span> <span class="n">still</span> <span class="n">alive</span>
</pre></div>
</div>
<p>seen:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="nb">int</span> <span class="n">last_chunk_id</span> <span class="o">-></span> <span class="n">chunk</span> <span class="n">that</span> <span class="n">knows</span> <span class="n">the</span> <span class="n">current</span> <span class="n">state</span>
<span class="n">char</span> <span class="n">uniqueid</span> <span class="o">-></span> <span class="n">mailbox</span> <span class="p">(</span><span class="ow">not</span> <span class="n">necessarily</span> <span class="n">ours</span><span class="p">)</span> <span class="n">this</span> <span class="n">applies</span> <span class="n">to</span>
<span class="n">timestamp</span> <span class="n">lastread</span>
<span class="nb">int</span> <span class="n">lastuid</span>
<span class="n">timestamp</span> <span class="n">lastchange</span>
<span class="n">char</span> <span class="n">seenuids</span> <span class="o">-></span> <span class="n">a</span> <span class="n">uid</span> <span class="n">sequence</span> <span class="n">encoded</span> <span class="k">as</span> <span class="n">a</span> <span class="n">string</span>
</pre></div>
</div>
<p>sieve:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="nb">int</span> <span class="n">chunk_id</span>
<span class="n">timestamp</span> <span class="n">last_update</span>
<span class="n">char</span> <span class="n">filename</span>
<span class="n">char</span> <span class="n">guid</span>
<span class="nb">int</span> <span class="n">offset</span> <span class="o">-></span> <span class="n">offset</span> <span class="n">within</span> <span class="n">chunk</span> <span class="n">of</span> <span class="n">the</span> <span class="n">dlist</span> <span class="n">containing</span> <span class="n">this</span> <span class="n">script</span>
<span class="n">timestamp</span> <span class="n">deleted</span> <span class="o">-></span> <span class="n">time</span> <span class="n">that</span> <span class="n">it</span> <span class="n">was</span> <span class="n">deleted</span><span class="p">,</span> <span class="ow">or</span> <span class="n">NULL</span> <span class="k">if</span> <span class="n">still</span> <span class="n">alive</span>
</pre></div>
</div>
<p>sieve scripts and messages are both identified by a GUID
but APPLY SIEVE doesn’t take a GUID, it seems to be generated locally?
the GUID in the response to APPLY SIEVE is generated in the process of
reading the script from disk (sync_sieve_list_generate)</p>
<p>can’t activate scripts because only bytecode files are activated, but
we neither receive bytecode files over sync protocol nor do we compile
them ourselves.</p>
<p>possibly reduce index size by breaking deleted/expunged values into their
own tables, such that we only store a deleted value for things that are
actually deleted. use left join + is null to find undeleted content</p>
<div class="section" id="messages">
<h2>messages<a class="headerlink" href="#messages" title="Permalink to this headline">¶</a></h2>
<p>APPLY MESSAGE is a list of messages, not necessarily only one message.
Actually, it’s a list of messages for potentially multiple users, but we avoid
this by rejecting GET MESSAGES requests that span multiple users (so that
sync_client retries at USER level, and so we only see APPLY MESSAGE requests
for a single user).</p>
<p>Cheap first implementation is to index the start/end of the entire APPLY
MESSAGE command identically for each message within it, and at restore time
we grab that chunk and loop over it looking for the correct guid.</p>
<p>Ideal implementation would be to index the offset and length of each message
exactly (even excluding the dlist wrapper), but this is rather complicated
by the dlist API.</p>
<p>For now, we just index the offset of the dlist entry for the message,
and we can parse the pure message data back out later from that, when
we need to. Slightly less efficient on reads, but works->good->fast. We
need to loop over the entries in the MESSAGE dlist to find the one with the
desired GUID.</p>
<p>The indexed length needs to be the length of the message, not the length of the
dlist wrapper, because we need to know this cheaply to supply RECORDs in
MAILBOX responses.</p>
</div>
<div class="section" id="renames">
<h2>renames<a class="headerlink" href="#renames" title="Permalink to this headline">¶</a></h2>
<p>APPLY RENAME %(OLDMBOXNAME old NEWMBOXNAME new PARTITION p UIDVALIDITY 123)</p>
<p>We identify mboxes by uniqueid, so when we start seeing sync data for the same
uniqueid with a new mboxname we just transparently update it anyway, without
needing to handle the APPLY RENAME. Not sure if this is a problem… Do we
need to record an mbox’s previous names somehow?</p>
<p>I think it’s possible to use this to rename a USER though, something like:</p>
<p>APPLY RENAME %(OLDMBOXNAME example.com!user.smithj NEWMBOXNAME example.com!user.jsmith …)</p>
<p>– in which case, without special handling of the RENAME command itself, there
will be a backup for the old user that ends with the RENAME, and a backup of
the new user that (probably) duplicates everything again (except for stuff
that’s been expunged).</p>
<p>And if someone else gets given the original name, like</p>
<p>APPLY RENAME %(OLDMBOXNAME example.com!user.samantha-mithj NEWMBOXNAME example.com!user.smithj …)</p>
<p>Then anything that was expunged from the original user but still available in
backup disappears? Or the two backups get conflated, and samantha can
“restore” the original smithj’s old mail?</p>
<p>Uggh.</p>
<p>if there’s a mailboxes database pointing to the backup files, then the backup
file names don’t need to be based on the userid, they could e.g. be based on
the user’s inbox’s uniqueid. this would make it easier to deal with user
renames because the backup filename wouldn’t need to change. but this depends
on the uniqueid(s) in question being present on most areas of the sync
protocol, otherwise when starting a backup of a brand new user we won’t be
able to tell where to store it. workaround in the meantime could be to make
some kind of backup id from the mailboxes database, and base the filename on
this.</p>
<p>actually, using “some kind of backup id from the mailboxes database” is probably
the best solution. otherwise the lock complexity of renaming a user while making
sure their new backup filename doesn’t already exist is frightful.</p>
<p>maybe do something with mkstemp()?</p>
<p>furthermore: what if a mailbox is moved from one user to another? like:</p>
<p>APPLY RENAME %(OLD… example.com!user.foo.something NEW… example.com!user.bar.something …)</p>
<p>when a different-uid rename IS a rename of a user (and not just a folder
being moved to a different user), what does it look like?
* does it do a single APPLY RENAME for the user, and expect their folders to shake out of that?
* does it do an APPLY RENAME for each of their folders?</p>
<p>in the latter case, we need to append each of those RENAMEs to the old backup
so they can take effect correctly, and THEN rename the backup file itself. but
how to tell when the appends are finished?</p>
<p>how can we tell the difference between folder(s) moved to a different user vs
user has been renamed?</p>
<p>there is a setting: ‘allowusermoves: 0’ which, when enabled, allows users to
be renamed via IMAP rename/xfer commands. but the default is that this is
disabled. we could initially require this to be disabled while using backups…</p>
<p>not sure what the workflow looks like for renaming a user if this is not enabled.</p>
<p>not sure what the sync flow looks like in either case.</p>
<p>looking at sync_apply_rename and mboxlist_renamemailbox, it seems like we’ll
see an APPLY RENAME for each affected mbox when a recursive rename is occurring.</p>
<p>there doesn’t seem to be anything preventing user/a/foo -> user/b/foo in the
general (non-INBOX) case.</p>
<p>renames might be a little easier to handle if the index replicated the mailbox
hierarchy rather than just being a flat structure. though this adds complexity
wrt hiersep handling. something like:</p>
<p>mailbox:</p>
<blockquote>
<div><dl class="simple">
<dt>mboxname</dt><dd><p># just the name of this mbox</p>
</dd>
<dt>parent_id</dt><dd><p># fk to parent mailbox</p>
</dd>
<dt>full_mboxname</dt><dd><p># cached value, parent.full_mboxname + mboxname</p>
</dd>
</dl>
</div></blockquote>
</div>
<div class="section" id="locking">
<h2>locking<a class="headerlink" href="#locking" title="Permalink to this headline">¶</a></h2>
<p>just use a normal flock/fcntl lock on the data file and only open the index
if that lock succeeded</p>
<ul class="simple">
<li><p>backup: needs to append foo and update foo.index</p></li>
<li><dl class="simple">
<dt>reindex: only needs to read foo, but needs a write lock to prevent</dt><dd><p>writes while it does so. needs to write to (replace) foo.index</p>
</dd>
</dl>
</li>
<li><p>compact: needs to re-write foo and foo.index</p></li>
<li><p>restore: needs to read</p></li>
</ul>
</div>
<div class="section" id="verifying-index">
<h2>verifying index<a class="headerlink" href="#verifying-index" title="Permalink to this headline">¶</a></h2>
<p>how to tell whether the .index file is the correct one for the backup data it
ostensibly represents?</p>
<p>one way to do this would be to have backup_index_end() store a checksum of
the corresponding data contents in the index.</p>
<p>when opening a backup, verify this checksum against the data, and refuse to
load the index if it doesn’t match.</p>
<ul class="simple">
<li><p>sha1sum of (compressed) contents of file prior to each chunk</p></li>
</ul>
<p>how to tell whether the chunk data is any good? store a checksum of the chunk
contents along with the rest of the chunk index</p>
<ul class="simple">
<li><p>sha1sum of (uncompressed) contents of each chunk</p></li>
</ul>
</div>
<div class="section" id="mailboxes-database">
<h2>mailboxes database<a class="headerlink" href="#mailboxes-database" title="Permalink to this headline">¶</a></h2>
<p>bron reckons use twoskip for this
userid -> backup_filename</p>
<p>lib/cyrusdb module implements this, look into that</p>
<p>look at conversations db code to see how to use it</p>
<p>need a tool:
* given a user, show their backup filename
* dump/undump
* rebuild based on files discovered in backup directory</p>
<p>where does this fit into the locking scheme?</p>
</div>
<div class="section" id="reindex">
<h2>reindex<a class="headerlink" href="#reindex" title="Permalink to this headline">¶</a></h2>
<ul class="simple">
<li><p>convert user mailbox name to backup name</p></li>
<li><p>complain if there’s no backup data file?</p></li>
<li><p>lock, rename .index to .index.old, init new .index</p></li>
<li><p>foreach file chunk:</p></li>
<li><p>timestamp is from first line in chunk</p></li>
<li><p>complain if timestamp has gone backwards?</p></li>
<li><p>index records from chunk</p></li>
<li><p>unlock</p></li>
<li><p>clean up .index.old</p></li>
</ul>
<p>on error:
* discard partial new index
* restore .index.old
* bail out</p>
</div>
<div class="section" id="backupd">
<h2>backupd<a class="headerlink" href="#backupd" title="Permalink to this headline">¶</a></h2>
<p>cmdloop:
* (periodic cleanup)
* read command, determine backup name
* already holding lock ? bump timestamp : obtain lock
* write data to gzname, flush immediately
* index data</p>
<p>periodic cleanup:
* check timestamp of each held lock
* if stale (define: stale?), release
* FIXME if we’ve appended more than the chunk size we would compact to, release</p>
<p>sync restart:
* release each held lock</p>
<p>exit:
* release each held lock</p>
<p>need a “backup_index_abort” to complete the backup_index_start/end set.
_start should create a transaction, _end should commit it, and _abort should
roll it back. then, if backupd fails to write to the gzip file for some
reason, the (now invalid) index info we added can be discarded too.</p>
<p>flushing immediately on write results in poor gzip compression, but for
incremental backups that’s not a problem. when the compact process hits the
file it will recompress the data more efficiently.</p>
</div>
<div class="section" id="questions">
<h2>questions<a class="headerlink" href="#questions" title="Permalink to this headline">¶</a></h2>
<ul class="simple">
<li><p>what does it look like when uidvalidity changes?</p></li>
</ul>
</div>
<div class="section" id="restore">
<h2>restore<a class="headerlink" href="#restore" title="Permalink to this headline">¶</a></h2>
<p>restoration is effectively a reverse-direction replication (replicating TO master),
which means we can’t necessarily supply things like uid, modseq, etc without racing
against normal message arrivals. so instead we add an extra command to the protocol
to restore a message to a folder but let the destination determine the tasty bits.</p>
<p>protocol flow looks something like:</p>
<p>c: APPLY RESERVE … # as usual
s: * MISSING (foo bar)
s: OK
c: APPLY MESSAGE … # as usual
s: OK
c: RESTORE MAILBOX … # new sync proto command
s: OK</p>
<p>we introduce a new command, RESTORE MAILBOX, which is similar to the existing
APPLY MAILBOX. it specifies, for a mailbox, the mailbox state plus the message
records relevant to the restore.</p>
<p>the imapd/sync_server receiving the RESTORE command creates the mailbox if necessary,
and then adds the message records to it as new records (i.e. generating new uid etc).
this will end up generating new events in the backup channel’s sync log, and then the
messages will be backed up again with their new uids, etc. additional wire transfer
of message data should be avoided by keeping the same guid.</p>
<p>if the mailbox already exists but its uniqueid does not match the one from the backup,
then what? this probably means user has deleted folder and contents, then made new
folder with same name. so it’s probably v common for mailbox uniqueid to not match
like this. so we don’t care about special handling for this case. just add any
messages that aren’t already there.</p>
<p>if the mailbox doesn’t already exist on the destination (e.g. if rebuilding a server
from backups) then it’s safe and good to reuse uidvalidity, uniqueid, uid, modseq etc,
such that connecting clients can preserve their state. so the imapd/sync_server
receiving the restore request accepts these fields as optional, but only preserves
them if it’s safe to do so.</p>
<ul class="simple">
<li><p>restore: sbin program for selecting and restoring messages</p></li>
</ul>
<p>restore command needs options:
+ whether or not to trim deletedprefix off mailbox names to be restored
+ whether or not to restore uniqueid, highestmodseq, uid and so on
+ whether or not to limit to/exclude expunged messages
+ whether or not to restore sub-mailboxes
+ sync_client-like options (servername, local_only, partition, …)
+ user/mailbox/backup file(s) to restore from
+ mailbox to restore to (override location in backup)
+ override acl?</p>
<dl>
<dt>can we heuristically determine whether an argument is an mboxname, uniqueid or guid?</dt><dd><p>=> libuuid uniqueid is 36 bytes of hyphen (at fixed positions) and hex digits
=> non-libuuid uniqueid is 24 bytes of hex digits
=> mboxname usually contains at least one . somewhere
=> guid is 40 bytes of hex digits</p>
</dd>
<dt>usage:</dt><dd><p>restore [options] server [mode] backup [mboxname | uniqueid | guid]…</p>
</dd>
<dt>options:</dt><dd><dl class="option-list">
<dt><kbd><span class="option">-A <var>acl</var></span></kbd></dt>
<dd><p># apply specified acl to restored mailboxes</p>
</dd>
<dt><kbd><span class="option">-C <var>alt_config</var></span></kbd></dt>
<dd><p># alternate config file</p>
</dd>
<dt><kbd><span class="option">-D</span></kbd></dt>
<dd><p># don’t trim deletedprefix before restoring</p>
</dd>
<dt><kbd><span class="option">-F <var>input-file</var></span></kbd></dt>
<dd><p># read mailboxes/messages from file rather than argv</p>
</dd>
<dt><kbd><span class="option">-L</span></kbd></dt>
<dd><p># local mailbox operations only (no mupdate)</p>
</dd>
<dt><kbd><span class="option">-M <var>mboxname</var></span></kbd></dt>
<dd><p># restore messages to specified mailbox</p>
</dd>
<dt><kbd><span class="option">-P <var>partition</var></span></kbd></dt>
<dd><p># restore mailboxes to specified partition</p>
</dd>
<dt><kbd><span class="option">-U</span></kbd></dt>
<dd><p># try to preserve uniqueid, uid, modseq, etc</p>
</dd>
<dt><kbd><span class="option">-X</span></kbd></dt>
<dd><p># don’t restore expunged messages</p>
</dd>
<dt><kbd><span class="option">-a</span></kbd></dt>
<dd><p># try to restore all mailboxes in backup</p>
</dd>
<dt><kbd><span class="option">-n</span></kbd></dt>
<dd><p># calculate work required but don’t perform restoration</p>
</dd>
<dt><kbd><span class="option">-r</span></kbd></dt>
<dd><p># recurse into submailboxes</p>
</dd>
<dt><kbd><span class="option">-v</span></kbd></dt>
<dd><p># verbose</p>
</dd>
<dt><kbd><span class="option">-w <var>seconds</var></span></kbd></dt>
<dd><p># wait before starting (useful for attaching a debugger)</p>
</dd>
<dt><kbd><span class="option">-x</span></kbd></dt>
<dd><p># only restore expunged messages (not sure if useful?)</p>
</dd>
<dt><kbd><span class="option">-z</span></kbd></dt>
<dd><p># require compression (abort if compression unavailable)</p>
</dd>
</dl>
</dd>
<dt>mode:</dt><dd><dl class="option-list">
<dt><kbd><span class="option">-f</span></kbd></dt>
<dd><p># specified backup interpreted as filename</p>
</dd>
<dt><kbd><span class="option">-m</span></kbd></dt>
<dd><p># specified backup interpreted as mboxname</p>
</dd>
<dt><kbd><span class="option">-u</span></kbd></dt>
<dd><p># specified backup interpreted as userid (default)</p>
</dd>
</dl>
</dd>
</dl>
</div>
<div class="section" id="compact">
<h2>compact<a class="headerlink" href="#compact" title="Permalink to this headline">¶</a></h2>
<p># finding messages that are to be kept (either exist as unexpunged somewhere,
# or exist as expunged but more recently than threshold)
# (to get unique rows, add “distinct” and remove mm.expunged from fields)
sqlite> select m.*, mm.expunged from message as m join mailbox_message as mm on m.id = mm.message_id and (mm.expunged is null or mm.expunged > 1437709300);
id|guid|partition|chunk_id|offset|length|expunged
1|1c7cca361502dfed2d918da97e506f1c1e97dfbe|default|1|458|2159|
1|1c7cca361502dfed2d918da97e506f1c1e97dfbe|default|1|458|2159|1446179047
1|1c7cca361502dfed2d918da97e506f1c1e97dfbe|default|1|458|2159|1446179047</p>
<p># finding chunks that are still needed (due to containing last state
# of mailbox or mailbox_message, or containing a message)
sqlite> select * from chunk where id in (select last_chunk_id from mailbox where deleted is null or deleted > 1437709300 union select last_chunk_id from mailbox_message where expunged is null or expunged > 1437709300 union select chunk_id from message as m join mailbox_message as mm on m.id = mm.message_id and (mm.expunged is null or mm.expunged > 1437709300));
id|timestamp|offset|length|file_sha1|data_sha1
1|1437709276|0|3397|da39a3ee5e6b4b0d3255bfef95601890afd80709|6836d0110252d08a0656c14c2d2d314124755491
3|1437709355|1977|2129|fee183c329c011ead7757f59182116500776eaaf|a5677cfa1f5f7b627763652f4bb9b99f5970748c
4|1437709425|2746|1719|3d9f02135bf964ff0b6a917921b862c3420e48f0|7b64ec321457715ee61fe238f178f5d72adaef64
5|1437709508|3589|2890|0cee599b1573110fee428f8323690cbcb9589661|90d104346ef3cba9e419461dd26045035f4cba02</p>
<p>remember: a single APPLY MESSAGE line can contain many messages!</p>
<p>thoughts:</p>
<ul>
<li><p>need a heuristic for quickly determining whether a backup needs to be compacted</p>
<blockquote>
<div><ul class="simple">
<li><p>sum(chunks to discard, chunks to combine, chunks to split) > threshold</p></li>
<li><p>can we detect chunks that are going to significantly reduce in size as result of discarding individual lines?</p></li>
</ul>
</div></blockquote>
</li>
<li><p>“quick” vs “full” compaction</p></li>
</ul>
<p>settings:</p>
<ul class="simple">
<li><p>backup retention period</p></li>
<li><p>chunk combination size (byte length or elapsed time)</p></li>
</ul>
<p>combining chunks:
* size threshold below which adjacent chunks can be joined
* size threshold above which chunks should be split
* duration threshold below which adjacent chunks can be joined
* duration threshold above which chunks should be split
backup_min_chunk_size: 0 for no minimum
backup_max_chunk_size: 0 for no maximum
backup_min_chunk_duration: 0 for no minimum
backup_max_chunk_duration: 0 for no maximum
priority: size or duration??</p>
<p>data we absolutely need to keep:</p>
<ul class="simple">
<li><p>the most recent APPLY MAILBOX for each mailbox we’re keeping (mailbox state)</p></li>
<li><p>the APPLY MAILBOX containing the most recent RECORD for each message we’re keeping (record state)</p></li>
<li><p>the APPLY MESSAGE for each message we’re keeping (message data)</p></li>
</ul>
<p>data that we should practically keep:</p>
<ul class="simple">
<li><p>all APPLY MAILBOXes for a given mailbox from the chunk identified as its last</p></li>
<li><p>all APPLY MAILBOXes containing a RECORD for a given message from the chunk identified as its last</p></li>
<li><p>the APPLY MESSAGE for each message we’re keeping</p></li>
</ul>
<p>four kinds of compaction (probably at least two simultaneously):</p>
<ul class="simple">
<li><p>removing unused chunks</p></li>
<li><p>combining adjacent chunks into a single chunk (for better gz compression)</p></li>
<li><p>removing unused message lines from within a chunk (important after combining)</p></li>
<li><p>removing unused messages from within a message line</p></li>
</ul>
<dl class="simple">
<dt>“unused messages”</dt><dd><p>messages for which all records have been expunged for longer
than the retention period</p>
</dd>
<dt>“unused chunks”</dt><dd><p>chunks which contain only unused messages</p>
</dd>
</dl>
<p>algorithm:</p>
<ul class="simple">
<li><p>open (and lock) backup and backup.new (or bail out)</p></li>
<li><p>use backup index to identify chunks we still need</p></li>
<li><p>create a chunk in backup.new</p></li>
<li><p>foreach chunk we still need:</p></li>
<li><p>foreach line in the chunk:</p></li>
<li><p>next line if we don’t need to keep it</p></li>
<li><p>create new line</p></li>
<li><p>foreach message in line:</p></li>
<li><p>if we still need the message, or if we’re not doing message granularity</p></li>
<li><p>add the message to the new line</p></li>
<li><p>write and index tmp line to backup.new</p></li>
<li><p>if the new chunk is big enough, or if we’re not combining</p></li>
<li><p>end chunk and start a new one</p></li>
<li><p>end the new chunk</p></li>
<li><p>rename backup->backup.old, backup.new->backup</p></li>
<li><p>close (and unlock) backup.old and backup</p></li>
</ul>
</div>
<div class="section" id="command-line-locking-utility">
<h2>command line locking utility<a class="headerlink" href="#command-line-locking-utility" title="Permalink to this headline">¶</a></h2>
<p>command line utility to lock a backup (for e.g. safely poking around in the
.index on a live system).</p>
<p>example failure:
$ctl_backups lock -f /path/to/backup
* Trying to obtain lock on /path/to/backup…
NO some error
<EOF></p>
<p>example success:
$ctl_backups lock -f /path/to/backup
* Trying to obtain lock on /path/to/backup…
[potentially a delay here if we need to wait for another process to release the lock]
OK locked
[waits for its stdin to close, then unlocks and exits]</p>
<p>if you need to rummage around in backup.index, run this program in another
shell, do your work, then ^D it when you’re finished.</p>
<p>you could also call this from e.g. perl over a bidirectional pipe - wait to
read “OK locked”, then you’ve got your lock. close the pipe to unlock when
you’re finished working. if you don’t read “OK locked” before the pipe closes
then something went wrong and you didn’t get the lock.</p>
<p>specify backups by -f filename, -m mailbox, -u userid
default run mode as above
-s to fork an sqlite of the index (and unlock when it exits)
-x to fork a command of your choosing (and unlock when it exits)</p>
</div>
<div class="section" id="reconstruct">
<h2>reconstruct<a class="headerlink" href="#reconstruct" title="Permalink to this headline">¶</a></h2>
<p>rebuilding backups.db from on disk files</p>
<dl class="simple">
<dt>scan each backup partition for backup files:</dt><dd><ul class="simple">
<li><p>skip timestamped files (i.e. backups from compact/reindex)</p></li>
<li><p>skip .old files (old backups from reindex)</p></li>
<li><p>.index files => skip???</p></li>
<li><p>skip unreadable files</p></li>
<li><p>skip empty files</p></li>
<li><p>skip directories etc</p></li>
</ul>
</dd>
</dl>
<p>what’s the correct procedure for repopulating a cyrus database?
keep copy of the previous (presumably broken) one?</p>
<p>trim off mkstemp suffix (if any) to find userid
can we use a recognisable character to delimit the mkstemp suffix?</p>
<p>what if there’s multiple backup files for a given userid? precedence?</p>
<p>verify found backups before recording. reindex?</p>
<p>locking? what if something has a filename and does stuff with it while
reconstruct runs?</p>
<p>backupd always uses db for opens, so as long as reconstruct keeps the db
locked while it works, the db won’t clash. but backupd might have backups
still open from before reconstruct started, which it will write to quite
happily, even though reconstruct might decide that some other file is the
correct one for that user…</p>
<p>a backup server would generally be used only for backups, and sync_client
is quite resilient when the destination isn’t there, so it’s actually
no problem to just shut down cyrus while reconstruct runs. no outage to
user-facing services, just maybe some sync backlog to catch up on once
cyrus is restarted.</p>
</div>
<div class="section" id="ctl-backups">
<h2>ctl_backups<a class="headerlink" href="#ctl-backups" title="Permalink to this headline">¶</a></h2>
<p>sbin tool for mass backup/index/database operations</p>
<dl>
<dt>needs:</dt><dd><ul class="simple">
<li><p>rebuild backups.db from disk contents</p></li>
<li><p>list backups/info</p></li>
<li><p>rename a backup</p></li>
<li><p>delete a backup</p></li>
<li><p>verify a backup (check all sha1’s, not just most recent)</p></li>
</ul>
</dd>
<dt>not sure if these should be included, or separate tools:</dt><dd><ul class="simple">
<li><p>reindex a backup (or more)</p></li>
<li><p>compact a backup (or more)</p></li>
<li><p>lock a backup</p></li>
<li><p>some sort of rolling compaction?</p></li>
</ul>
</dd>
<dt>usage:</dt><dd><p>ctl_backups [options] reconstruct # reconstruct backups.db from disk files
ctl_backups [options] list [list_opts] [[mode] backup…] # list backup info for given/all users
ctl_backups [options] move new_fname [mode] backup # rename a backup (think about this more)
ctl_backups [options] delete [mode] backup # delete a backup
ctl_backups [options] verify [mode] backup… # verify specified backups
ctl_backups [options] reindex [mode] backup… # reindex specified backups
ctl_backups [options] compact [mode] backup… # compact specified backups
ctl_backups [options] lock [lock_opts] [mode] backup # lock specified backup</p>
</dd>
<dt>options:</dt><dd><dl class="option-list">
<dt><kbd><span class="option">-C <var>alt_config</var></span></kbd></dt>
<dd><p># alternate config file</p>
</dd>
<dt><kbd><span class="option">-F</span></kbd></dt>
<dd><p># force (run command even if not needed)</p>
</dd>
<dt><kbd><span class="option">-S</span></kbd></dt>
<dd><p># stop on error</p>
</dd>
<dt><kbd><span class="option">-v</span></kbd></dt>
<dd><p># verbose</p>
</dd>
<dt><kbd><span class="option">-w</span></kbd></dt>
<dd><p># wait for locks (i.e. don’t skip locked backups)</p>
</dd>
</dl>
</dd>
<dt>mode:</dt><dd><dl class="option-list">
<dt><kbd><span class="option">-A</span></kbd></dt>
<dd><p># all known backups (not valid for single backup commands)</p>
</dd>
<dt><kbd><span class="option">-D</span></kbd></dt>
<dd><p># specified backups interpreted as domains (nvfsbc)</p>
</dd>
<dt><kbd><span class="option">-P</span></kbd></dt>
<dd><p># specified backups interpreted as userid prefixes (nvfsbc)</p>
</dd>
<dt><kbd><span class="option">-f</span></kbd></dt>
<dd><p># specified backups interpreted as filenames</p>
</dd>
<dt><kbd><span class="option">-m</span></kbd></dt>
<dd><p># specified backups interpreted as mboxnames</p>
</dd>
<dt><kbd><span class="option">-u</span></kbd></dt>
<dd><p># specified backups interpreted as userids (default)</p>
</dd>
</dl>
</dd>
<dt>lock_opts:</dt><dd><dl class="option-list">
<dt><kbd><span class="option">-c</span></kbd></dt>
<dd><p># exclusively create backup</p>
</dd>
<dt><kbd><span class="option">-s</span></kbd></dt>
<dd><p># lock backup and open index in sqlite</p>
</dd>
<dt><kbd><span class="option">-x <var>cmd</var></span></kbd></dt>
<dd><p># lock backup and execute cmd</p>
</dd>
<dt><kbd><span class="option">-p</span></kbd></dt>
<dd><p># lock backup and wait for eof on stdin (default)</p>
</dd>
</dl>
</dd>
<dt>list_opts:</dt><dd><p>-t [hours] # “stale” (no update in hours) backups only (default: 24)</p>
</dd>
</dl>
</div>
<div class="section" id="cyr-backup">
<h2>cyr_backup<a class="headerlink" href="#cyr-backup" title="Permalink to this headline">¶</a></h2>
<p>sbin tool for inspecting backups</p>
<dl class="simple">
<dt>needs:</dt><dd><ul class="simple">
<li><p>better name?</p></li>
<li><p>list stuff</p></li>
<li><p>show stuff</p></li>
<li><p>dump stuff</p></li>
<li><p>restore?</p></li>
</ul>
</dd>
</dl>
<ul class="simple">
<li><p>should lock/move/delete (single backup commands) from ctl_backups be moved here?</p></li>
</ul>
<dl>
<dt>usage:</dt><dd><p>cyr_backup [options] [mode] backup list [all | chunks | mailboxes | messages]…
cyr_backup [options] [mode] backup show chunks [id…]
cyr_backup [options] [mode] backup show messages [guid…]
cyr_backup [options] [mode] backup show mailboxes [mboxname | uniqueid]…
cyr_backup [options] [mode] backup dump [dump_opts] chunk id
cyr_backup [options] [mode] backup dump [dump_opts] message guid
cyr_backup [options] [mode] backup json [chunks | mailboxes | messages]…</p>
</dd>
<dt>options:</dt><dd><dl class="option-list">
<dt><kbd><span class="option">-C <var>alt_config</var></span></kbd></dt>
<dd><p># alternate config file</p>
</dd>
<dt><kbd><span class="option">-v</span></kbd></dt>
<dd><p># verbose</p>
</dd>
</dl>
</dd>
<dt>mode:</dt><dd><dl class="option-list">
<dt><kbd><span class="option">-f</span></kbd></dt>
<dd><p># backup interpreted as filename</p>
</dd>
<dt><kbd><span class="option">-m</span></kbd></dt>
<dd><p># backup interpreted as mboxname</p>
</dd>
<dt><kbd><span class="option">-u</span></kbd></dt>
<dd><p># backup interpreted as userid (default)</p>
</dd>
</dl>
</dd>
<dt>commands:</dt><dd><p>list: table of contents, one per line
show: indexed details of listed items, one per paragraph, detail per line
dump: relevant contents from backup stream
json: indexed details of listed items in json format</p>
</dd>
<dt>dump options:</dt><dd><dl class="option-list">
<dt><kbd><span class="option">-o <var>filename</var></span></kbd></dt>
<dd><p># dump to named file instead of stdout</p>
</dd>
</dl>
</dd>
</dl>
</div>
<div class="section" id="partitions">
<h2>partitions<a class="headerlink" href="#partitions" title="Permalink to this headline">¶</a></h2>
<p>not enough information in sync protocol to handle partitions easily?</p>
<p>we know what the partition is when we do an APPLY operation (mailbox, message,
etc), but the initial GET operations don’t include it. so we need to already
know where the appropriate backup is partitioned in order to find the backup
file in order to look inside it to respond to the GET request</p>
<p>if we have a mailboxes database (indexed by mboxname, uniqueid and userid) then
maybe that would make it feasible? if it’s not in the mailboxes database then
we don’t have a backup for it yet, so we respond accordingly, and get sent
enough information to create it.</p>
<p>does that mean the backup api needs to take an mbname on open, and it handles
the job of looking it up in the mailboxes database to find the appropriate
thing to open?</p>
<p>can we use sqlite for such a database, or is the load on it going to be too
heavy? locking? we have lots of database formats up our sleeves here, so
even though we use sqlite for the backup index there isn’t any particular
reason we’re beholden to it for the mailboxes db too</p>
<p>if we have a mailboxes db then we need a reconstruct tool for that, too</p>
<p>what if we support multiple backup partitions, but don’t expect these
to necessarily correspond with mailbox partitions. they’re just for spreading
disk usage around.</p>
<ul class="simple">
<li><p>when creating a backup for a previously-unseen user we’d pick a random
partition to put them on</p></li>
<li><p>ctl_backups would need a command to move an existing backup to a
given partition</p></li>
<li><p>ctl_backups would need a command to pre-create a user backup on a
given partition for initial distribution</p></li>
<li><p>instead of “backup_data_path” setting, have one-or-more
“backuppartition-<name>” settings, ala partition- and friends</p></li>
</ul>
<p>see imap/partlist.[ch] for partition list management stuff. it’s complicated
and doesn’t have a test suite, so maybe save this implementation until needed.</p>
<p>but… maybe rename backup_data_path to backuppartition-default in the meantime,
so that when we do add this it’s not a complicated reconfig to update?</p>
<p>partlist_local_select (and lazy-loaded partlist_local_init) are where the
mailbox partitions come from (see also mboxlist_create_partition), do something
similar for backup partitions</p>
</div>
<div class="section" id="data-corruption">
<h2>data corruption<a class="headerlink" href="#data-corruption" title="Permalink to this headline">¶</a></h2>
<dl class="simple">
<dt>backups.db:</dt><dd><ul class="simple">
<li><p>can be reconstructed from on disk files at any time</p></li>
<li><p>how to detect corruption? does cyrus_db detect/repair on its own?</p></li>
</ul>
</dd>
<dt>backup indexes:</dt><dd><ul class="simple">
<li><p>can be reindexed at any time from backup data</p></li>
<li><p>how to detect corruption? assume sqlite will notice, complain?</p></li>
</ul>
</dd>
<dt>backup data:</dt><dd><ul class="simple">
<li><p>what’s zlib’s failure mode? do we lose the entire chunk or just the corrupt bit?</p></li>
<li><p>verify will notice sha1sum mismatches</p></li>
<li><p>dlist format will reject some kinds of corruption (but not all)</p></li>
<li><p>reindex: should skip unparseable dlist lines</p></li>
<li><p>message data has its own checksums (guid)</p></li>
<li><p>reindex: should skip messages that don’t match their own checksums</p></li>
<li><p>compact: “full” compact will only keep useful data according to index</p></li>
<li><p>backupd: will sync anything that’s in user mailbox but not in backup index</p></li>
</ul>
</dd>
</dl>
<p>i think this means that if a message or mailbox state becomes corrupted in
the backup data file, and it still exists in the user’s real mailbox, you
recover from the corruption by reindexing and then letting the sync process
copy the missing data back in again. and you can tidy up the data file by
running a compact over it.</p>
<p>you detect data corruption in most recent chunk reactively as soon as the
backup system needs to open it again (quick verify on open)</p>
<p>you detect data corruption in older chunks reactively by trying to restore from
it. may be too late: if a message needs restoring it’s because user mailbox no
longer has it</p>
<p>you detect data corruption preemptively by running the verify tool over it.
recommend scheduling this in EVENTS/cron?</p>
<p>if data corruption occurs in message that’s no longer in user’s mailbox, that
message is lost. it was going to be deleted from the backup after $retention
period anyway (by compact), but if it needs restoring in the meantime, sorry</p>
</div>
<div class="section" id="installation-instructions">
<h2>installation instructions<a class="headerlink" href="#installation-instructions" title="Permalink to this headline">¶</a></h2>
<p>(obviously, most of this won’t work at this point, because the code doesn’t
exist. but this is, approximately, where things are heading.)</p>
<dl class="simple">
<dt>on your backup server:</dt><dd><ul class="simple">
<li><p>compile with –enable-backup configure option and install</p></li>
<li><dl class="simple">
<dt>imapd.conf:</dt><dd><p>backuppartition-default: /var/spool/backup # FIXME better example
backup_db: twoskip
backup_db_path: /var/imap/backups.db
backup_staging_path: /var/spool/backup
backup_retention_days: 7</p>
</dd>
</dl>
</li>
<li><dl class="simple">
<dt>cyrus.conf SERVICES:</dt><dd><p>backupd cmd=”backupd” listen=”csync” prefork=0
(remove other services, most likely)
(should i create a master/conf/backup.conf example file?)</p>
</dd>
</dl>
</li>
<li><dl class="simple">
<dt>cyrus.conf EVENTS:</dt><dd><p>compact cmd=”ctl_backups compact -A” at=0400</p>
</dd>
</dl>
</li>
<li><p>start server as usual</p></li>
<li><p>do i want a special port for backupd?</p></li>
</ul>
</dd>
<dt>on your imap server:</dt><dd><ul class="simple">
<li><dl class="simple">
<dt>imapd.conf:</dt><dd><p>sync_log_channels: backup
sync_log: 1
backup_sync_host: backup-server.example.com
backup_sync_port: csync
backup_sync_authname: …
backup_sync_password: …
backup_sync_repeat_interval: … # seconds, smaller value = livelier backups but more i/o
backup_sync_shutdown_file: ….</p>
</dd>
</dl>
</li>
<li><dl class="simple">
<dt>cyrus.conf STARTUP:</dt><dd><p>backup_sync cmd=”sync_client -r -n backup”</p>
</dd>
</dl>
</li>
<li><dl class="simple">
<dt>cyrus.conf SERVICES:</dt><dd><p>restored cmd=”restored” […]</p>
</dd>
</dl>
</li>
<li><p>start/restart master</p></li>
</ul>
</dd>
<dt>files and such:</dt><dd><p>{configdirectory}/backups.db - database mapping userids to backup locations
{backuppartition-name}/<hash>/<userid>_XXXXXX - backup data stream for userid
{backuppartition-name}/<hash>/<userid>_XXXXXX.index - index into userid’s backup data stream</p>
</dd>
<dt>do i want rhost in the path?</dt><dd><ul class="simple">
<li><p>protects from issue if multiple servers are trying to back up their own version of same user
(though this is its own problem that the backup system shouldn’t have to compensate for)</p></li>
<li><p>but makes location of undifferentiated user unpredictable</p></li>
<li><p>so probably not, actually</p></li>
</ul>
</dd>
</dl>
</div>
<div class="section" id="chatting-about-implementation-20-10">
<h2>chatting about implementation 20/10<a class="headerlink" href="#chatting-about-implementation-20-10" title="Permalink to this headline">¶</a></h2>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>09:54 @elliefm
here's a fun sync question
APPLY MESSAGE provides a list of messages
can a single APPLY MESSAGE contain messages for multiple mailboxes and/or users?
my first hunch is that it doesn't cross users, since the broadest granularity for a single sync run is USER
10:06 kmurchison
We'd have to check with Bron, but I *think* messages can cross mailboxes for a single user
10:06 @brong
yes
APPLY MESSAGE just adds it to the reserve list
10:07 @elliefm
nah apply message uploads the message, APPLY RESERVE adds it to the reserve list :P
10:07 @brong
same same
APPLY RESERVE copies it from a local mailbox
APPLY MESSAGE uploads it
10:07 @elliefm
yep
10:07 @brong
they both wind up in the reserve list
10:07 @elliefm
ahh i see what you mean, gotcha
10:07 @brong
until you send a RESTART
ideally you want it reserve in the same partition, but it will copy the message over if it's not on the same partition
there's no restriction on which mailbox it came from/went to
good for user renames, and good for an append to a bunch of mailboxes in different users / shared space all at once
(which LMTP can do)
10:10 @elliefm
i can handle the case where a single APPLY MESSAGE contains messages for multiple mailboxes belonging to the same user
but i'm in trouble if a single APPLY MESSAGE can contain messages belonging to different users
10:14 @brong
@elliefm: why?
10:14 @brong
you don't have to keep them if they aren't used
10:15 @elliefm
for backups - when i see the apply, i need to know which user's backup to add it to. that's easy enough if it doesn't cross users but gets mega fiddly if it does
i'm poking around in sync client to see if it's likely to be an issue or not
11:00 @brong_
@elliefm: I would stage it, and add it to users as it gets refcounted in by an index file
11:07 @elliefm
that's pretty much what we do for ordinary sync and delivery stuff yeah?
11:08 @brong_
yep
and it's what the backup thing does
11:09 @elliefm
i'm pretty sure that APPLY RESERVE and APPLY MESSAGE don't give a damn about users, they're just "here's every message you might not have already had since last time we spoke" and it lets the APPLY MAILBOX work out where to attach them later
11:09 @brong_
yep
11:09 @elliefm
so yeah, i'll need to do something here
i've been working so far on the idea that a single user's backup consists of 1) an append-only gzip stream of the sync protocol chat that built it, and 2) an index that tracks current state of mailboxes, and offsets within (1) of message data
that gets us good compression (file per user, not file per message), and if the index gets corrupted or lost, it's rebuildable purely from (1), it doesn't need a live copy of the original mailbox
11:12 @brong
yep, that all works
11:12 @elliefm
(so if you lose your imap server, you're not unable to rebuild a broken index on the backup)
11:13 @brong
it's easy enough to require the sync protocol stream to only contain messages per user
though "apply reserve" is messy
because you need to return "yes, I have that message"
11:13 @elliefm
with that implementation i can't (easily) keep user.a's messages from not existing in user.b's data stream (though they won't be indexed)
11:14 @brong
I'm not too adverse to the idea of just unpacking each message as it comes off the wire into a temporary directory
11:14 @elliefm
(because at the time i'm receiving the sync data i don't know which it needs to go in, so if they come in in the same reserve i'd need to append them to both data streams)
which isn't a huge problem, just… irks me a bit
11:14 @brong
and then reading the indexes as they come in, checking against the state DB to see if we already have them, and streaming them into the gzip if they aren't there yet
what we can do is something like the current format, where files go into a tar
11:16 @elliefm
i guess the fiddly bit there is that there's one more moving part to keep synchronised across failure states
a backup for a single user becomes 1) data stream + 2) any messages that were uploaded but not yet added to a mailbox + 3) index (which doesn't know what to do with (2))
which in the general case is fine, the next sync will update the mailboxes, which will push (2) into (1) and index it nicely, and on we go
but it's just a little bit more mess if there's a failure that you need to recover from between those states — it's no longer a simple case of "it's in the backup and we know everything about it" or "it doesn't exist", there's a third case of "well we might have the data but don't really know what to do with it"
the other fiddly bit is that the process of appending to the data stream is suddenly in the business of crafting output rather than simply dumping what it gets, which isn't really burdensome, but it is one more little crack for bugs to crawl into
i guess in terms of sync protocol, one thing i could do on my end is identify apply operations that seem to contain multiple users' data, and just return an error on those. the sync client on the other end will promote them until they're eventually user syncs, which i think are always user granularity
11:50 @elliefm
i think for now, first stage implementation will be to stream the reserve/message commands in full to every user backup they might apply to. and optimising that down so that each stream only contains messages belonging to that user can be a future optimisation
</pre></div>
</div>
</div>
<div class="section" id="todo-list">
<h2>todo list<a class="headerlink" href="#todo-list" title="Permalink to this headline">¶</a></h2>
<ul class="simple">
<li><p>clean up error handling</p></li>
<li><p>perl tool to anonymise sync proto talk</p></li>
<li><p>verification step to check entire data stream for errors (even chunks that aren’t indexed)</p></li>
<li><p>prot_fill_cb: extra argument to pass back an error string to prot_fill</p></li>
<li><p>ctl_backups verify: set level</p></li>
<li><p>backupd: don’t block on locked backups, return mailbox locked – but sync_client doesn’t handle this</p></li>
<li><p>test multiple backup partitions</p></li>
<li><p>configure: error if backups requested and we don’t have zlib</p></li>
<li><p>valgrind</p></li>
<li><p>finish reconstruct</p></li>
<li><p>compact: split before append?</p></li>
</ul>
<dl class="simple">
<dt>compact implementation steps:</dt><dd><p>1 remove unused chunks, keep everything else as is
2 join adjacent chunks if small enough, split large chunks
3 parse/rebuild message lines
4 discard unused mailbox lines</p>
</dd>
</dl>
</div>
</div>
</div>
<footer>
<div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
<a href="bytecode.html" class="btn btn-neutral float-right" title="Cyrus IMAP Server: Sieve Bytecode" accesskey="n">Next <span class="fa fa-arrow-circle-right"></span></a>
<a href="../thoughts.html" class="btn btn-neutral" title="Developer Thoughts & Notes" accesskey="p"><span class="fa fa-arrow-circle-left"></span> Previous</a>
</div>
<hr/>
<div role="contentinfo">
<p>
© Copyright 1993–2023, The Cyrus Team. Last updated on Mar 06 2018
</p>
</div>
Built with <a href="http://sphinx-doc.org/">Sphinx</a> 3.4.3 using a modified <a href="https://readthedocs.org">Read the Docs</a> <a href="https://github.com/snide/sphinx_rtd_theme">theme</a>.
</footer>
</div>
</div>
</section>
</div>
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../',
VERSION:'3.6.1',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true
};
</script>
<script type="text/javascript" src="../../../_static/jquery.js"></script>
<script type="text/javascript" src="../../../_static/underscore.js"></script>
<script type="text/javascript" src="../../../_static/doctools.js"></script>
<script type="text/javascript" src="https://cdn.mathjax.org/mathjax/latest/MathJax.js"></script>
<script type="text/javascript" src="../../../_static/js/theme.js"></script>
<script type="text/javascript">
<!-- jQuery(function () {
SphinxRtdTheme.StickyNav.enable();
}); -->
</script>
</body>
</html>
|
