1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
|
Last-Update: 2014-05-23
Forwarded: yes
Author: Dmitry Smirnov <onlyjob@member.fsf.org>
Description: sample.ceph.conf update:
* corrected URLs.
* added [client] section.
* more options and descriptions.
* filestore settings were moved under [osd].
* cephx settings to reflect lack of support for authentication in kernel RBD client.
* many minor corrections and updates.
--- a/src/sample.ceph.conf
+++ b/src/sample.ceph.conf
@@ -31,7 +31,7 @@
# ; Example: /var/run/ceph/$cluster-$name.asok
[global]
-### http://ceph.com/docs/master/rados/configuration/general-config-ref/
+### http://ceph.com/docs/firefly/rados/configuration/general-config-ref/
;fsid = {UUID} # use `uuidgen` to generate your own UUID
;public network = 192.168.0.0/24
@@ -51,8 +51,8 @@
;max open files = 131072
-### http://ceph.com/docs/master/rados/operations/authentication
-### http://ceph.com/docs/master/rados/configuration/auth-config-ref/
+### http://ceph.com/docs/firefly/rados/operations/authentication
+### http://ceph.com/docs/firefly/rados/configuration/auth-config-ref/
# If enabled, the Ceph Storage Cluster daemons (i.e., ceph-mon, ceph-osd,
# and ceph-mds) must authenticate with each other.
@@ -78,7 +78,11 @@
# comprising the Ceph Storage Cluster.
# Type: Boolean (optional)
# (Default: false)
- cephx require signatures = true ; everywhere possible
+ ;cephx require signatures = true
+
+ # kernel RBD client do not support authentication yet:
+ cephx cluster require signatures = true
+ cephx service require signatures = false
# The path to the keyring file.
# Type: String (optional)
@@ -86,13 +90,13 @@
;keyring = /etc/ceph/$cluster.$name.keyring
-### http://ceph.com/docs/master/rados/configuration/pool-pg-config-ref/
+### http://ceph.com/docs/firefly/rados/configuration/pool-pg-config-ref/
## Replication level, number of data copies.
# Type: 32-bit Integer
# (Default: 2)
- ;osd pool default size = 2
+ ;osd pool default size = 3
## Replication level in degraded state, less than 'osd pool default size' value.
# Sets the minimum number of written replicas for objects in the
@@ -102,7 +106,7 @@
# operating in degraded mode.
# Type: 32-bit Integer
# (Default: 0), which means no particular minimum. If 0, minimum is size - (size / 2).
- ;osd pool default min size = 1
+ ;osd pool default min size = 2
## Ensure you have a realistic number of placement groups. We recommend
## approximately 100 per OSD. E.g., total number of OSDs multiplied by 100
@@ -114,14 +118,14 @@
# default value is the same as pg_num with mkpool.
# Type: 32-bit Integer
# (Default: 8)
- ;osd pool default pg num = 100
+ ;osd pool default pg num = 128
# Description: The default number of placement groups for placement for a
# pool. The default value is the same as pgp_num with mkpool.
# PG and PGP should be equal (for now).
# Type: 32-bit Integer
# (Default: 8)
- ;osd pool default pgp num = 100
+ ;osd pool default pgp num = 128
# The default CRUSH ruleset to use when creating a pool
# Type: 32-bit Integer
@@ -135,15 +139,22 @@
;osd crush chooseleaf type = 1
-### http://ceph.com/docs/bobtail/rados/configuration/log-and-debug-ref/
+### http://ceph.com/docs/firefly/rados/troubleshooting/log-and-debug/
+ # The location of the logging file for your cluster.
+ # Type: String
+ # Required: No
# Default: /var/log/ceph/$cluster-$name.log
;log file = /var/log/ceph/$cluster-$name.log
+ # Determines if logging messages should appear in syslog.
+ # Type: Boolean
+ # Required: No
+ # (Default: false)
;log to syslog = true
-### http://ceph.com/docs/master/rados/configuration/ms-ref/
+### http://ceph.com/docs/firefly/rados/configuration/ms-ref/
# Enable if you want your daemons to bind to IPv6 address instead of
# IPv4 ones. (Not required if you specify a daemon or cluster IP.)
@@ -151,29 +162,13 @@
# (Default: false)
;ms bind ipv6 = true
-
-### http://ceph.com/docs/master/rados/configuration/filestore-config-ref/
-
- # The maximum interval in seconds for synchronizing the filestore.
- # Type: Double (optional)
- # (Default: 5)
- ;filestore max sync interval = 5
-
- # Use object map for XATTRS. Set to true for ext4 file systems only.
- # Type: Boolean (optional)
- # (Default: false)
- ;filestore xattr use omap = true
-
-### http://ceph.com/docs/master/rados/configuration/journal-ref/
-
##################
## Monitors
## You need at least one. You need at least three if you want to
## tolerate any node failures. Always create an odd number.
[mon]
-### http://ceph.com/docs/argonaut/config-ref/mon-config/
-### http://ceph.com/docs/master/rados/configuration/mon-config-ref/
-### http://ceph.com/docs/dumpling/rados/configuration/mon-osd-interaction/
+### http://ceph.com/docs/firefly/rados/configuration/mon-config-ref/
+### http://ceph.com/docs/firefly/rados/configuration/mon-osd-interaction/
# The IDs of initial monitors in a cluster during startup.
# If specified, Ceph requires an odd number of monitors to form an
@@ -185,7 +180,7 @@
;mon host = cephhost01,cephhost02
;mon addr = 192.168.0.101,192.168.0.102
- # The monitor’s data location
+ # The monitor's data location
# Default: /var/lib/ceph/mon/$cluster-$id
;mon data = /var/lib/ceph/mon/$name
@@ -197,7 +192,7 @@
# Exponential backoff for clock drift warnings
# Type: Float
# (Default: 5)
- ;mon clock drift warn backoff = 30 ; Tell the monitor to backoff from this warning for 30 seconds
+ ;mon clock drift warn backoff = 30 # Tell the monitor to backoff from this warning for 30 seconds
# The percentage of disk space used before an OSD is considered full.
# Type: Float
@@ -209,8 +204,19 @@
# (Default: .85)
;mon osd nearfull ratio = .85
+ # The number of seconds Ceph waits before marking a Ceph OSD
+ # Daemon "down" and "out" if it doesn't respond.
+ # Type: 32-bit Integer
+ # (Default: 300)
+ ;mon osd down out interval = 300
+
+ # The grace period in seconds before declaring unresponsive Ceph OSD
+ # Daemons "down".
+ # Type: 32-bit Integer
+ # (Default: 900)
+ ;mon osd report timeout = 300
-### http://ceph.com/docs/next/rados/troubleshooting/log-and-debug/
+### http://ceph.com/docs/firefly/rados/troubleshooting/log-and-debug/
# logging, for debugging monitor crashes, in order of
# their likelihood of being helpful :)
@@ -239,16 +245,28 @@
# experimental support for running multiple metadata servers. Do not run
# multiple metadata servers in production.
[mds]
-### http://ceph.com/docs/argonaut/config-ref/mds-config/
-### http://ceph.com/docs/master/cephfs/mds-config-ref/
+### http://ceph.com/docs/firefly/cephfs/mds-config-ref/
# where the mds keeps it's secret encryption keys
;keyring = /var/lib/ceph/mds/$name/keyring
+ # Determines whether a 'ceph-mds' daemon should poll and
+ # replay the log of an active MDS (hot standby).
+ # Type: Boolean
+ # (Default: false)
+ ;mds standby replay = true
+
; mds logging to debug issues.
;debug ms = 1
;debug mds = 20
+ ;debug journaler = 20
+
+ # The number of inodes to cache.
+ # Type: 32-bit Integer
+ # (Default: 100000)
+ ;mds cache size = 250000
+ ;mds mem max = 1048576 # KB
;[mds.alpha]
; host = alpha
@@ -261,8 +279,7 @@
# You need at least one. Two or more if you want data to be replicated.
# Define as many as you like.
[osd]
-### http://ceph.com/docs/argonaut/config-ref/osd-config/
-### http://ceph.com/docs/bobtail/rados/configuration/osd-config-ref/
+### http://ceph.com/docs/firefly/rados/configuration/osd-config-ref/
# The path to the OSDs data.
# You must create the directory when deploying Ceph.
@@ -282,6 +299,10 @@
# (Default: 5)
;osd recovery max active = 3
+ # The maximum number of simultaneous scrub operations for a Ceph OSD Daemon.
+ # Type: 32-bit Int
+ # (Default: 1)
+ ;osd max scrubs = 2
# You may add settings for mkcephfs so that it will create and mount
# the file system for you. Remove the comment `#` character for
@@ -303,7 +324,7 @@
## disks. You can use a file under the osd data dir if need be
## (e.g. /data/$name/journal), but it will be slower than a
## separate disk or partition.
- # The path to the OSD’s journal. This may be a path to a file or a block
+ # The path to the OSD's journal. This may be a path to a file or a block
# device (such as a partition of an SSD). If it is a file, you must
# create the directory to contain it.
# We recommend using a drive separate from the osd data drive.
@@ -311,7 +332,12 @@
# Default: /var/lib/ceph/osd/$cluster-$id/journal
;osd journal = /var/lib/ceph/osd/$name/journal
-### http://ceph.com/docs/master/rados/configuration/journal-ref/
+ # Check log files for corruption. Can be computationally expensive.
+ # Type: Boolean
+ # (Default: false)
+ ;osd check for log corruption = true
+
+### http://ceph.com/docs/firefly/rados/configuration/journal-ref/
# The size of the journal in megabytes. If this is 0,
# and the journal is a block device, the entire block device is used.
@@ -321,28 +347,47 @@
# (Default: 5120)
# Recommended: Begin with 1GB. Should be at least twice the product
# of the expected speed multiplied by "filestore max sync interval".
- ;osd journal size = 1000 ; journal size, in megabytes
+ ;osd journal size = 2048 ; journal size, in megabytes
## If you want to run the journal on a tmpfs, disable DirectIO
# Enables direct i/o to the journal.
- # Requires journal block align set to true.
+ # Requires "journal block align" set to "true".
# Type: Boolean
# Required: Yes when using aio.
# (Default: true)
;journal dio = false
- # Check log files for corruption. Can be computationally expensive.
- # Type: Boolean
- # (Default: false)
- ;osd check for log corruption = true
-
# osd logging to debug osd issues, in order of likelihood of being helpful
;debug ms = 1
;debug osd = 20
;debug filestore = 20
;debug journal = 20
+### http://ceph.com/docs/firefly/rados/configuration/filestore-config-ref/
+
+ # The maximum interval in seconds for synchronizing the filestore.
+ # Type: Double (optional)
+ # (Default: 5)
+ ;filestore max sync interval = 5
+
+ # Enable snapshots for a btrfs filestore.
+ # Type: Boolean
+ # Required: No. Only used for btrfs.
+ # (Default: true)
+ ;filestore btrfs snap = false
+ # Enables the filestore flusher.
+ # Type: Boolean
+ # Required: No
+ # (Default: false)
+ ;filestore flusher = true
+
+ # Defines the maximum number of in progress operations the file store
+ # accepts before blocking on queuing new operations.
+ # Type: Integer
+ # Required: No. Minimal impact on performance.
+ # (Default: 500)
+ ;filestore queue max ops = 500
;[osd.0]
; host = delta
@@ -362,3 +407,53 @@
;[osd.3]
; host = eta
; devs = /dev/sdy
+
+
+##################
+## client settings
+[client]
+
+### http://ceph.com/docs/firefly/rbd/rbd-config-ref/
+
+ # Enable caching for RADOS Block Device (RBD).
+ # Type: Boolean
+ # Required: No
+ # (Default: false)
+ rbd cache = true
+
+ # The RBD cache size in bytes.
+ # Type: 64-bit Integer
+ # Required: No
+ # (Default: 32 MiB)
+ ;rbd cache size = 33554432
+
+ # The dirty limit in bytes at which the cache triggers write-back.
+ # If 0, uses write-through caching.
+ # Type: 64-bit Integer
+ # Required: No
+ # Constraint: Must be less than rbd cache size.
+ # (Default: 24 MiB)
+ ;rbd cache max dirty = 25165824
+
+ # The dirty target before the cache begins writing data to the data storage.
+ # Does not block writes to the cache.
+ # Type: 64-bit Integer
+ # Required: No
+ # Constraint: Must be less than rbd cache max dirty.
+ # (Default: 16 MiB)
+ ;rbd cache target dirty = 16777216
+
+ # The number of seconds dirty data is in the cache before writeback starts.
+ # Type: Float
+ # Required: No
+ # (Default: 1.0)
+ ;rbd cache max dirty age = 1.0
+
+ # Start out in write-through mode, and switch to write-back after the
+ # first flush request is received. Enabling this is a conservative but
+ # safe setting in case VMs running on rbd are too old to send flushes,
+ # like the virtio driver in Linux before 2.6.32.
+ # Type: Boolean
+ # Required: No
+ # (Default: false)
+ ;rbd cache writethrough until flush = false
|
