File: blake3_hash_support.mdwn

package info (click to toggle)
git-annex 10.20250416-2
  • links: PTS, VCS
  • area: main
  • in suites: trixie
  • size: 73,572 kB
  • sloc: haskell: 90,656; javascript: 9,103; sh: 1,469; makefile: 211; perl: 137; ansic: 44
file content (377 lines) | stat: -rw-r--r-- 13,252 bytes parent folder | download | duplicates (4)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
This is a patch that seems to work for my personal use.
BLAKE3 does support variable lengths, but my code does not implement support for anything other than 256-bit (32-byte) digests.
I'm not familiar enough with the codebase to be sure whether adding variable length support later is a backwards compatibility hazard or not.

[[!format patch """
From a661bf0ba954b86909fa2f6b58727294068ac082 Mon Sep 17 00:00:00 2001
From: edef <edef@edef.eu>
Date: Fri, 2 Dec 2022 12:16:44 +0000
Subject: [PATCH v3 1/2] support BLAKE3

This uses the blake3 package from Hackage, since cryptonite does not
have BLAKE3 support yet.
---
 Backend/Hash.hs | 29 +++++++++++++++++++++++++++++
 Types/Key.hs    |  6 ++++++
 git-annex.cabal |  1 +
 stack.yaml      |  1 +
 4 files changed, 37 insertions(+)

diff --git a/Backend/Hash.hs b/Backend/Hash.hs
index 550d8fc6c..809a82599 100644
--- a/Backend/Hash.hs
+++ b/Backend/Hash.hs
@@ -27,8 +27,11 @@ import qualified Data.ByteString as S
 import qualified Data.ByteString.Short as S (toShort, fromShort)
 import qualified Data.ByteString.Char8 as S8
 import qualified Data.ByteString.Lazy as L
+import Data.IORef
+import Control.Arrow
 import Control.DeepSeq
 import Control.Exception (evaluate)
+import qualified BLAKE3
 
 data Hash
 	= MD5Hash
@@ -40,6 +43,7 @@ data Hash
 	| Blake2bpHash HashSize
 	| Blake2sHash HashSize
 	| Blake2spHash HashSize
+	| Blake3Hash
 
 cryptographicallySecure :: Hash -> Bool
 cryptographicallySecure (SHA2Hash _) = True
@@ -49,6 +53,7 @@ cryptographicallySecure (Blake2bHash _) = True
 cryptographicallySecure (Blake2bpHash _) = True
 cryptographicallySecure (Blake2sHash _) = True
 cryptographicallySecure (Blake2spHash _) = True
+cryptographicallySecure Blake3Hash = True
 cryptographicallySecure SHA1Hash = False
 cryptographicallySecure MD5Hash = False
 
@@ -63,6 +68,7 @@ hashes = concat
 	, map (Blake2bpHash . HashSize) [512]
 	, map (Blake2sHash . HashSize) [256, 160, 224]
 	, map (Blake2spHash . HashSize) [256, 224]
+	, [Blake3Hash]
 	, [SHA1Hash]
 	, [MD5Hash]
 	]
@@ -99,6 +105,7 @@ hashKeyVariety (Blake2bHash size) he = Blake2bKey size he
 hashKeyVariety (Blake2bpHash size) he = Blake2bpKey size he
 hashKeyVariety (Blake2sHash size) he = Blake2sKey size he
 hashKeyVariety (Blake2spHash size) he = Blake2spKey size he
+hashKeyVariety Blake3Hash he = Blake3Key he
 
 {- A key is a hash of its contents. -}
 keyValue :: Hash -> KeySource -> MeterUpdate -> Annex Key
@@ -219,6 +226,7 @@ hasher (Blake2bHash hashsize) = blake2bHasher hashsize
 hasher (Blake2bpHash hashsize) = blake2bpHasher hashsize
 hasher (Blake2sHash hashsize) = blake2sHasher hashsize
 hasher (Blake2spHash hashsize) = blake2spHasher hashsize
+hasher Blake3Hash = blake3Hasher
 
 mkHasher :: HashAlgorithm h => (L.ByteString -> Digest h) -> Context h -> Hasher
 mkHasher h c = (show . h, mkIncrementalVerifier c descChecksum . sameCheckSum)
@@ -272,6 +280,27 @@ blake2spHasher (HashSize hashsize)
 	| hashsize == 224 = mkHasher blake2sp_224 blake2sp_224_context
 	| otherwise = error $ "unsupported BLAKE2SP size " ++ show hashsize
 
+blake3Hasher :: Hasher
+blake3Hasher = (hash, incremental) where
+	finalize :: BLAKE3.Hasher -> BLAKE3.Digest BLAKE3.DEFAULT_DIGEST_LEN
+	finalize = BLAKE3.finalize
+
+	hash :: L.ByteString -> String
+	hash = show . finalize . L.foldlChunks ((. pure) . BLAKE3.update) BLAKE3.hasher
+
+	incremental :: Key -> IO IncrementalVerifier
+	incremental k = do
+		v <- newIORef (Just (BLAKE3.hasher, 0))
+		return $ IncrementalVerifier
+			{ updateIncrementalVerifier = \b ->
+				modifyIORef' v . fmap $ flip BLAKE3.update [b] *** (fromIntegral (S.length b) +)
+			, finalizeIncrementalVerifier =
+				fmap (sameCheckSum k . show . finalize . fst) <$> readIORef v
+			, unableIncrementalVerifier = writeIORef v Nothing
+			, positionIncrementalVerifier = fmap snd <$> readIORef v
+			, descIncrementalVerifier = descChecksum
+			}
+
 sha1Hasher :: Hasher
 sha1Hasher = mkHasher sha1 sha1_context
 
diff --git a/Types/Key.hs b/Types/Key.hs
index 271723982..43f64a74d 100644
--- a/Types/Key.hs
+++ b/Types/Key.hs
@@ -214,6 +214,7 @@ data KeyVariety
 	| Blake2bpKey HashSize HasExt
 	| Blake2sKey HashSize HasExt
 	| Blake2spKey HashSize HasExt
+	| Blake3Key HasExt
 	| SHA1Key HasExt
 	| MD5Key HasExt
 	| WORMKey
@@ -247,6 +248,7 @@ hasExt (Blake2bKey _ (HasExt b)) = b
 hasExt (Blake2bpKey _ (HasExt b)) = b
 hasExt (Blake2sKey _ (HasExt b)) = b
 hasExt (Blake2spKey _ (HasExt b)) = b
+hasExt (Blake3Key (HasExt b)) = b
 hasExt (SHA1Key (HasExt b)) = b
 hasExt (MD5Key (HasExt b)) = b
 hasExt WORMKey = False
@@ -262,6 +264,7 @@ sameExceptExt (Blake2bKey sz1 _) (Blake2bKey sz2 _) = sz1 == sz2
 sameExceptExt (Blake2bpKey sz1 _) (Blake2bpKey sz2 _) = sz1 == sz2
 sameExceptExt (Blake2sKey sz1 _) (Blake2sKey sz2 _) = sz1 == sz2
 sameExceptExt (Blake2spKey sz1 _) (Blake2spKey sz2 _) = sz1 == sz2
+sameExceptExt (Blake3Key _) (Blake3Key _) = True
 sameExceptExt (SHA1Key _) (SHA1Key _) = True
 sameExceptExt (MD5Key _) (MD5Key _) = True
 sameExceptExt _ _ = False
@@ -275,6 +278,7 @@ formatKeyVariety v = case v of
 	Blake2bpKey sz e -> adde e (addsz sz "BLAKE2BP")
 	Blake2sKey sz e -> adde e (addsz sz "BLAKE2S")
 	Blake2spKey sz e -> adde e (addsz sz "BLAKE2SP")
+	Blake3Key e -> adde e "BLAKE3_256"
 	SHA1Key e -> adde e "SHA1"
 	MD5Key e -> adde e "MD5"
 	WORMKey -> "WORM"
@@ -337,6 +341,8 @@ parseKeyVariety "BLAKE2SP224"  = Blake2spKey (HashSize 224) (HasExt False)
 parseKeyVariety "BLAKE2SP224E" = Blake2spKey (HashSize 224) (HasExt True)
 parseKeyVariety "BLAKE2SP256"  = Blake2spKey (HashSize 256) (HasExt False)
 parseKeyVariety "BLAKE2SP256E" = Blake2spKey (HashSize 256) (HasExt True)
+parseKeyVariety "BLAKE3_256"   = Blake3Key (HasExt False)
+parseKeyVariety "BLAKE3_256E"  = Blake3Key (HasExt True)
 parseKeyVariety "SHA1"         = SHA1Key (HasExt False)
 parseKeyVariety "SHA1E"        = SHA1Key (HasExt True)
 parseKeyVariety "MD5"          = MD5Key (HasExt False)
diff --git a/git-annex.cabal b/git-annex.cabal
index eb9b68cbc..30614e156 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -362,6 +362,7 @@ Executable git-annex
    securemem,
    crypto-api,
    cryptonite (>= 0.23),
+   blake3,
    memory,
    deepseq,
    split,
diff --git a/stack.yaml b/stack.yaml
index 7dbfb657a..936ee841b 100644
--- a/stack.yaml
+++ b/stack.yaml
@@ -25,3 +25,4 @@ extra-deps:
 - base64-bytestring-1.0.0.3
 - bencode-0.6.1.1
 - http-client-0.7.9
+- blake3-0.2@sha256:d1146b9a51ccfbb0532780778b6d016a614e3d44c05d8c1923dde9a8be869045,2448
"""]]

[[!format patch """
From 9385299c5d9d7862ec2768ef57cab1fb7f59695d Mon Sep 17 00:00:00 2001
From: edef <edef@edef.eu>
Date: Fri, 16 Dec 2022 14:37:53 +0000
Subject: [PATCH v3 2/2] make BLAKE3 support optional

This sticks it behind a build flag, since not all distros ship the
blake3 package.
---
 Backend/Hash.hs | 19 +++++++++++++++++--
 Types/Key.hs    | 11 +++++++++++
 git-annex.cabal |  9 ++++++++-
 3 files changed, 36 insertions(+), 3 deletions(-)

diff --git a/Backend/Hash.hs b/Backend/Hash.hs
index 809a82599..4e9a75725 100644
--- a/Backend/Hash.hs
+++ b/Backend/Hash.hs
@@ -5,6 +5,7 @@
  - Licensed under the GNU AGPL version 3 or higher.
  -}
 
+{-# LANGUAGE CPP #-}
 {-# LANGUAGE OverloadedStrings #-}
 
 module Backend.Hash (
@@ -27,11 +28,13 @@ import qualified Data.ByteString as S
 import qualified Data.ByteString.Short as S (toShort, fromShort)
 import qualified Data.ByteString.Char8 as S8
 import qualified Data.ByteString.Lazy as L
-import Data.IORef
-import Control.Arrow
 import Control.DeepSeq
 import Control.Exception (evaluate)
+#ifdef WITH_BLAKE3
+import Data.IORef
+import Control.Arrow
 import qualified BLAKE3
+#endif
 
 data Hash
 	= MD5Hash
@@ -43,7 +46,9 @@ data Hash
 	| Blake2bpHash HashSize
 	| Blake2sHash HashSize
 	| Blake2spHash HashSize
+#ifdef WITH_BLAKE3
 	| Blake3Hash
+#endif
 
 cryptographicallySecure :: Hash -> Bool
 cryptographicallySecure (SHA2Hash _) = True
@@ -53,7 +58,9 @@ cryptographicallySecure (Blake2bHash _) = True
 cryptographicallySecure (Blake2bpHash _) = True
 cryptographicallySecure (Blake2sHash _) = True
 cryptographicallySecure (Blake2spHash _) = True
+#ifdef WITH_BLAKE3
 cryptographicallySecure Blake3Hash = True
+#endif
 cryptographicallySecure SHA1Hash = False
 cryptographicallySecure MD5Hash = False
 
@@ -68,7 +75,9 @@ hashes = concat
 	, map (Blake2bpHash . HashSize) [512]
 	, map (Blake2sHash . HashSize) [256, 160, 224]
 	, map (Blake2spHash . HashSize) [256, 224]
+#ifdef WITH_BLAKE3
 	, [Blake3Hash]
+#endif
 	, [SHA1Hash]
 	, [MD5Hash]
 	]
@@ -105,7 +114,9 @@ hashKeyVariety (Blake2bHash size) he = Blake2bKey size he
 hashKeyVariety (Blake2bpHash size) he = Blake2bpKey size he
 hashKeyVariety (Blake2sHash size) he = Blake2sKey size he
 hashKeyVariety (Blake2spHash size) he = Blake2spKey size he
+#ifdef WITH_BLAKE3
 hashKeyVariety Blake3Hash he = Blake3Key he
+#endif
 
 {- A key is a hash of its contents. -}
 keyValue :: Hash -> KeySource -> MeterUpdate -> Annex Key
@@ -226,7 +237,9 @@ hasher (Blake2bHash hashsize) = blake2bHasher hashsize
 hasher (Blake2bpHash hashsize) = blake2bpHasher hashsize
 hasher (Blake2sHash hashsize) = blake2sHasher hashsize
 hasher (Blake2spHash hashsize) = blake2spHasher hashsize
+#ifdef WITH_BLAKE3
 hasher Blake3Hash = blake3Hasher
+#endif
 
 mkHasher :: HashAlgorithm h => (L.ByteString -> Digest h) -> Context h -> Hasher
 mkHasher h c = (show . h, mkIncrementalVerifier c descChecksum . sameCheckSum)
@@ -280,6 +293,7 @@ blake2spHasher (HashSize hashsize)
 	| hashsize == 224 = mkHasher blake2sp_224 blake2sp_224_context
 	| otherwise = error $ "unsupported BLAKE2SP size " ++ show hashsize
 
+#ifdef WITH_BLAKE3
 blake3Hasher :: Hasher
 blake3Hasher = (hash, incremental) where
 	finalize :: BLAKE3.Hasher -> BLAKE3.Digest BLAKE3.DEFAULT_DIGEST_LEN
@@ -300,6 +314,7 @@ blake3Hasher = (hash, incremental) where
 			, positionIncrementalVerifier = fmap snd <$> readIORef v
 			, descIncrementalVerifier = descChecksum
 			}
+#endif
 
 sha1Hasher :: Hasher
 sha1Hasher = mkHasher sha1 sha1_context
diff --git a/Types/Key.hs b/Types/Key.hs
index 43f64a74d..806bdc034 100644
--- a/Types/Key.hs
+++ b/Types/Key.hs
@@ -5,6 +5,7 @@
  - Licensed under the GNU AGPL version 3 or higher.
  -}
 
+{-# LANGUAGE CPP #-}
 {-# LANGUAGE OverloadedStrings, DeriveGeneric #-}
 
 module Types.Key (
@@ -214,7 +215,9 @@ data KeyVariety
 	| Blake2bpKey HashSize HasExt
 	| Blake2sKey HashSize HasExt
 	| Blake2spKey HashSize HasExt
+#ifdef WITH_BLAKE3
 	| Blake3Key HasExt
+#endif
 	| SHA1Key HasExt
 	| MD5Key HasExt
 	| WORMKey
@@ -248,7 +251,9 @@ hasExt (Blake2bKey _ (HasExt b)) = b
 hasExt (Blake2bpKey _ (HasExt b)) = b
 hasExt (Blake2sKey _ (HasExt b)) = b
 hasExt (Blake2spKey _ (HasExt b)) = b
+#ifdef WITH_BLAKE3
 hasExt (Blake3Key (HasExt b)) = b
+#endif
 hasExt (SHA1Key (HasExt b)) = b
 hasExt (MD5Key (HasExt b)) = b
 hasExt WORMKey = False
@@ -264,7 +269,9 @@ sameExceptExt (Blake2bKey sz1 _) (Blake2bKey sz2 _) = sz1 == sz2
 sameExceptExt (Blake2bpKey sz1 _) (Blake2bpKey sz2 _) = sz1 == sz2
 sameExceptExt (Blake2sKey sz1 _) (Blake2sKey sz2 _) = sz1 == sz2
 sameExceptExt (Blake2spKey sz1 _) (Blake2spKey sz2 _) = sz1 == sz2
+#ifdef WITH_BLAKE3
 sameExceptExt (Blake3Key _) (Blake3Key _) = True
+#endif
 sameExceptExt (SHA1Key _) (SHA1Key _) = True
 sameExceptExt (MD5Key _) (MD5Key _) = True
 sameExceptExt _ _ = False
@@ -278,7 +285,9 @@ formatKeyVariety v = case v of
 	Blake2bpKey sz e -> adde e (addsz sz "BLAKE2BP")
 	Blake2sKey sz e -> adde e (addsz sz "BLAKE2S")
 	Blake2spKey sz e -> adde e (addsz sz "BLAKE2SP")
+#ifdef WITH_BLAKE3
 	Blake3Key e -> adde e "BLAKE3_256"
+#endif
 	SHA1Key e -> adde e "SHA1"
 	MD5Key e -> adde e "MD5"
 	WORMKey -> "WORM"
@@ -341,8 +350,10 @@ parseKeyVariety "BLAKE2SP224"  = Blake2spKey (HashSize 224) (HasExt False)
 parseKeyVariety "BLAKE2SP224E" = Blake2spKey (HashSize 224) (HasExt True)
 parseKeyVariety "BLAKE2SP256"  = Blake2spKey (HashSize 256) (HasExt False)
 parseKeyVariety "BLAKE2SP256E" = Blake2spKey (HashSize 256) (HasExt True)
+#ifdef WITH_BLAKE3
 parseKeyVariety "BLAKE3_256"   = Blake3Key (HasExt False)
 parseKeyVariety "BLAKE3_256E"  = Blake3Key (HasExt True)
+#endif
 parseKeyVariety "SHA1"         = SHA1Key (HasExt False)
 parseKeyVariety "SHA1E"        = SHA1Key (HasExt True)
 parseKeyVariety "MD5"          = MD5Key (HasExt False)
diff --git a/git-annex.cabal b/git-annex.cabal
index 30614e156..0778948fb 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -288,6 +288,10 @@ Flag GitLfs
   Description: Build with git-lfs library (rather than vendored copy)
   Default: True
 
+Flag Blake3
+  Description: Enable blake3 support
+  Default: True
+
 source-repository head
   type: git
   location: git://git-annex.branchable.com/
@@ -362,7 +366,6 @@ Executable git-annex
    securemem,
    crypto-api,
    cryptonite (>= 0.23),
-   blake3,
    memory,
    deepseq,
    split,
@@ -572,6 +575,10 @@ Executable git-annex
       CPP-Options: -DWITH_DBUS -DWITH_DESKTOP_NOTIFY -DWITH_DBUS_NOTIFICATIONS
       Other-Modules: Utility.DBus
 
+  if flag(Blake3)
+    Build-Depends: blake3
+    CPP-Options: -DWITH_BLAKE3
+
   if flag(Pairing)
     Build-Depends: network-multicast, network-info
     CPP-Options: -DWITH_PAIRING
"""]]