File: evaluate

package info (click to toggle)
stateless-openpgp-docs 13.1-1
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid, trixie
  • size: 380 kB
  • sloc: sh: 951; ansic: 279; makefile: 49; python: 36
file content (320 lines) | stat: -rwxr-xr-x 9,989 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
#!/bin/bash

if [ "$1" = --keep ]; then
    KEEP_WORKDIR=true
    shift
fi

sop_bin="$1"

declare -a errors=()
declare -A errors_by_version=()
declare -A errors_by_tag=()
declare -a successes=()

if [ -z "$sop_bin" ]; then
    printf 'usage: ./evaluate [--keep] /path/to/sop\n' >&2
    exit 1
fi

err() {
    printf "$@" >&2
    exit 1
}

sop_bin=$(realpath "$(which "$sop_bin")") || err "Could not find path\n"

if [ -v AUTOPKGTEST_ARTIFACTS ]; then
    workdir=$(mktemp -p "$AUTOPKGTEST_ARTIFACTS" -d)
    KEEP_WORKDIR=true
else
    workdir=$(mktemp -d)
fi
cd "$workdir"

cleanup() {
    unset ret
    printf '\n%d Successes\n' "${#successes[@]}" >&2
    if [ "${#errors[@]}" -gt 0 ]; then
        printf '\n%d Failure(s):\n' "${#errors[@]}" >&2
        printf ' - %s\n' "${errors[@]}" >&2
        printf '\nFailures by draft version:\n' >&2
        for v in $(sort -n <(printf '%d\n' "${!errors_by_version[@]}")); do
            printf ' - v%02d: %d\n' "$v" "${errors_by_version[$v]}" >&2
        done
        if [ "${#errors_by_tag[@]}" -gt 0 ]; then
            printf '\nFailures by tag:\n' >&2
            for tag in $(sort <(printf '%s\n' "${!errors_by_tag[@]}")); do
                printf ' - %s: %d\n' "$tag" "${errors_by_tag[$tag]}" >&2
            done
        fi
        ret=1
    fi
    if [ -z "$KEEP_WORKDIR" ]; then
        rm -rf "$workdir"
    else
        printf 'Artifacts are available in %s\n' "$workdir"
    fi
    if [ -n "$ret" ]; then
        exit "$ret"
    fi
}

trap cleanup EXIT

sop() {
    "$sop_bin" "$@"
}

t_end() {
    if ! "$@"; then
        errors+=("$(printf '%s (v%02d)' "$curtest" "$curver")")
        errors_by_version[$curver]=$(( "${errors_by_version[$curver]:-0}" + 1 ))
        for tag in "${curtags[@]}"; do
            errors_by_tag[$tag]=$(( "${errors_by_tag[$tag]:-0}" + 1 ))
        done
    else
        successes+=("$(printf '%s (v%02d)' "$curtest" "$curver")")
    fi
}

t_start() {
    curver="$1"
    curtest="$2"
    shift 2
    curtags=("$@")
    printf "=== %s (v%02d) ===\n" "$curtest" "$curver"
}

confirmsig() {
    local file="$1"
    local count="${2:-1}"
    local SIG_VERIFIER='^[0-9]{4}(-[0-9]{2}){2}T[0-9]{2}(:[0-9]{2}){2}Z( ([0-9A-Fa-f]{64}|[0-9A-Fa-f]{40})){2}( .*)?$'
    if [ $(egrep -c "$SIG_VERIFIER" "$file") -ne "$count" ] || \
           [ $(wc -l "$file" | awk '{ print $1 }') -ne "$count" ]; then
        cat "$file"
        return 1
    fi
    return 0
}

threematch() {
    diff -u "$1" "$2" && diff -u "$1" "$3"
}

non_empty_files() {
    local file
    unset ret
    for file in "$@"; do
        if ! test -s "$file"; then
            printf '%q does not exist\n' "$file"
            ret=1
        fi
    done
    if [ -n "$ret" ]; then
        return 1
    fi
    return 0
}

printf "Testing SOP implementation %s\n" "$sop_bin"

t_start 0 "Version" version

t_end sop version

t_start 3 "Extended Version" version

t_end sop version --extended

t_start 3 "Backend Version" version

t_end sop version --backend

t_start 1 "Key Generation" generate-key extract-cert

for x in alice bob; do
    sop generate-key "$x" > "$x.key"
    sop extract-cert < "$x.key" > "$x.crt"
done

t_end non_empty_files {alice,bob}.{key,crt}

printf '%s\n' test 'test test' 'test test test' > test.txt

t_start 1 "Sign/Verify roundtrip" sign verify

sop sign bob.key < test.txt > test.sig
sop verify test.sig bob.crt < test.txt > verify-detached.txt

t_end confirmsig verify-detached.txt

t_start 1 "Asymmetric Encrypt/Decrypt roundtrip" encrypt decrypt

sop encrypt bob.crt < test.txt > test.pgp
sop decrypt bob.key < test.pgp > test.txt.out

t_end diff -u test.txt test.txt.out

t_start 1 "Asymmetric Decrypt with session key" decrypt session-key

sop decrypt --session-key-out session.key bob.key < test.pgp > test.txt.skey1.out
sop decrypt --with-session-key session.key < test.pgp > test.txt.skey2.out

t_end threematch test.txt test.txt.skey1.out test.txt.skey2.out

t_start 1 "Asymmetric Encrypt/Decrypt roundtrip with signature" encrypt decrypt verify

sop encrypt --sign-with alice.key bob.crt < test.txt > test.pgp
sop decrypt --verify-with alice.crt --verify-out verify.txt bob.key < test.pgp > test.txt.sig.out

t_end confirmsig verify.txt
t_end diff -u test.txt test.txt.sig.out

t_start 1 "Asymmetric Encrypt/Decrypt roundtrip with two signatures" encrypt decrypt verify

sop encrypt --sign-with alice.key --sign-with bob.key bob.crt < test.txt > test.twosigs.pgp
sop decrypt --verify-with alice.crt --verify-with bob.crt --verify-out verify.twosigs.txt bob.key < test.twosigs.pgp > test.txt.twosigs.out

t_end confirmsig verify.twosigs.txt 2
t_end diff -u test.txt test.txt.twosigs.out

t_start 1 "Password-based Encrypt/Decrypt roundtrip" encrypt decrypt password

printf abc123 > password.txt

sop encrypt --with-password password.txt < test.txt > test.pass.pgp
sop decrypt --with-password password.txt < test.pass.pgp > test.txt.pass.out

t_end diff -u test.txt test.txt.pass.out

t_start 1 "Password-based Encrypt/Decrypt roundtrip with password (different filenames)" encrypt decrypt password

cp password.txt password2.txt

sop decrypt --with-password password2.txt < test.pass.pgp > test.txt.pass2.out

t_end diff -u test.txt test.txt.pass2.out

t_start 1 "Password-based Encrypt/Decrypt roundtrip using @ENV special designator" encrypt decrypt password @ENV

THISPASS=abc123 sop encrypt --with-password @ENV:THISPASS < test.txt > test.txt.pass.env.pgp
THISPASS=abc123 sop decrypt --with-password @ENV:THISPASS < test.txt.pass.env.pgp > test.txt.pass.env.out

t_end diff -u test.txt test.txt.pass.env.out

t_start 1 "Password-based Encrypt/Decrypt roundtrip using @ENV special designator (different varnames)" encrypt decrypt password @ENV

INPASS=abc123 sop encrypt --with-password @ENV:INPASS < test.txt > test.txt.pass.env2.pgp
OUTPASS=abc123 sop decrypt --with-password @ENV:OUTPASS < test.txt.pass.env2.pgp > test.txt.pass.env2.out

t_end diff -u test.txt test.txt.pass.env2.out

t_start 1 "Password-based Encrypt/Decrypt roundtrip using @FD special designator" encrypt decrypt password @FD

4<<<abc123 sop encrypt --with-password @FD:4 < test.txt > test.txt.pass.fd.pgp
4<<<abc123 sop decrypt --with-password @FD:4 < test.txt.pass.fd.pgp > test.txt.pass.fd.out

t_end diff -u test.txt test.txt.pass.fd.out

t_start 1 "Password-based Encrypt/Decrypt roundtrip using @FD special designator (different FDs)" encrypt decrypt password @FD

5<<<abc123 sop encrypt --with-password @FD:5 < test.txt > test.txt.pass.fd2.pgp
4<<<abc123 sop decrypt --with-password @FD:4 < test.txt.pass.fd2.pgp > test.txt.pass.fd2.out

t_end diff -u test.txt test.txt.pass.fd2.out

t_start 1 "Dearmor/Armor roundtrips" armor dearmor

pgpfiles=({alice,bob}.{key,crt} test.pgp test.pass.pgp test.sig)

err=0
for ff in "${pgpfiles[@]}"; do
    if ! sop dearmor < "$ff" > "$ff.bin"; then
        printf 'dearmoring %q failed\n' "$ff"
        err=1
    fi
    if ! sop armor < "$ff.bin" > "$ff.asc"; then
        printf 'armoring %q failed\n' "$ff.bin"
        err=1
    fi
    if ! sop dearmor < "$ff.asc" > "$ff.bin.again"; then
        printf 'dearmoring %q failed\n' "$ff.asc"
        err=1
    fi
done

test_armor_sizes() {
    local file
    local ret=$1
    shift
    for file in "${@}"; do
        if ! diff -u "$file" "$file.asc" >&2 ; then
            printf 'Warning! (variant ascii-armored forms are not necessarily failures, but they are strange)\n' >&2
        fi
        if test $(stat -c %s "$file") -le  $(stat -c %s "$file.bin"); then
            printf "%q should be larger than %q\n" "$file" "$file.bin" >&2
            ret=1
        fi
        if test $(stat -c %s "$file.bin") -ge  $(stat -c %s "$file.asc"); then
            printf "%q should be smaller than %q\n" "$file.bin" "$file.asc" >&2
            ret=1
        fi
        if ! cmp "$file.bin" "$file.bin.again"; then
            printf "dearmored form %q should be bytewise identical to %q\n" "$file.bin" "$file.bin.again" >&2
            ret=1
        fi
    done
    return "$ret"
}
t_end test_armor_sizes "$err" "${pgpfiles[@]}"

t_start 1 "Armor/Dearmor idempotency" armor dearmor

err=0

for ff in "${pgpfiles[@]}"; do
    if ! sop dearmor < "$ff.bin" > "$ff.bin2" ; then
        printf 'Failed to re-dearmor %s\n' "$ff.bin"
        err=1
    fi
    if ! sop armor < "$ff.asc" > "$ff.asc2" ; then
        printf 'Failed to re-armor %s\n' "$ff.asc"
        err=1
    fi
done

test_armor_idempotency() {
    local file
    local ret="$1"
    shift
    for file in "${@}"; do
        cmp "$file.bin" "$file.bin2" || ret=1
        diff -u "$file.asc" "$file.asc2" || ret=1
    done
    return "$ret"
}
t_end test_armor_idempotency "$err" "${pgpfiles[@]}"

# TODO: (new tests)
# - tests each substantive change in draft-02
# - tests for each substantive change in draft-03
# - tests for obscure options like --as
# - decryption with two session keys (only one of which is the correct one)
# - decryption with two passwords (only one of which is correct)
# - encryption with two passwords (both of which should decrypt)
# - symmetric encryption with signature verification
# - passwords with trailing whitespace
#  - if encrypted without trailing whitespace, and decrypted with trailing
#  - if encrypted with trailing, and decrypted without trailing
# - joint symmetric and asymmetric encryption
#  - can we encrypt to both symmetric and asymmetric at once?
#  - can we decrypt such a message with either?
#  - for a message encrypted only with a password, can we decrypt while also offering a key (that will be unused)?
#  - for a message encrypted only to a certificate, can we decrypt while also offering a password (that will be unused)?
# - signature verification date window (faketime?)
# - distinct error codes
# - multiple certs (separate files) encrypt detach-verify
# - multiple certs (single file) encrypt verify detach-verify
# - multiple keys (separate files) decrypt detach-sign
# - multiple keys (single file) decrypt sign detach-sign