File: pixel-tests

package info (click to toggle)
cockpit 337-1
  • links: PTS, VCS
  • area: main
  • in suites: trixie
  • size: 36,232 kB
  • sloc: javascript: 47,090; python: 38,766; ansic: 35,470; xml: 6,048; sh: 3,413; makefile: 614
file content (261 lines) | stat: -rwxr-xr-x 8,488 bytes parent folder | download | duplicates (2)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
#!/bin/bash

set -eu

TEST_REFERENCE_SUBDIR="${TEST_REFERENCE_SUBDIR:-test/reference}"
REPO=pixel-test-reference

GITHUB_BASE="${GITHUB_BASE:-cockpit-project/cockpit}"
GITHUB_REPOSITORY="${GITHUB_BASE%/*}/${REPO}"
CLONE_REMOTE="https://github.com/${GITHUB_REPOSITORY}"
PUSH_REMOTE="git@github.com:${GITHUB_REPOSITORY}"

message() {
    [ "${V-}" != 0 ] || printf "  %-8s %s\n" "$1" "$2"
}

cmd_init() {
    git submodule add -b empty "$CLONE_REMOTE" "$TEST_REFERENCE_SUBDIR"
}

cmd_update() {
    git submodule update --init -- "$TEST_REFERENCE_SUBDIR" || (
        echo ""
        echo "Updating test/reference has failed, maybe because of"
        echo "local changes that have been accidentally made while"
        echo "it was out of date."
        echo ""
        echo "If you want to throw away these local changes, run"
        echo ""
        echo "    $ ./test/common/pixel-tests reset"
        echo ""
        exit 1
    )
}

cmd_pull() {
    cmd_update
}

cmd_status() {
    cmd_update
    ( cd "$TEST_REFERENCE_SUBDIR"
      git rm --force --cached --quiet '*.png'
      git add *.png
      if git diff-index --name-status --cached --exit-code HEAD; then
          echo No changes
      fi
    )
}

cmd_push() {
    cmd_update
    ( cd "$TEST_REFERENCE_SUBDIR"
      git rm --force --cached --quiet '*.png'
      git add *.png
      if ! git diff-index --name-status --cached --exit-code HEAD; then
          git fetch origin empty:empty
          git reset --soft empty
          git commit --quiet -m "$(date)"
      else
          echo No changes
      fi
      tag="sha-$(git rev-parse HEAD)"
      [ $(git tag -l "$tag") ] || git tag "$tag" HEAD
      git push "$PUSH_REMOTE" "$tag"
    )
    git add "$TEST_REFERENCE_SUBDIR"
    if [ -n "$(git status --porcelain "$TEST_REFERENCE_SUBDIR")" ]; then
        echo ""
        echo "The test/reference link has changed.  The next step is to commit and"
        echo "push this change, just like any other change to a file."
        echo ""
        echo "The change has already been added with 'git add', so you could now"
        echo "amend your current HEAD commit with it like this:"
        echo ""
        echo "    $ git commit --amend"
        echo ""
        echo "Then the HEAD commit can be pushed like normally.  There is nothing"
        echo "special about committing and pushing a change to test/reference."
    fi
}

cmd_reset() {
    rm -rf "$TEST_REFERENCE_SUBDIR"
    cmd_update
}

pixel_test_logs_urls() {
    arg=${1:-}

    if [[ "$arg" == http* ]]; then
        echo $arg
        return
    fi

    repo=$(git remote get-url origin | sed -re 's,git@github.com:|https://github.com/,,' -e 's,\.git$,,')

    if [ -n "$arg" ]; then
        revision=$(curl -s "https://api.github.com/repos/$repo/pulls/$arg" | python3 -c "
import json
import sys

print(json.load(sys.stdin)['head']['sha'])
")
    else
        revision=$(git rev-parse @{upstream})
    fi

    context=$(cat test/reference-image)
    curl -s "https://api.github.com/repos/$repo/statuses/$revision?per_page=100" | python3 -c "
import json
import sys
import os

seen = set()
for s in json.load(sys.stdin):
    c = s['context']
    if 'firefox' in c or 'devel' in c:
        continue
    if c.split('/')[0] == sys.argv[1] and s['target_url']:
        url=os.path.dirname(s['target_url'])
        if url not in seen:
            seen.add(url)
            print(url)
" "$context"
}

cmd_fetch() {
    urls=$(pixel_test_logs_urls ${1:-})
    if [ -z "$urls" ]; then
        echo >&2 "Can't find test results for $(cat test/reference-image), sorry."
        exit 1
    fi
    cmd_update
    for url in ${urls}; do
        url=${url/\/log.html/}
        echo "Fetching new pixel test references from $url"
        pixels=$(curl -s "$url/index.html" | grep '[^=><"]*-pixels.png' -o | uniq)
        for f in ${pixels}; do
            echo "$f"
            curl -s --output-dir test/reference/ -O "$url/$f"
        done
    done
}

cmd_help() {
    cat <<EOF
$0 - Maintain the test/reference directory

The following commands are available:

update - Ensure that test/reference matches the checked out branch

    The test/reference directory is a git submodule, and it needs to
    be kept in sync with the main worktree (just like any other
    submodule).  This is necessary whenever you check out a different
    branch and then start working on pixel tests, for example.

    You can use "git checkout --recurse-submodules" when changing
    branches to achieve this, or some variation of "git submodule
    update test/reference", or run "pixel-tests update".

    Most other pixel-tests commands will run "pixel-tests update"
    automatically, so you might not actually need to run it explicitly
    very often, but it's good to remember that this step is
    unfortunately necessary.

    Like other ways to update submodules, this command will not
    overwrite your local changes, so it is safe to use often and "just
    in case".  If your local changes conflict with the changes that
    would be necessary for updating test/reference, the update will
    not be done.

    When "pixel-tests update" fails, it is probably easiest to throw
    away local changes with "pixel-tests reset" and re-acquire the new
    reference images that you want to install, maybe with "pixel-tests
    fetch".

pull - Old name for "update".

reset - Throw away local changes

    If you want to get rid of unpushed local changes, run "pixel-tests
    reset".  This will remove the test/reference directory completely
    and then run "pixel-tests update" to recreate it.

status - Show local changes

    After writing new reference images into test/reference, running
    "pixel-test status" will summarize the local changes.  It will
    list all images that have been modified (with a "M" prefix), added
    ("A"), or deleted ("D").

    The "status" command will run "update" as the first step, so it
    might fail if it detects conflicts.  To avoid that risk, run
    "update" explicitly before making any changes in test/reference.

push - Upload local changes and prepare test/reference for committing

    Once you have finished writing new reference images into
    test/reference (maybe by running "pixel-tests fetch") and are
    ready to make them part of your pull request, you need to upload
    them to Github and record them in the main source repository.

    First, run "pixel-tests push" to do the uploading and to stage a
    changed test/reference in the main source repository.

    In the main source reository, test/reference is a special kind of
    object.  It's not a file, or directory, and not even a symlink
    (although it is similar to a symlink).  It's a "gitlink".  But
    whatever it is, changes to it need to be staged, committed, and
    pushed, just like changes to regular files.

    When "pixel-tests push" is done, the change to test/reference has
    already been staged with "git add" to remind you that there is
    something to commit.

    Committing the change is identical to committing any other change.

    "push" will also run "update" as the first step.

fetch - Download fresh reference images from a test run

    When code or tests are changed, we often need to install new or
    changed reference images for the pixel tests.  A good way to do
    that is to run the tests in our CI machinery, let them fail, and
    grab the new reference images from the test log directory.
    Running "pixel-tests fetch" can automate this.

    Without any arguments, "fetch" will figure out all by itself where
    to download the images from.  For this to work, your "origin"
    remote needs to point to the repository on Github that the current
    PR will be merged into.  For our main Cockpit repository, that
    would be "cockpit-project/cockpit", for example.

    You can also pass a PR number to "fetch" if the current branch
    doesn't correspond to the PR with the new pixels.  Or you can pass
    the URL of the log results.

    "push" will also run "update" as the first step.
EOF
}

main() {
    local cmd="${1-}"

    if [ -z "${cmd}" ]; then
        echo 'This command requires a subcommand: update status push reset fetch'
        echo "Run '$0 help' for a longer explanation"
        exit 1
    elif ! type -t "cmd_${cmd}" | grep -q function; then
        echo "Unknown subcommand ${cmd}"
        exit 1
    fi

    shift
    [ "${V-0}" = 0 ] || set -x
    "cmd_$cmd" "$@"
}

main "$@"