1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
|
---
name: Performance Test
permissions: read-all
on:
workflow_call:
inputs:
ref_lib_artifact:
description: 'name of artifact for refrence lib'
required: true
type: string
lib_artifact:
description: 'name of artifact for lib under test'
required: true
type: string
artifact_name:
description: 'Artifact name'
required: false
type: string
jobs:
dispatcher-performance:
name: Dispatcher Performance Test
runs-on: [self-hosted, linux]
steps:
- name: Cleanup workspace
run: sudo rm -rf ..?* .[!.]* *
- name: Download lib+dev test package
uses: actions/download-artifact@v4
with:
name: ${{ inputs.lib_artifact }}
path: test_package
- name: Extract test package
run: unzip test_package/\*.zip -d _install_test
- name: Download lib+dev ref package
uses: actions/download-artifact@v4
with:
name: ${{ inputs.ref_lib_artifact }}
path: ref_package
- name: Extract ref package
run: unzip ref_package/\*.zip -d _install_ref
- name: Checkout dispatcher repo
uses: actions/checkout@v4
with:
path: source
- name: Create Report Folders
run: |
mkdir -p _report
mkdir -p _report/ref_results
mkdir -p _report/test_results
- name: Dry Run
if: false
# yamllint disable rule:line-length
run: |
mkdir -p _report/ref_results/data/artifacts/measure/perf
pushd _report/ref_results/data/artifacts/measure/perf
cat > msperf_FFMPEG_HEVC-HEVC_performance.csv <<EOL
clipname, performance_stream, fps, runtime(s), GPU_Vid0(%), GPU_Vid1(%), GPU_Render(%), AVG_CPU(%), RC6(%), GPU_Freq_Avg(MHz), MEM_RES_Avg(MB/stream), MEM_RES_Total(MB), AVG_MEM(%), PHYSICAL_MEM(MB), CPU_IPC, TOTAL_CPU(%), GPU_MEM_RES_Avg(MB/stream), GPU_MEM_RES_Total(MB)
hevc_1920x1080p_bits.h265,1,30.0,100.2,0
EOL
cat > msperf_FFMPEG_HEVC-AVC_performance.csv <<EOL
clipname, performance_stream, fps, runtime(s), GPU_Vid0(%), GPU_Vid1(%), GPU_Render(%), AVG_CPU(%), RC6(%), GPU_Freq_Avg(MHz), MEM_RES_Avg(MB/stream), MEM_RES_Total(MB), AVG_MEM(%), PHYSICAL_MEM(MB), CPU_IPC, TOTAL_CPU(%), GPU_MEM_RES_Avg(MB/stream), GPU_MEM_RES_Total(MB)
hevc_1920x1080p_bits.h265,1,30.0,100.19,0
EOL
cat > msperf_SMT_HEVC-HEVC_performance.csv <<EOL
clipname, performance_stream, fps, runtime(s), GPU_Vid0(%), GPU_Vid1(%), GPU_Render(%), AVG_CPU(%), RC6(%), GPU_Freq_Avg(MHz), MEM_RES_Avg(MB/stream), MEM_RES_Total(MB), AVG_MEM(%), PHYSICAL_MEM(MB), CPU_IPC, TOTAL_CPU(%), GPU_MEM_RES_Avg(MB/stream), GPU_MEM_RES_Total(MB)
hevc_1920x1080p_bits.h265,1,29.9,100.27,0
EOL
cat > msperf_SMT_HEVC-AVC_performance.csv <<EOL
clipname, performance_stream, fps, runtime(s), GPU_Vid0(%), GPU_Vid1(%), GPU_Render(%), AVG_CPU(%), RC6(%), GPU_Freq_Avg(MHz), MEM_RES_Avg(MB/stream), MEM_RES_Total(MB), AVG_MEM(%), PHYSICAL_MEM(MB), CPU_IPC, TOTAL_CPU(%), GPU_MEM_RES_Avg(MB/stream), GPU_MEM_RES_Total(MB)
hevc_1920x1080p_bits.h265,1,29.9,100.27,0
EOL
popd
mkdir -p _report/test_results/data/artifacts/measure/perf
pushd _report/test_results/data/artifacts/measure/perf
# HEVC-HEVC,FFMPEG => Expect: Pass -0.99% (Faster than ref)
cat > msperf_FFMPEG_HEVC-HEVC_performance.csv <<EOL
clipname, performance_stream, fps, runtime(s), GPU_Vid0(%), GPU_Vid1(%), GPU_Render(%), AVG_CPU(%), RC6(%), GPU_Freq_Avg(MHz), MEM_RES_Avg(MB/stream), MEM_RES_Total(MB), AVG_MEM(%), PHYSICAL_MEM(MB), CPU_IPC, TOTAL_CPU(%), GPU_MEM_RES_Avg(MB/stream), GPU_MEM_RES_Total(MB)
hevc_1920x1080p_bits.h265,1,30.0,99.2,0
EOL
# HEVC-AVC,FFMPEG => Expect: Pass -9.98% (Faster than ref)
cat > msperf_FFMPEG_HEVC-AVC_performance.csv <<EOL
clipname, performance_stream, fps, runtime(s), GPU_Vid0(%), GPU_Vid1(%), GPU_Render(%), AVG_CPU(%), RC6(%), GPU_Freq_Avg(MHz), MEM_RES_Avg(MB/stream), MEM_RES_Total(MB), AVG_MEM(%), PHYSICAL_MEM(MB), CPU_IPC, TOTAL_CPU(%), GPU_MEM_RES_Avg(MB/stream), GPU_MEM_RES_Total(MB)
hevc_1920x1080p_bits.h265,1,30.0,90.19,0
EOL
# HEVC-HEVC,SMT => Expect: Pass 0.83% (Slower than ref)
cat > msperf_SMT_HEVC-HEVC_performance.csv <<EOL
clipname, performance_stream, fps, runtime(s), GPU_Vid0(%), GPU_Vid1(%), GPU_Render(%), AVG_CPU(%), RC6(%), GPU_Freq_Avg(MHz), MEM_RES_Avg(MB/stream), MEM_RES_Total(MB), AVG_MEM(%), PHYSICAL_MEM(MB), CPU_IPC, TOTAL_CPU(%), GPU_MEM_RES_Avg(MB/stream), GPU_MEM_RES_Total(MB)
hevc_1920x1080p_bits.h265,1,29.9,101.27,0
EOL
# HEVC-AVC,SMT => Expect: FAIL 4.97% (Slower than ref)
cat > msperf_SMT_HEVC-AVC_performance.csv <<EOL
clipname, performance_stream, fps, runtime(s), GPU_Vid0(%), GPU_Vid1(%), GPU_Render(%), AVG_CPU(%), RC6(%), GPU_Freq_Avg(MHz), MEM_RES_Avg(MB/stream), MEM_RES_Total(MB), AVG_MEM(%), PHYSICAL_MEM(MB), CPU_IPC, TOTAL_CPU(%), GPU_MEM_RES_Avg(MB/stream), GPU_MEM_RES_Total(MB)
hevc_1920x1080p_bits.h265,1,29.9,105.26,0
EOL
popd
# yamllint enable rule:line-length
- name: Checkout media delivery repo
uses: actions/checkout@v4
with:
path: media-delivery
repository: intel/media-delivery
- name: Build media-delivery intel-gfx Docker image
run: >
docker build
-t intel-media-delivery
--file media-delivery/docker/ubuntu22.04/intel-gfx/Dockerfile
media-delivery/
- name: Build Docker image
run: |
docker build source/.github/workflows/performance \
-f source/.github/workflows/performance/Dockerfile \
--build-arg BASE_IMAGE="intel-media-delivery" \
--build-arg USER_ID=$(id -u) \
--build-arg GROUP_ID=$(id -g) \
-t vpl_performance:linux
- name: Get Reference Performance Results
run: |
docker run --rm -u root \
-v $(pwd):/work \
-w /work \
-v "$(realpath _report/ref_results):/results" \
-v "$(realpath _install_ref):/package" \
-e DEVICE=/dev/dri/renderD128 \
--device /dev/dri/renderD128 \
--group-add=$(getent group render | cut -d: -f3) \
--cap-add SYS_ADMIN \
-p 8080:8080 \
vpl_performance:linux \
/setup/performance.sh
- name: Get Test Performance Results
run: |
docker run --rm -u root \
-v $(pwd):/work \
-w /work \
-v "$(realpath _report/test_results):/results" \
-v "$(realpath _install_test):/package" \
-e DEVICE=/dev/dri/renderD128 \
--device /dev/dri/renderD128 \
--group-add=$(getent group render | cut -d: -f3) \
--cap-add SYS_ADMIN \
-p 8080:8080 \
vpl_performance:linux \
/setup/performance.sh
- name: Combine Test Results
run: |
write_perf_result() {
CODEC="$1"
TOOL="$2"
TEST_RESULT_DIR="$3"
REF_RESULT_DIR="$4"
PERF_DIR=data/artifacts/measure/perf
PERF_FILE=msperf_${TOOL}_${CODEC}_performance.csv
TEST_DATA_FILE="${TEST_RESULT_DIR}/${PERF_DIR}/${PERF_FILE}"
REF_DATA_FILE="${REF_RESULT_DIR}/${PERF_DIR}/${PERF_FILE}"
LAST_LINE="$( tail -n 1 "$TEST_DATA_FILE" )"
IFS=',' read -r -a TEST_FIELDS <<< "$LAST_LINE"
LAST_LINE="$( tail -n 1 "$REF_DATA_FILE" )"
IFS=',' read -r -a REF_FIELDS <<< "$LAST_LINE"
echo "${CODEC},${TOOL},${REF_FIELDS[3]},${TEST_FIELDS[3]}"
}
write_perf_report() {
TEST_RESULT_DIR="$1"
REF_RESULT_DIR="$2"
echo "Codec,Tool,Ref,Test"
write_perf_result HEVC-HEVC SMT \
"${TEST_RESULT_DIR}" "${REF_RESULT_DIR}"
write_perf_result HEVC-HEVC FFMPEG \
"${TEST_RESULT_DIR}" "${REF_RESULT_DIR}"
write_perf_result HEVC-AVC SMT \
"${TEST_RESULT_DIR}" "${REF_RESULT_DIR}"
write_perf_result HEVC-AVC FFMPEG \
"${TEST_RESULT_DIR}" "${REF_RESULT_DIR}"
}
write_perf_report _report/test_results _report/ref_results \
> _report/perf_numbers.csv
- name: Summarize Results
if: success() || failure()
run: |
set +e
function join {
local delim=${1-}
local list=${2-}
if shift 2; then
printf %s "$list" "${@/#/$delim}"
fi
}
function csv() {
local mode=${1-}
shift 1
printf "$(join , "$@")\n"
}
function md() {
local mode=${1-}
shift 1
printf "| $(join ' | ' "$@") |\n"
if [[ "$mode" == "head" ]]
then
local seps=()
for s in "$@"
do
local sep=""
for ((i=0; i<${#s}; i++))
do
sep="${sep}-"
done
seps=("${seps[@]}" "${sep}")
done
printf "| $(join ' | ' "${seps[@]}") |\n"
fi
}
function summarize() {
local result=0
local acceptable="$1"
local fmt="${2}"
local data_file="${3}"
local count="0"
$fmt head "Codec" "Tool" "Refrence Time(s)" \
"Test Time(s)" "Percent" "Status"
while IFS= read -r line || [[ -n "$line" ]]; do
if [[ "$count" -gt '0' ]]; then
IFS=',' read -a array <<< $line
codec=${array[0]}
tool=${array[1]}
ref_time=${array[2]}
test_time=${array[3]}
formula="100 * ($test_time - $ref_time) / $ref_time"
pct=$( echo "scale=2; $formula" |bc )
status="Passed"
if (( $(echo "${pct} > $acceptable" |bc -l) )); then
status="Failed"
result=1
fi
$fmt data "${codec}" "${tool}" "${ref_time}" \
"${test_time}" "${pct}" "${status}"
fi
count=$[ $count+1 ]
done < "$data_file"
return $result
}
# check performance difference between N and N-1
summarize 1 "csv" _report/perf_numbers.csv \
>> _report/summary.csv
summarize 1 "md" _report/perf_numbers.csv \
>> _report/summary.md
retcode=$?
cat _report/summary.md
cat _report/summary.md >> $GITHUB_STEP_SUMMARY
if [ $retcode -ne 0 ]; then
exit 1
fi
- name: Upload test results
uses: actions/upload-artifact@v4
if: success() || failure()
with:
name: ${{ inputs.artifact_name }}
path: _report
- name: Cleanup workspace
if: success() || failure()
run: sudo rm -rf ..?* .[!.]* *
|
