File: update_pch_autotune.sh

package info (click to toggle)
libreoffice 1:6.1.5-3
  • links: PTS, VCS
  • area: main
  • in suites: buster
  • size: 2,699,584 kB
  • sloc: cpp: 4,078,558; xml: 324,485; java: 281,935; python: 48,236; ansic: 36,476; perl: 32,383; sh: 13,671; yacc: 10,820; makefile: 9,085; cs: 6,600; lex: 2,184; objc: 1,904; awk: 978; pascal: 949; asm: 866; php: 79; csh: 20; sed: 5
file content (229 lines) | stat: -rwxr-xr-x 7,272 bytes parent folder | download | duplicates (6)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
#! /bin/bash
#
# This file is part of the LibreOffice project.
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#

# Finds the optimal update_pch settings that results in,
# per module and library, the fastest build time and
# smallest intermediate files (.o/.obj) output.

# Usage: update_pch_autotune.sh [<module1> <module2>]
# Invoke: /opt/lo/bin/make cmd cmd="./bin/update_pch_autotune.sh [..]"

# The resulting values may be entered in update_pch
# to be use for generating PCH in the future.
# Run this script after major header changes.

root=`dirname $0`
root=`cd $root/.. && pwd`
cd $root

if test -z "$1"; then
    modules=`ls ./*/inc/pch/precompiled_*.hxx | sed -e s%./%% -e s%/.*%% | uniq`
else
    modules="$@"
fi

if [[ "$OSTYPE" == "cygwin" ]]; then
    MAKE=/opt/lo/bin/make
else
    MAKE=make
fi

function build()
{
    local START=$(date +%s.%N)

    $MAKE -sr "$module" > /dev/null
    status=$?
    if [ $status -ne 0 ];
    then
        # Spurious failures happen.
        $MAKE "$module.build" > /dev/null
        status=$?
    fi

    local END=$(date +%s.%N1)
    build_time=$(printf %.1f $(echo "$END - $START" | bc))

    size="FAILED"
    score="FAILED"
    if [ $status -eq 0 ];
    then
        # The total size of the object files.
        size="$(du -s workdir/CxxObject/$module/ | awk '{print $1}')"
        # Add the pch file size.
        filename_rel="workdir/PrecompiledHeader/nodebug/$(basename $header)*"
        filename_dbg="workdir/PrecompiledHeader/debug/$(basename $header)*"
        if [[ $filename_rel -nt $filename_dbg ]]; then
            pch_size="$(du -s $filename_rel | awk '{print $1}' | paste -sd+ | bc)"
        else
            pch_size="$(du -s $filename_dbg | awk '{print $1}' | paste -sd+ | bc)"
        fi
        size="$(echo "$pch_size + $size" | bc)"

        # Compute a score based on the build time and size.
        # The shorter the build time, and smaller disk usage, the higher the score.
        score=$(printf %.2f $(echo "10000 / ($build_time * e($size/1048576))" | bc -l))
    fi
}

function run()
{
    local msg="$module.$libname, ${@:3}, "
    printf "$msg"
    ./bin/update_pch "$module" "$libname" "${@:3}" --silent
    status=$?

    if [ $status -eq 0 ];
    then
        build

        summary="$build_time, $size, $score"
        if [ $status -eq 0 ];
        then
            new_best_for_cuttof=$(echo "$score > $best_score_for_cuttof" | bc -l)
            if [ $new_best_for_cuttof -eq 1 ];
            then
                best_score_for_cuttof=$score
            fi

            new_best=$(echo "$score > $best_score" | bc -l)
            if [ $new_best -eq 1 ];
            then
                best_score=$score
                best_args="${@:3}"
                best_time=$build_time
                best_cutoff=$cutoff
                summary="$build_time, $size, $score,*"
            fi
        fi
    else
        # Skip if pch is not updated.
        summary="0, 0, 0"
    fi

    echo "$summary"
}

function args_to_table()
{
    local sys="EXCLUDE"
    local mod="EXCLUDE"
    local loc="EXCLUDE"
    local cutoff=0
    IFS=' ' read -r -a aargs <<< $best_args
    for index in "${!aargs[@]}"
    do
        if [ "${aargs[index]}" = "--include:system" ];
        then
            sys="INCLUDE"
        elif [ "${aargs[index]}" = "--exclude:system" ];
        then
            sys="EXCLUDE"
        elif [ "${aargs[index]}" = "--include:module" ];
        then
            mod="INCLUDE"
        elif [ "${aargs[index]}" = "--exclude:module" ];
        then
            mod="EXCLUDE"
        elif [ "${aargs[index]}" = "--include:local" ];
        then
            loc="INCLUDE"
        elif [ "${aargs[index]}" = "--exclude:local" ];
        then
            loc="EXCLUDE"
        elif [[ "${aargs[index]}" == *"cutoff"* ]]
        then
            cutoff=$(echo "${aargs[index]}" | grep -Po '\-\-cutoff\=\K\d+')
        fi
    done

    local key=$(printf "'%s.%s'" $module $libname)
    echo "$(printf "    %-36s: (%2d, %s, %s, %s), # %5.1f" $key $cutoff $sys $mod $loc $best_time)"
}

for module in $modules; do

    # Build without pch includes as sanity check.
    #run "$root" "$module" --cutoff=999

    # Build before updating pch.
    $MAKE "$module.build" > /dev/null
    if [ $? -ne 0 ];
    then
        # Build with dependencies before updating pch.
        echo "Failed to build $module, building known state with dependencies..."
        ./bin/update_pch.sh "$module" > /dev/null
        $MAKE "$module.clean" > /dev/null
        $MAKE "$module.all" > /dev/null
        if [ $? -ne 0 ];
        then
            # Build all!
            echo "Failed to build $module with dependencies, building all..."
            $MAKE build-nocheck > /dev/null
            if [ $? -ne 0 ];
            then
                >&2 echo "Broken build. Please revert changes and try again."
                exit 1
            fi
        fi
    fi

    # Find pch files in the module to update.
    headers=`find $root/$module/ -type f -iname "precompiled_*.hxx"`

    # Each pch belongs to a library.
    for header in $headers; do
        libname=`echo $header | sed -e s/.*precompiled_// -e s/\.hxx//`
        #TODO: Backup the header and restore when last tune fails.

        # Force update on first try below.
        echo "Autotuning $module.$libname..."
        ./bin/update_pch "$module" "$libname" --cutoff=999 --silent --force

        best_score=0
        best_args=""
        best_time=0
        best_cutoff=0
        for i in {1..16}; do
            cutoff=$i
            best_score_for_cuttof=0
            #run "$root" "$module" "--cutoff=$i" --include:system --exclude:module --exclude:local
            run "$root" "$module" "--cutoff=$i" --exclude:system --exclude:module --exclude:local
            #run "$root" "$module" "--cutoff=$i" --include:system --include:module --exclude:local
            run "$root" "$module" "--cutoff=$i" --exclude:system --include:module --exclude:local
            #run "$root" "$module" "--cutoff=$i" --include:system --exclude:module --include:local
            run "$root" "$module" "--cutoff=$i" --exclude:system --exclude:module --include:local
            #run "$root" "$module" "--cutoff=$i" --include:system --include:module --include:local
            run "$root" "$module" "--cutoff=$i" --exclude:system --include:module --include:local

            if [ $i -gt $((best_cutoff+2)) ];
            then
                score_too_low=$(echo "$best_score_for_cuttof < $best_score / 1.10" | bc -l)
                if [ $score_too_low -eq 1 ];
                then
                    echo "Score hit low of $best_score_for_cuttof, well below overall best of $best_score. Stopping."
                    break;
                fi
            fi
        done

        ./bin/update_pch "$module" "$libname" $best_args --force --silent
        echo "> $module.$libname, $best_args, $best_time, $size, $score"
        echo

        table+=$'\n'
        table+="$(args_to_table)"
    done

done

echo "Update the relevant lines in ./bin/update_pch script:"
>&2 echo "$table"

exit 0