File: shellHelperFunctions.sh

package info (click to toggle)
keyman 18.0.245-1
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid
  • size: 21,316 kB
  • sloc: python: 52,784; cpp: 21,278; sh: 7,633; ansic: 4,823; xml: 3,617; perl: 959; makefile: 139; javascript: 138
file content (365 lines) | stat: -rwxr-xr-x 11,709 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
#!/usr/bin/env bash

# We only import JQ if not already defined; jq is used by
# _select_node_version_with_nvm()

if [[ -z "${JQ+x}" ]]; then
  if [[ "$BUILDER_OS" == win ]]; then
    . "$KEYMAN_ROOT/resources/build/jq.inc.sh"
  else
    JQ=jq
  fi
fi

# Allows for a quick macOS check for those scripts requiring a macOS environment.
verify_on_mac() {
  if [[ "${OSTYPE}" != "darwin"* ]]; then
    builder_die "This build script will only run in a Mac environment."
    exit 1
  fi
}

# The list of valid projects that our build scripts ought expect.
projects=("android" "ios" "linux" "lmlayer" "mac" "web" "windows")

# Used to validate a specified 'project' parameter.
verify_project() {
  match=false
  for proj in ${projects[@]}
  do
    if [ $proj = $1 ]; then
      match=true
    fi
  done

  if [ $match = false ]; then
    builder_die "Invalid project specified!"
  fi
}

displayInfo() {
    if [ "$QUIET" != true ]; then
        while [[ $# -gt 0 ]] ; do
            echo $1
            shift # past argument
        done
    fi
}

assertFileExists() {
    if ! [ -f $1 ]; then
        builder_die "Build failed:  missing $1"
    fi
}

assertDirExists() {
    if ! [ -d $1 ]; then
        builder_die "Build failed:  missing $1"
    fi
}

assertValidVersionNbr()
{
    if [[ "$1" == "" || ! "$1" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
        builder_die "Specified version not valid: '$1'. Version should be in the form Major.Minor.BuildCounter"
    fi
}

assertValidPRVersionNbr()
{
    if [[ "$1" == "" || ! "$1" =~ ^[0-9]+\.[0-9]+\.pull\.[0-9]+$ ]]; then
        builder_die "Specified version not valid: '$1'. Version should be in the form Major.Minor.pull.BuildCounter"
    fi
}

dl_info_display_usage() {
    echo "Used to create a metadata file needed on the download site"
    echo "for it to connect to the download.keyman.com API functions."
    echo
    echo "usage: write-download_info <name> <filepath> <version> <tier> <platform>"
    echo
    echo "  name           Specifies the user-friendly name of the product represented by the file."
    echo "  filepath       Specifies the path and file in need of a .download_info metadata file."
    echo "  version        Specifies the build version number, which should be in the"
    echo "                   form Major.Minor.BuildCounter"
    echo "  tier           Specifies tier (typically one of: alpha, beta, stable)."
    echo "  platform       Specifies the target platforms for the file."
    echo "                 (Should be one of: android, ios, mac, web, windows)"
    echo
    echo "The resulting .downloadinfo file will be automatically placed in the same directory"
    echo "as the originally-specified file."
    exit 1
}

write_download_info() {
  #Process file & path information.
  PRODUCT_NAME="$1"
  BASE_PATH="$2"
  KM_VERSION="$3"
  KM_TIER="$4"
  KM_PLATFORM="$5"

  verify_project "$KM_PLATFORM"

  BASE_DIR=$(dirname "${BASE_PATH}")
  BASE_FILE=$(basename "${BASE_PATH}");

  if ([ -h "${BASE_DIR}" ]) then
    while([ -h "${BASE_DIR}" ]) do BASE_PATH=`readlink "${BASE_DIR}"`;
    done
  fi

  assertFileExists "$2"

  pushd . > /dev/null
  cd `dirname ${BASE_DIR}` > /dev/null
  BASE_PATH=`pwd`;
  popd  > /dev/null

  DEST_DIR="$BASE_DIR"

  #Process version parameter.
  assertValidVersionNbr "$3"
  KM_BLD_COUNTER="$((${KM_VERSION##*.}))"

  if [ "$KM_VERSION" = "" ]; then
    builder_die "Required -version parameter not specified!"
  fi

  if [ "$KM_TIER" = "" ]; then
    builder_die "Required -tier parameter not specified!"
  fi

  DOWNLOAD_INFO_FILEPATH="${BASE_PATH}/${BASE_FILE}.download_info"
  if [[ ! -f "${BASE_PATH}/${BASE_FILE}" ]]; then
    builder_die "Cannot compute file size or MD5 for non-existent DMG file: ${BASE_PATH}/${BASE_FILE}"
  fi

  FILE_EXTENSION="${BASE_FILE##*.}"

  FILE_SIZE=$(/usr/bin/stat -f"%z" "${BASE_PATH}/${BASE_FILE}")
  MD5_HASH=$(md5 -q "${BASE_PATH}/${BASE_FILE}")

  if [[ -f "$DOWNLOAD_INFO_FILEPATH" ]]; then
    builder_warn "Overwriting $DOWNLOAD_INFO_FILEPATH"
  fi

  echo { > "$DOWNLOAD_INFO_FILEPATH"
  echo "  \"name\": \"${PRODUCT_NAME}\"," >> "$DOWNLOAD_INFO_FILEPATH"
  echo "  \"version\": \"${KM_VERSION}\"," >> "$DOWNLOAD_INFO_FILEPATH"
  echo "  \"date\": \"$(date "+%Y-%m-%d")\"," >> "$DOWNLOAD_INFO_FILEPATH"
  echo "  \"platform\": \"${KM_PLATFORM}\"," >> "$DOWNLOAD_INFO_FILEPATH"
  echo "  \"stability\": \"${KM_TIER}\"," >> "$DOWNLOAD_INFO_FILEPATH"
  echo "  \"file\": \"${BASE_FILE}\"," >> "$DOWNLOAD_INFO_FILEPATH"
  echo "  \"md5\": \"${MD5_HASH}\"," >> "$DOWNLOAD_INFO_FILEPATH"
  echo "  \"type\": \"${FILE_EXTENSION}\"," >> "$DOWNLOAD_INFO_FILEPATH"
  echo "  \"build\": \"${KM_BLD_COUNTER}\"," >> "$DOWNLOAD_INFO_FILEPATH"
  echo "  \"size\": \"${FILE_SIZE}\"" >> "$DOWNLOAD_INFO_FILEPATH"
  echo } >> "$DOWNLOAD_INFO_FILEPATH"
}

# set_version sets the file version on mac/ios projects
set_version ( ) {
  PRODUCT_PATH=$1

  if [ $VERSION ]; then
    if [ $2 ]; then  # $2 = product name.
      echo "Setting version numbers in $2 to $VERSION."
    fi
    /usr/libexec/Plistbuddy -c "Set CFBundleVersion $VERSION" "$PRODUCT_PATH/Info.plist"
    /usr/libexec/Plistbuddy -c "Set CFBundleShortVersionString $VERSION" "$PRODUCT_PATH/Info.plist"
  fi
}


# Uses npm to set the current package version (package.json).
#
# This sets the version according to the current VERSION_WITH_TAG.
#
# Usage:
#
#   set_npm_version
#
set_npm_version () {
  # We use --no-git-tag-version because our CI system controls version numbering and
  # already tags releases. We also want to have the version of this match the
  # release of Keyman Developer -- these two versions should be in sync. Because this
  # is a large repo with multiple projects and build systems, it's better for us that
  # individual build systems don't take too much ownership of git tagging. :)
  npm version --allow-same-version --no-git-tag-version --no-commit-hooks "$VERSION_WITH_TAG"
}

#
# Re-runs the specified command-line instruction up to 5 times should it fail, waiting a
# random delay between each attempt.  No re-runs are attempted after successful commands.
#
# ### Usage
#   try_multiple_times command [param1 param2...]
#
# ### Parameters
#   1: $@         command-line arguments
try_multiple_times ( ) {
  _try_multiple_times 0 "$@"
}

# $1  The current retry count
# $2+ (everything else) the command to retry should it fail
_try_multiple_times ( ) {
  local RETRY_MAX=5

  # Configuration:  wait between 10 sec and 120 sec.

  # in seconds.
  local RETRY_MAX_WAIT_RANGE=111
  # in seconds
  local RETRY_MIN_WAIT=10

  local retryCount=$1
  shift

  if (( "$retryCount" == "$RETRY_MAX" )); then
    builder_die "Retry limit of $RETRY_MAX attempts reached."
  fi

  retryCount=$(( $retryCount + 1 ))

  if (( $retryCount != 1 )); then
    local wait_length=$(( RANDOM % RETRY_MAX_WAIT_RANGE + RETRY_MIN_WAIT ))
    builder_echo "Delaying $wait_length seconds before attempt $retryCount: \`$@\`"
    sleep $wait_length
  fi

  local code=0
  "$@" || code=$?
  if (( $code != 0 )); then
    builder_echo "Command failed with error $code"
    _try_multiple_times $retryCount "$@"
  fi
}

#
# Verifies that node is installed, and installs npm packages, but only once per
# build invocation
#
verify_npm_setup() {
  # We'll piggy-back on the builder module dependency build state to determine
  # if npm ci has been called in the current script invocation. Adding the
  # prefix /external/ to module name in order to differentiate between this and
  # internal modules (although it is unlikely to ever collide!); we will also
  # use this pattern for other similar external dependencies in future. These
  # functions are safe to call even in a non-builder context (they do nothing or
  # return 1 -- not built)
  if builder_has_module_been_built /external/npm-ci; then
    return 0
  fi
  builder_set_module_has_been_built /external/npm-ci

  # If we are on CI environment, automatically select a node version with nvm
  # Also, a developer can set KEYMAN_USE_NVM variable to get this behaviour
  # automatically too (see /docs/build/node.md)
  if [[ "$VERSION_ENVIRONMENT" != local || ! -z "${KEYMAN_USE_NVM+x}" ]]; then
    # For npm publishing, we currently need to use an alternate version of node
    # that includes npm 11.5.1 or later (see #15040). Once we move to node v24,
    # we can probably remove this check
    if [[ -z "${KEYMAN_CI_SKIP_NVM+x}" ]]; then
      _select_node_version_with_nvm
    fi
  fi

  # Check if Node.JS/npm is installed.
  type npm >/dev/null ||\
    builder_die "Build environment setup error detected!  Please ensure Node.js is installed!"

  pushd "$KEYMAN_ROOT" > /dev/null

  offline_param=
  if builder_try_offline; then
    builder_echo "Trying offline build"
    offline_param=--prefer-offline
  fi
  try_multiple_times npm ${offline_param} ci

  popd > /dev/null
}

_print_expected_node_version() {
"$JQ" -r '.engines.node' "$KEYMAN_ROOT/package.json"
}

# Use nvm to select a node version according to package.json
# see /docs/build/node.md
_select_node_version_with_nvm() {
  local REQUIRED_NODE_VERSION="$(_print_expected_node_version)"
  local CURRENT_NODE_VERSION

  if [[ $BUILDER_OS != win ]]; then
    # launch nvm in a sub process, see _builder_nvm.sh for details
    "$KEYMAN_ROOT/resources/build/_builder_nvm.sh" "$REQUIRED_NODE_VERSION"
  else
    CURRENT_NODE_VERSION="$(node --version)"
    if [[ "$CURRENT_NODE_VERSION" != "v$REQUIRED_NODE_VERSION" ]]; then
      start //wait //b nvm install "$REQUIRED_NODE_VERSION"
      start //wait //b nvm use "$REQUIRED_NODE_VERSION"
    fi
  fi

  # Now, check that the node version is correct, on all systems

  # Note: On windows, `nvm use` and `nvm install` always return success.
  # https://github.com/coreybutler/nvm-windows/issues/738

  # note the 'v' prefix that node emits (and npm doesn't!)
  CURRENT_NODE_VERSION="$(node --version)"
  if [[ "$CURRENT_NODE_VERSION" != "v$REQUIRED_NODE_VERSION" ]]; then
    builder_die "Attempted to select node.js version $REQUIRED_NODE_VERSION but found $CURRENT_NODE_VERSION instead"
  fi
}

check-markdown() {
  node "$KEYMAN_ROOT/resources/tools/check-markdown" --root "$1"
}

#
# Runs eslint, builds tests, and then runs tests with mocha + c8 (coverage)
#
# Usage:
#   builder_run_action  test    builder_do_typescript_tests [coverage_threshold]
# Parameters:
#   1: coverage_threshold   optional, minimum coverage for c8 to pass tests,
#                           defaults to 90 (percent)
#
# Todo:
#   Move to builder.typescript.inc.sh when this is established
#
builder_do_typescript_tests() {
  local MOCHA_FLAGS=

  if [[ "${TEAMCITY_GIT_PATH:-}" != "" ]]; then
    # we're running in TeamCity
    MOCHA_FLAGS="-reporter mocha-teamcity-reporter"
  fi

  eslint .
  tsc --build test/

  local THRESHOLD_PARAMS=
  local C8_THRESHOLD=
  if [[ $# -gt 0 ]]; then
    C8_THRESHOLD=$1
    THRESHOLD_PARAMS="--lines $C8_THRESHOLD --statements $C8_THRESHOLD --branches $C8_THRESHOLD --functions $C8_THRESHOLD"
  else
    # Seems like a bug in TeamCity reporter if we don't list default thresholds,
    # see #13418.
    #
    # Making branch and function thresholds slightly lower, because the default
    # for c8 is 0 for these anyway.
    THRESHOLD_PARAMS="--lines 90 --statements 90 --branches 80 --functions 80"
  fi

  c8 --reporter=lcov --reporter=text --exclude-after-remap --check-coverage $THRESHOLD_PARAMS mocha ${MOCHA_FLAGS} "${builder_extra_params[@]}"

  if [[ ! -z "$C8_THRESHOLD" ]]; then
    builder_echo warning "Coverage thresholds are currently $C8_THRESHOLD%, which is lower than ideal."
    builder_echo warning "Please increase threshold in build.sh as test coverage improves."
  fi
}