1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252
|
#!/bin/sh
# shellcheck disable=SC3043 # local is not POSIX, but every shell has it
# shellcheck disable=SC3013,SC3045 # ditto for test {-nt,-t}
GITHUB_REPO='node-cache'
SUBDIR='node_modules'
V="${V-0}" # default to friendly messages
set -eu
cd "${0%/*}/.."
# shellcheck source-path=SCRIPTDIR/..
. test/common/git-utils.sh
cmd_remove() {
# if we did this for ourselves the rm is enough, but it might be the case
# that someone actually used git-submodule to fetch this, so clean up after
# that as well. NB: deinit nicely recreates the empty directory for us.
message REMOVE node_modules
rm -rf node_modules
git submodule deinit node_modules
rm -rf -- "$(git rev-parse --absolute-git-dir)/modules/node_modules"
}
cmd_checkout() {
# we default to check out the node_modules corresponding to the gitlink in the index
local force=""
if [ "${1-}" = "--force" ]; then
force="1"
shift
fi
local sha="${1-$(get_index_gitlink node_modules)}"
# fetch by sha to prevent us from downloading something we don't want
fetch_sha_to_cache "${sha}"
# verify that our package.json is equal to the one the cached node_modules
# was created with, unless --force is given
if [ -z "$force" ]; then
if ! cmp_from_cache "${sha}" '.package.json' 'package.json'; then
cat >&2 <<EOF
*** node_modules ${sha} doesn't match our package.json
*** refusing to automatically check out node_modules
Options:
- tools/node-modules checkout --force # disable this check
- tools/node-modules install # npm install with our package.json
$0: *** aborting
EOF
exit 1
fi
fi
# we're actually going to do this; let's remove the old one
cmd_remove
# and check out the new one
# we need to use the tag name here, unfortunately
clone_from_cache "${sha}"
}
cmd_install() {
test -e bots || test/common/make-bots
# We first read the result directly into the cache, then we unpack it.
tree="$(bots/npm download < package.json | tar_to_cache)"
commit="$(sha256sum package.json | git_cache commit-tree "${tree}")"
git_cache tag "sha-${commit}" "${commit}" "--no-sign"
cmd_checkout "${commit}"
cat <<EOF
Next steps:
- git add node_modules && git commit
- tools/node-modules push
EOF
}
cmd_push() {
# push via the cache: the shared history with the remote helps to thin out the pack we send
tag="sha-$(git -C node_modules rev-parse HEAD)"
message PUSH "${GITHUB_REPO} ${tag}"
git_cache push "${SSH_REMOTE}" "${tag}"
}
cmd_verify() {
test -e bots || test/common/make-bots
# Verifies that the package.json and node_modules of the given commit match.
commit="$(git rev-parse "$1:node_modules")"
fetch_sha_to_cache "${commit}"
committed_tree="$(git_cache rev-parse "${commit}^{tree}")"
expected_tree="$(git cat-file blob "$1:package.json" | bots/npm download | tar_to_cache)"
if [ "${committed_tree}" != "${expected_tree}" ]; then
exec >&2
printf "\nCommit %s package.json and node_modules aren't in sync!\n\n" "$1"
git --no-pager show --stat "$1"
printf "\nThe above commit refers to the following node_modules commit:\n\n"
git_cache --no-pager show --no-patch "${commit}"
printf "\nOur attempt to recreate that commit differs as follows:\n\n"
git_cache --no-pager diff --stat "${commit}" "${expected_tree}" --
git_cache --no-pager diff "${commit}" "${expected_tree}" -- .package-lock.json
exit 1
fi
}
cmd_runtime_tar() {
if [ -z "${1:-}" ]; then
echo "Usage: $0 runtime-tar <tarball-path>" >&2
exit 1
fi
if [ ! -e node_modules/.package-lock.json ]; then
echo "Error: This requires a checked out node_modules, run 'tools/node-modules checkout' first" >&2
exit 1
fi
local tar_path="$1"
# Verify lockfile version
local lockfile_version
lockfile_version="$(jq -r '.lockfileVersion' node_modules/.package-lock.json)"
if [ "$lockfile_version" != "3" ]; then
echo "Error: this code is only certified for lockfileVersion 3, got $lockfile_version" >&2
exit 1
fi
# Collect runtime packages using jq, excluding:
# - devDependencies
# - esbuild (arch dependent, must use distribution packages)
# - unnecessary binary packages (use JavaScript sass, don't need watcher)
local runtime_pkgs
runtime_pkgs="$(jq -r '
.packages | to_entries[] |
select(.value.dev != true) |
.key |
select(test(".*/@?esbuild(/|$)") | not) |
select(test("watcher-linux|sass-embedded") | not)
' node_modules/.package-lock.json | while read -r pkg; do
# Only include if exists and is not empty
[ -n "$(ls -A "$pkg" 2>/dev/null)" ] && echo "$pkg"
done)"
# Validate that we only have architecture independent files
# special-case node_modules/iconv-lite/encodings/sbcs-data-generated.js, it's JS with binary strings in it
# and resolve/test/resolver/cup.coffee which is a single-byte test file
# Ubuntu 24.04's file (or at least not that in the GitHub actions runner) does not detect *.woff2 file type properly
for pkg in $runtime_pkgs; do
if find "$pkg" -type f -print0 | xargs -0 file --mime |
grep -Ev 'text/|inode/x-empty|application/(javascript|json)|font/|image/|sbcs-data-generated.js|cup.coffee|\.woff'; then
echo "Error: package '$pkg' contains architecture dependent files, cannot include in runtime tarball" >&2
exit 1
fi
done
# Create tarball
{
echo "node_modules/.package-lock.json"
echo "$runtime_pkgs"
} | tar --format=ustar --sort=name --owner=root:0 --group=root:0 --xz -cf "$tar_path" -C . -T -
}
# called from Makefile.am
cmd_make_package_lock_json() {
# Run from make to ensure package-lock.json is up to date
# package-lock.json is used as the stamp file for all things that use
# node_modules, so this is the main bit of glue that drives the entire process
# We try our best not to touch package-lock.json unless it actually changes
# This isn't going to work for a tarball, but as long as
# package-lock.json is already there, and newer than package.json,
# we're OK
if [ ! -e .git ]; then
if [ package-lock.json -nt package.json ]; then
exit 0
fi
echo "*** Can't update node modules unless running from git" >&2
exit 1
fi
# Otherwise, our main goal is to ensure that the node_modules from
# the index is the one that we actually have.
local sha
sha="$(get_index_gitlink node_modules)"
if [ ! -e node_modules/.git ]; then
# nothing there yet...
cmd_checkout
elif [ "$(git -C node_modules rev-parse HEAD)" != "${sha}" ]; then
# wrong thing there...
cmd_checkout
fi
# This check is more about catching local changes to package.json than
# about validating something we just checked out:
if ! cmp -s node_modules/.package.json package.json; then
cat 2>&1 <<EOF
*** package.json is out of sync with node_modules
*** If you modified package.json, please run:
***
*** tools/node-modules install
***
*** and add the result to the index.
EOF
exit 1
fi
# Only copy the package-lock.json if it differs from the one we have
if ! cmp -s node_modules/.package-lock.json package-lock.json; then
message COPY package-lock.json
cp node_modules/.package-lock.json package-lock.json
fi
# We're now in a situation where:
# - the checked out node_modules is equal to the gitlink in the index
# - the package.json in the tree is equal to the one in node_modules
# - ditto package-lock.json
exit 0
}
main() {
if [ $# = 0 ]; then
# don't list the "private" ones
echo 'This command requires a subcommand: remove checkout install push verify runtime-tar'
exit 1
fi
local fname
fname="$(printf 'cmd_%s' "$1" | tr '-' '_')"
if ! type -t "${fname}" | grep -q function; then
echo "Unknown subcommand '$1'"
exit 1
fi
shift
[ -n "${quiet}" ] || set -x
"${fname}" "$@"
}
main "$@"
|