File: publish.sh

package info (click to toggle)
python-line-profiler 5.0.0-1
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid
  • size: 1,256 kB
  • sloc: python: 8,119; sh: 810; ansic: 297; makefile: 14
file content (573 lines) | stat: -rwxr-xr-x 15,901 bytes parent folder | download | duplicates (2)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
#!/usr/bin/env bash
__doc__='
Script to publish a new version of this library on PyPI.

If your script has binary dependencies then we assume that you have built a
proper binary wheel with auditwheel and it exists in the wheelhouse directory.
Otherwise, for source tarballs and wheels this script runs the
setup.py script to create the wheels as well.

Running this script with the default arguments will perform any builds and gpg
signing, but nothing will be uploaded to pypi unless the user explicitly sets
DO_UPLOAD=True or answers yes to the prompts.

Args:
    TWINE_USERNAME (str) :
        username for pypi. This must be set if uploading to pypi.
        Defaults to "".

    TWINE_PASSWORD (str) :
        password for pypi. This must be set if uploading to pypi.
        Defaults to "".

    DO_GPG (bool) :
        If True, sign the packages with a GPG key specified by `GPG_KEYID`.
        defaults to auto.

    DO_OTS (bool) :
        If True, make an opentimestamp for the package and signature (if
        available)

    DO_UPLOAD (bool) :
        If True, upload the packages to the pypi server specified by
        `TWINE_REPOSITORY_URL`.

    DO_BUILD (bool) :
        If True, will execute the setup.py build script, which is
        expected to use setuptools. In the future we may add support for other
        build systems. If False, this script will expect the pre-built packages
        to exist in "wheelhouse/{NAME}-{VERSION}-{SUFFIX}.{EXT}".

        Defaults to "auto".

    DO_TAG (bool) :
        if True, will "git tag" the current HEAD with

    TWINE_REPOSITORY_URL (url) :
         The URL of the pypi server to upload to.
         Defaults to "auto", which if on the release branch, this will default
         to the live pypi server `https://upload.pypi.org/legacy` otherwise
         this will default to the test.pypi server:
         `https://test.pypi.org/legacy`

     GPG_KEYID (str) :
        The keyid of the gpg key to sign with. (if DO_GPG=True). Defaults to
        the local git config user.signingkey

    DEPLOY_REMOTE (str) :
        The git remote to push any tags to. Defaults to "origin"

    GPG_EXECUTABLE (path) :
        Path to the GPG executable.
        Defaults to "auto", which chooses "gpg2" if it exists, otherwise "gpg".

    MODE (str):
        Can be pure, binary, or all. Defaults to pure unless a CMakeLists.txt
        exists in which case it defaults to binary.

Requirements:
     twine >= 1.13.0
     gpg2 >= 2.2.4
     OpenSSL >= 1.1.1c

Notes:
    # NEW API TO UPLOAD TO PYPI
    # https://docs.travis-ci.com/user/deployment/pypi/
    # https://packaging.python.org/tutorials/distributing-packages/
    # https://stackoverflow.com/questions/45188811/how-to-gpg-sign-a-file-that-is-built-by-travis-ci

    Based on template in

    # github.com/Erotemic/xcookie/
    ~/code/xcookie/publish.sh

Usage:
    load_secrets
    # TODO: set a trap to unload secrets?
    cd <YOUR REPO>
    # Set your variables or load your secrets
    export TWINE_USERNAME=<pypi-username>
    export TWINE_PASSWORD=<pypi-password>
    TWINE_REPOSITORY_URL="https://test.pypi.org/legacy/"
'

DEBUG=${DEBUG:=''}
if [[ "${DEBUG}" != "" ]]; then
    set -x
fi

check_variable(){
    KEY=$1
    HIDE=$2
    VAL=${!KEY}
    if [[ "$HIDE" == "" ]]; then
        echo "[DEBUG] CHECK VARIABLE: $KEY=\"$VAL\""
    else
        echo "[DEBUG] CHECK VARIABLE: $KEY=<hidden>"
    fi
    if [[ "$VAL" == "" ]]; then
        echo "[ERROR] UNSET VARIABLE: $KEY=\"$VAL\""
        exit 1;
    fi
}


normalize_boolean(){
    ARG=$1
    ARG=$(echo "$ARG" | awk '{print tolower($0)}')
    if [ "$ARG" = "true" ] || [ "$ARG" = "1" ] || [ "$ARG" = "yes" ] || [ "$ARG" = "y" ] || [ "$ARG" = "on" ]; then
        echo "True"
    elif [ "$ARG" = "false" ] || [ "$ARG" = "0" ] || [ "$ARG" = "no" ] || [ "$ARG" = "n" ] || [ "$ARG" = "off" ]; then
        echo "False"
    else
        echo "$ARG"
    fi
}


####
# Parameters
###

# Options
DEPLOY_REMOTE=${DEPLOY_REMOTE:=origin}
NAME=${NAME:=$(python -c "import setup; print(setup.NAME)")}
VERSION=$(python -c "import setup; print(setup.VERSION)")

check_variable DEPLOY_REMOTE

ARG_1=$1

DO_UPLOAD=${DO_UPLOAD:=$ARG_1}
DO_TAG=${DO_TAG:=$ARG_1}

DO_GPG=${DO_GPG:="auto"}
if [ "$DO_GPG" == "auto" ]; then
    DO_GPG="True"
fi

DO_OTS=${DO_OTS:="auto"}
if [ "$DO_OTS" == "auto" ]; then
    # Do opentimestamp if it is available
    # python -m pip install opentimestamps-client
    if type ots ; then
        DO_OTS="True"
    else
        DO_OTS="False"
    fi
fi

DO_BUILD=${DO_BUILD:="auto"}
# Verify that we want to build
if [ "$DO_BUILD" == "auto" ]; then
    DO_BUILD="True"
fi

DO_GPG=$(normalize_boolean "$DO_GPG")
DO_OTS=$(normalize_boolean "$DO_OTS")
DO_BUILD=$(normalize_boolean "$DO_BUILD")
DO_UPLOAD=$(normalize_boolean "$DO_UPLOAD")
DO_TAG=$(normalize_boolean "$DO_TAG")

TWINE_USERNAME=${TWINE_USERNAME:=""}
TWINE_PASSWORD=${TWINE_PASSWORD:=""}

DEFAULT_TEST_TWINE_REPO_URL="https://test.pypi.org/legacy/"
DEFAULT_LIVE_TWINE_REPO_URL="https://upload.pypi.org/legacy/"

TWINE_REPOSITORY_URL=${TWINE_REPOSITORY_URL:="auto"}
if [[ "${TWINE_REPOSITORY_URL}" == "auto" ]]; then
    #if [[ "$(cat .git/HEAD)" != "ref: refs/heads/release" ]]; then
    #    # If we are not on release, then default to the test pypi upload repo
    #    TWINE_REPOSITORY_URL=${TWINE_REPOSITORY_URL:="https://test.pypi.org/legacy/"}
    #else
    if [[ "$DEBUG" == "" ]]; then
        TWINE_REPOSITORY_URL="live"
    else
        TWINE_REPOSITORY_URL="test"
    fi
fi

if [[ "${TWINE_REPOSITORY_URL}" == "live" ]]; then
    TWINE_REPOSITORY_URL=$DEFAULT_LIVE_TWINE_REPO_URL
elif [[ "${TWINE_REPOSITORY_URL}" == "test" ]]; then
    TWINE_REPOSITORY_URL=$DEFAULT_TEST_TWINE_REPO_URL
fi

GPG_EXECUTABLE=${GPG_EXECUTABLE:="auto"}
if [[ "$GPG_EXECUTABLE" == "auto" ]]; then
    if [[ "$(which gpg2)" != "" ]]; then
        GPG_EXECUTABLE="gpg2"
    else
        GPG_EXECUTABLE="gpg"
    fi
fi

GPG_KEYID=${GPG_KEYID:="auto"}
if [[ "$GPG_KEYID" == "auto" ]]; then
    GPG_KEYID=$(git config --local user.signingkey)
    if [[ "$GPG_KEYID" == "" ]]; then
        GPG_KEYID=$(git config --global user.signingkey)
    fi
fi


if [ -f CMakeLists.txt ] ; then
    DEFAULT_MODE="binary"
else
    DEFAULT_MODE="pure"
fi


# TODO: parameterize
# The default should change depending on the application
MODE=${MODE:=$DEFAULT_MODE}
if [[ "$MODE" == "all" ]]; then
    MODE_LIST=("sdist" "native" "bdist")
elif [[ "$MODE" == "pure" ]]; then
    MODE_LIST=("sdist" "native")
elif [[ "$MODE" == "binary" ]]; then
    MODE_LIST=("sdist" "bdist")
else
    MODE_LIST=("$MODE")
fi
MODE_LIST_STR=$(printf '"%s" ' "${MODE_LIST[@]}")
#echo "MODE_LIST_STR = $MODE_LIST_STR"


####
# Logic
###

WAS_INTERACTION="False"

echo "
=== PYPI BUILDING SCRIPT ==
NAME='$NAME'
VERSION='$VERSION'
TWINE_USERNAME='$TWINE_USERNAME'
TWINE_REPOSITORY_URL = $TWINE_REPOSITORY_URL
GPG_KEYID = '$GPG_KEYID'

DO_UPLOAD=${DO_UPLOAD}
DO_TAG=${DO_TAG}
DO_GPG=${DO_GPG}
DO_OTS=${DO_OTS}
DO_BUILD=${DO_BUILD}
MODE_LIST_STR=${MODE_LIST_STR}
"


# Verify that we want to tag
if [[ "$DO_TAG" == "True" ]]; then
    echo "About to tag VERSION='$VERSION'"
else
    if [[ "$DO_TAG" == "False" ]]; then
        echo "We are NOT about to tag VERSION='$VERSION'"
    else
        # shellcheck disable=SC2162
        read -p "Do you want to git tag and push version='$VERSION'? (input 'yes' to confirm)" ANS
        echo "ANS = $ANS"
        WAS_INTERACTION="True"
        DO_TAG="$ANS"
        DO_TAG=$(normalize_boolean "$DO_TAG")
        if [ "$DO_BUILD" == "auto" ]; then
            DO_BUILD=""
            DO_GPG=""
        fi
    fi
fi



if [[ "$DO_BUILD" == "True" ]]; then
    echo "About to build wheels"
else
    if [[ "$DO_BUILD" == "False" ]]; then
        echo "We are NOT about to build wheels"
    else
        # shellcheck disable=SC2162
        read -p "Do you need to build wheels? (input 'yes' to confirm)" ANS
        echo "ANS = $ANS"
        WAS_INTERACTION="True"
        DO_BUILD="$ANS"
        DO_BUILD=$(normalize_boolean "$DO_BUILD")
    fi
fi


# Verify that we want to publish
if [[ "$DO_UPLOAD" == "True" ]]; then
    echo "About to directly publish VERSION='$VERSION'"
else
    if [[ "$DO_UPLOAD" == "False" ]]; then
        echo "We are NOT about to directly publish VERSION='$VERSION'"
    else
        # shellcheck disable=SC2162
        read -p "Are you ready to directly publish version='$VERSION'? ('yes' will twine upload)" ANS
        echo "ANS = $ANS"
        WAS_INTERACTION="True"
        DO_UPLOAD="$ANS"
        DO_UPLOAD=$(normalize_boolean "$DO_UPLOAD")
    fi
fi


if [[ "$WAS_INTERACTION" == "True" ]]; then
    echo "
    === PYPI BUILDING SCRIPT ==
    VERSION='$VERSION'
    TWINE_USERNAME='$TWINE_USERNAME'
    TWINE_REPOSITORY_URL = $TWINE_REPOSITORY_URL
    GPG_KEYID = '$GPG_KEYID'

    DO_UPLOAD=${DO_UPLOAD}
    DO_TAG=${DO_TAG}
    DO_GPG=${DO_GPG}
    DO_BUILD=${DO_BUILD}
    MODE_LIST_STR='${MODE_LIST_STR}'
    "
    # shellcheck disable=SC2162
    read -p "Look good? Ready to build? Enter any text to continue" ANS
fi



if [ "$DO_BUILD" == "True" ]; then

    echo "
    === <BUILD WHEEL> ===
    "

    echo "LIVE BUILDING"
    # Build wheel and source distribution
    for _MODE in "${MODE_LIST[@]}"
    do
        echo "_MODE = $_MODE"
        if [[ "$_MODE" == "sdist" ]]; then
            python setup.py sdist || { echo 'failed to build sdist wheel' ; exit 1; }
        elif [[ "$_MODE" == "native" ]]; then
            python setup.py bdist_wheel || { echo 'failed to build native wheel' ; exit 1; }
        elif [[ "$_MODE" == "bdist" ]]; then
            echo "Assume wheel has already been built"
        else
            echo "ERROR: bad mode"
            exit 1
        fi
    done

    echo "
    === <END BUILD WHEEL> ===
    "

else
    echo "DO_BUILD=False, Skipping build"
fi


ls_array(){
    __doc__='
    Read the results of a glob pattern into an array

    Args:
        arr_name
        glob_pattern

    Example:
        arr_name="myarray"
        glob_pattern="*"
        pass
    '
    local arr_name="$1"
    local glob_pattern="$2"
    shopt -s nullglob
    # shellcheck disable=SC2206
    array=($glob_pattern)
    shopt -u nullglob # Turn off nullglob to make sure it doesn't interfere with anything later
    # FIXME; for some reason this does not always work properly
    # Copy the array into the dynamically named variable
    # shellcheck disable=SC2086
    readarray -t $arr_name < <(printf '%s\n' "${array[@]}")
}


WHEEL_FPATHS=()
for _MODE in "${MODE_LIST[@]}"
do
    if [[ "$_MODE" == "sdist" ]]; then
        ls_array "_NEW_WHEEL_PATHS" "dist/${NAME}-${VERSION}*.tar.gz"
    elif [[ "$_MODE" == "native" ]]; then
        ls_array "_NEW_WHEEL_PATHS" "dist/${NAME}-${VERSION}*.whl"
    elif [[ "$_MODE" == "bdist" ]]; then
        ls_array "_NEW_WHEEL_PATHS" "wheelhouse/${NAME}-${VERSION}-*.whl"
    else
        echo "ERROR: bad mode"
        exit 1
    fi
    # hacky CONCAT because for some reason ls_array will return
    # something that looks empty but has one empty element
    for new_item in "${_NEW_WHEEL_PATHS[@]}"
    do
        if [[ "$new_item" != "" ]]; then
            WHEEL_FPATHS+=("$new_item")
        fi
    done
done

# Dedup the paths
readarray -t WHEEL_FPATHS < <(printf '%s\n' "${WHEEL_FPATHS[@]}" | sort -u)

WHEEL_PATHS_STR=$(printf '"%s" ' "${WHEEL_FPATHS[@]}")
echo "WHEEL_PATHS_STR = $WHEEL_PATHS_STR"

echo "
MODE=$MODE
VERSION='$VERSION'
WHEEL_FPATHS='$WHEEL_PATHS_STR'
"


WHEEL_SIGNATURE_FPATHS=()
if [ "$DO_GPG" == "True" ]; then

    echo "
    === <GPG SIGN> ===
    "

    for WHEEL_FPATH in "${WHEEL_FPATHS[@]}"
    do
        echo "WHEEL_FPATH = $WHEEL_FPATH"
        check_variable WHEEL_FPATH
            # https://stackoverflow.com/questions/45188811/how-to-gpg-sign-a-file-that-is-built-by-travis-ci
            # secure gpg --export-secret-keys > all.gpg

            # REQUIRES GPG >= 2.2
            check_variable GPG_EXECUTABLE || { echo 'failed no gpg exe' ; exit 1; }
            check_variable GPG_KEYID || { echo 'failed no gpg key' ; exit 1; }

            echo "Signing wheels"
            GPG_SIGN_CMD="$GPG_EXECUTABLE --batch --yes --detach-sign --armor --local-user $GPG_KEYID"
            echo "GPG_SIGN_CMD = $GPG_SIGN_CMD"
            $GPG_SIGN_CMD --output "$WHEEL_FPATH".asc "$WHEEL_FPATH"

            echo "Checking wheels"
            twine check "$WHEEL_FPATH".asc "$WHEEL_FPATH" || { echo 'could not check wheels' ; exit 1; }

            echo "Verifying wheels"
            $GPG_EXECUTABLE --verify "$WHEEL_FPATH".asc "$WHEEL_FPATH" || { echo 'could not verify wheels' ; exit 1; }

            WHEEL_SIGNATURE_FPATHS+=("$WHEEL_FPATH".asc)
    done
    echo "
    === <END GPG SIGN> ===
    "
else
    echo "DO_GPG=False, Skipping GPG sign"
fi



if [ "$DO_OTS" == "True" ]; then

    echo "
    === <OTS SIGN> ===
    "
    if [ "$DO_GPG" == "True" ]; then
        # Stamp the wheels and the signatures
        ots stamp "${WHEEL_FPATHS[@]}" "${WHEEL_SIGNATURE_FPATHS[@]}"
    else
        # Stamp only the wheels
        ots stamp "${WHEEL_FPATHS[@]}"
    fi
    echo "
    === <END OTS SIGN> ===
    "
else
    echo "DO_OTS=False, Skipping OTS sign"
fi


if [[ "$DO_TAG" == "True" ]]; then
    TAG_NAME="v${VERSION}"
    # if we messed up we can delete the tag
    # git push origin :refs/tags/$TAG_NAME
    # and then tag with -f
    #
    git tag "$TAG_NAME" -m "tarball tag $VERSION"
    git push --tags "$DEPLOY_REMOTE"
    echo "Should also do a: git push $DEPLOY_REMOTE main:release"
    echo "For github should draft a new release: https://github.com/PyUtils/line_profiler/releases/new"
else
    echo "Not tagging"
fi


if [[ "$DO_UPLOAD" == "True" ]]; then
    check_variable TWINE_USERNAME
    check_variable TWINE_PASSWORD "hide"

    for WHEEL_FPATH in "${WHEEL_FPATHS[@]}"
    do
        twine upload --username "$TWINE_USERNAME" "--password=$TWINE_PASSWORD" \
            --repository-url "$TWINE_REPOSITORY_URL" \
            "$WHEEL_FPATH" --skip-existing --verbose || { echo 'failed to twine upload' ; exit 1; }
    done
    echo """
        !!! FINISH: LIVE RUN !!!
    """
else
    echo """
        DRY RUN ... Skipping upload

        DEPLOY_REMOTE = '$DEPLOY_REMOTE'
        DO_UPLOAD = '$DO_UPLOAD'
        WHEEL_FPATH = '$WHEEL_FPATH'
        WHEEL_PATHS_STR = '$WHEEL_PATHS_STR'
        MODE_LIST_STR = '$MODE_LIST_STR'

        VERSION='$VERSION'
        NAME='$NAME'
        TWINE_USERNAME='$TWINE_USERNAME'
        GPG_KEYID = '$GPG_KEYID'

        To do live run set DO_UPLOAD=1 and ensure deploy and current branch are the same

        !!! FINISH: DRY RUN !!!
    """
fi

__devel__='
# Checking to see how easy it is to upload packages to gitlab.
# This logic should go in the CI script, not sure if it belongs here.


export HOST=https://gitlab.kitware.com
export GROUP_NAME=computer-vision
export PROJECT_NAME=geowatch
PROJECT_VERSION=$(geowatch --version)
echo "$PROJECT_VERSION"

load_secrets
export PRIVATE_GITLAB_TOKEN=$(git_token_for "$HOST")
TMP_DIR=$(mktemp -d -t ci-XXXXXXXXXX)

curl --header "PRIVATE-TOKEN: $PRIVATE_GITLAB_TOKEN" "$HOST/api/v4/groups" > "$TMP_DIR/all_group_info"
GROUP_ID=$(cat "$TMP_DIR/all_group_info" | jq ". | map(select(.name==\"$GROUP_NAME\")) | .[0].id")
echo "GROUP_ID = $GROUP_ID"

curl --header "PRIVATE-TOKEN: $PRIVATE_GITLAB_TOKEN" "$HOST/api/v4/groups/$GROUP_ID" > "$TMP_DIR/group_info"
PROJ_ID=$(cat "$TMP_DIR/group_info" | jq ".projects | map(select(.name==\"$PROJECT_NAME\")) | .[0].id")
echo "PROJ_ID = $PROJ_ID"

ls_array DIST_FPATHS "dist/*"

for FPATH in "${DIST_FPATHS[@]}"
do
    FNAME=$(basename $FPATH)
    echo $FNAME
    curl --header "PRIVATE-TOKEN: $PRIVATE_GITLAB_TOKEN" \
         --upload-file $FPATH \
         "https://gitlab.kitware.com/api/v4/projects/$PROJ_ID/packages/generic/$PROJECT_NAME/$PROJECT_VERSION/$FNAME"
done

'