File: python_wheel_xlinux_build.sh

package info (click to toggle)
apache-arrow 23.0.1-1
  • links: PTS
  • area: main
  • in suites: sid
  • size: 76,220 kB
  • sloc: cpp: 654,608; python: 70,522; ruby: 45,964; ansic: 18,742; sh: 7,365; makefile: 669; javascript: 125; xml: 41
file content (194 lines) | stat: -rwxr-xr-x 6,657 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

set -ex

function check_arrow_visibility {
    nm --demangle --dynamic /tmp/arrow-dist/lib/libarrow.so > nm_arrow.log

    # Filter out Arrow symbols and see if anything remains.
    # '_init' and '_fini' symbols may or not be present, we don't care.
    # (note we must ignore the grep exit status when no match is found)
    grep ' T ' nm_arrow.log | grep -v -E '(arrow|\b_init\b|\b_fini\b)' | cat - > visible_symbols.log

    if [[ -f visible_symbols.log && `cat visible_symbols.log | wc -l` -eq 0 ]]; then
        return 0
    else
        echo "== Unexpected symbols exported by libarrow.so =="
        cat visible_symbols.log
        echo "================================================"

        exit 1
    fi
}

echo "=== (${PYTHON_VERSION}) Clear output directories and leftovers ==="
# Clear output directories and leftovers
rm -rf /tmp/arrow-build
rm -rf /arrow/python/dist
rm -rf /arrow/python/build
rm -rf /arrow/python/repaired_wheels
rm -rf /arrow/python/pyarrow/*.so
rm -rf /arrow/python/pyarrow/*.so.*

echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ==="
: ${ARROW_ACERO:=ON}
: ${ARROW_AZURE:=ON}
: ${ARROW_DATASET:=ON}
: ${ARROW_FLIGHT:=ON}
: ${ARROW_GANDIVA:=OFF}
: ${ARROW_GCS:=ON}
: ${ARROW_HDFS:=ON}
: ${ARROW_MIMALLOC:=ON}
: ${ARROW_ORC:=ON}
: ${ARROW_PARQUET:=ON}
: ${PARQUET_REQUIRE_ENCRYPTION:=ON}
: ${ARROW_SUBSTRAIT:=ON}
: ${ARROW_S3:=ON}
: ${ARROW_TENSORFLOW:=ON}
: ${ARROW_WITH_BROTLI:=ON}
: ${ARROW_WITH_BZ2:=ON}
: ${ARROW_WITH_LZ4:=ON}
: ${ARROW_WITH_SNAPPY:=ON}
: ${ARROW_WITH_ZLIB:=ON}
: ${ARROW_WITH_ZSTD:=ON}
: ${CMAKE_BUILD_TYPE:=release}
: ${CMAKE_UNITY_BUILD:=ON}
: ${CMAKE_GENERATOR:=Ninja}
: ${VCPKG_ROOT:=/opt/vcpkg}
: ${VCPKG_FEATURE_FLAGS:=-manifests}
: ${VCPKG_TARGET_TRIPLET:=${VCPKG_DEFAULT_TRIPLET:-x64-linux-static-${CMAKE_BUILD_TYPE}}}

if [[ "$(uname -m)" == arm* ]] || [[ "$(uname -m)" == aarch* ]]; then
    # Build jemalloc --with-lg-page=16 in order to make the wheel work on both
    # 4k and 64k page arm64 systems. For more context see
    # https://github.com/apache/arrow/issues/10929
    export ARROW_EXTRA_CMAKE_FLAGS="-DARROW_JEMALLOC_LG_PAGE=16"
    : ${ARROW_JEMALLOC:=OFF}
else
    : ${ARROW_JEMALLOC:=ON}
fi

if [[ "${LINUX_WHEEL_KIND:-}" == "musllinux" ]]; then
    : ${CMAKE_INTERPROCEDURAL_OPTIMIZATION:=OFF}
else
    : ${CMAKE_INTERPROCEDURAL_OPTIMIZATION:=ON}
fi

mkdir /tmp/arrow-build
pushd /tmp/arrow-build

cmake \
    -DARROW_ACERO=${ARROW_ACERO} \
    -DARROW_AZURE=${ARROW_AZURE} \
    -DARROW_BUILD_SHARED=ON \
    -DARROW_BUILD_STATIC=OFF \
    -DARROW_BUILD_TESTS=OFF \
    -DARROW_COMPUTE=ON \
    -DARROW_CSV=ON \
    -DARROW_DATASET=${ARROW_DATASET} \
    -DARROW_DEPENDENCY_SOURCE="VCPKG" \
    -DARROW_DEPENDENCY_USE_SHARED=OFF \
    -DARROW_FILESYSTEM=ON \
    -DARROW_FLIGHT=${ARROW_FLIGHT} \
    -DARROW_GANDIVA=${ARROW_GANDIVA} \
    -DARROW_GCS=${ARROW_GCS} \
    -DARROW_HDFS=${ARROW_HDFS} \
    -DARROW_JEMALLOC=${ARROW_JEMALLOC} \
    -DARROW_JSON=ON \
    -DARROW_MIMALLOC=${ARROW_MIMALLOC} \
    -DARROW_ORC=${ARROW_ORC} \
    -DARROW_PACKAGE_KIND="python-wheel-${LINUX_WHEEL_KIND}${LINUX_WHEEL_VERSION}" \
    -DARROW_PARQUET=${ARROW_PARQUET} \
    -DARROW_RPATH_ORIGIN=ON \
    -DARROW_S3=${ARROW_S3} \
    -DARROW_SUBSTRAIT=${ARROW_SUBSTRAIT} \
    -DARROW_TENSORFLOW=${ARROW_TENSORFLOW} \
    -DARROW_USE_CCACHE=ON \
    -DARROW_WITH_BROTLI=${ARROW_WITH_BROTLI} \
    -DARROW_WITH_BZ2=${ARROW_WITH_BZ2} \
    -DARROW_WITH_LZ4=${ARROW_WITH_LZ4} \
    -DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY} \
    -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB} \
    -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD} \
    -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
    -DCMAKE_INSTALL_LIBDIR=lib \
    -DCMAKE_INSTALL_PREFIX=/tmp/arrow-dist \
    -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=${CMAKE_INTERPROCEDURAL_OPTIMIZATION} \
    -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \
    -DPARQUET_REQUIRE_ENCRYPTION=${PARQUET_REQUIRE_ENCRYPTION} \
    -DVCPKG_MANIFEST_MODE=OFF \
    -DVCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET} \
    -Dxsimd_SOURCE=BUNDLED \
    ${ARROW_EXTRA_CMAKE_FLAGS} \
    -G ${CMAKE_GENERATOR} \
    /arrow/cpp
cmake --build . --target install
popd

# Check that we don't expose any unwanted symbols
check_arrow_visibility

echo "=== (${PYTHON_VERSION}) Building wheel ==="
export PYARROW_BUILD_TYPE=${CMAKE_BUILD_TYPE}
export PYARROW_BUNDLE_ARROW_CPP=1
export PYARROW_CMAKE_GENERATOR=${CMAKE_GENERATOR}
export PYARROW_CMAKE_OPTIONS="-DCMAKE_INTERPROCEDURAL_OPTIMIZATION=${CMAKE_INTERPROCEDURAL_OPTIMIZATION}"
export PYARROW_WITH_ACERO=${ARROW_ACERO}
export PYARROW_WITH_AZURE=${ARROW_AZURE}
export PYARROW_WITH_DATASET=${ARROW_DATASET}
export PYARROW_WITH_FLIGHT=${ARROW_FLIGHT}
export PYARROW_WITH_GANDIVA=${ARROW_GANDIVA}
export PYARROW_WITH_GCS=${ARROW_GCS}
export PYARROW_WITH_HDFS=${ARROW_HDFS}
export PYARROW_WITH_ORC=${ARROW_ORC}
export PYARROW_WITH_PARQUET=${ARROW_PARQUET}
export PYARROW_WITH_PARQUET_ENCRYPTION=${PARQUET_REQUIRE_ENCRYPTION}
export PYARROW_WITH_SUBSTRAIT=${ARROW_SUBSTRAIT}
export PYARROW_WITH_S3=${ARROW_S3}
export ARROW_HOME=/tmp/arrow-dist
# PyArrow build configuration
export CMAKE_PREFIX_PATH=/tmp/arrow-dist

pushd /arrow/python
python -m build --sdist --wheel . --no-isolation

echo "=== Strip symbols from wheel ==="
mkdir -p dist/temp-fix-wheel
mv dist/pyarrow-*.whl dist/temp-fix-wheel

pushd dist/temp-fix-wheel
wheel_name=$(ls pyarrow-*.whl)
# Unzip and remove old wheel
unzip $wheel_name
rm $wheel_name
for filename in $(ls pyarrow/*.so pyarrow/*.so.*); do
    echo "Stripping debug symbols from: $filename";
    strip --strip-debug $filename
done
# Zip wheel again after stripping symbols
zip -r $wheel_name .
mv $wheel_name ..
popd

rm -rf dist/temp-fix-wheel

echo "=== (${PYTHON_VERSION}) Tag the wheel with ${LINUX_WHEEL_KIND}${LINUX_WHEEL_VERSION} ==="
auditwheel repair dist/pyarrow-*.whl -w repaired_wheels
popd