1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567
|
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import json
import logging
import os
import shutil
import subprocess
import sys
from pathlib import Path
logger = logging.getLogger(__name__)
class CCAppBundleConfig:
output_dependencies: bool
embed_python: bool
cc_bin_path: Path
extra_pathlib: Path
frameworks_path: Path
plugin_path: Path
embedded_python_rootpath: Path
python_version: str # pythonMajor.Minor
base_python_binary: Path # prefix/bin/python
base_python_libs: Path # prefix/lib/pythonMajor.Minor
def __init__(
self,
install_path: Path,
extra_pathlib: Path,
output_dependencies: bool,
embed_python: bool,
) -> None:
"""Construct a configuration.
Args:
----
install_path (str): Path where CC is "installed".
extra_pathlib (str): A Path where additional libs can be found.
output_dependencies (bool): boolean that control the level of debug. If true some extra
files will be created (macos_bundle_warnings.json macos_bundle_dependencies.json).
embed_python (bool): Whether or not python should be embedded into the bundle.
"""
self.output_dependencies = output_dependencies
self.bundle_abs_path = (install_path / "CloudCompare" / "CloudCompare.app").absolute()
self.cc_bin_path = self.bundle_abs_path / "Contents" / "MacOS" / "CloudCompare"
self.extra_pathlib = extra_pathlib
self.frameworks_path = self.bundle_abs_path / "Contents" / "Frameworks"
self.plugin_path = self.bundle_abs_path / "Contents" / "PlugIns"
# If we want to embed Python we populate the needed variables
self.embed_python = embed_python
if embed_python:
self._query_python()
self.embedded_python_rootpath = self.bundle_abs_path / "Contents" / "Resources" / "python"
self.embedded_python_path = self.embedded_python_rootpath / "bin"
self.embedded_python_binary = self.embedded_python_path / "python"
self.embedded_python_libpath = self.embedded_python_rootpath / "lib"
self.embedded_python_lib = self.embedded_python_libpath / f"python{self.python_version}"
self.embedded_python_site_package = self.embedded_python_lib / "site-packages"
def __str__(self) -> str:
"""Return a string representation of the class."""
res = (
f"--- Frameworks path: {self.frameworks_path} \n"
f" --- plugin path: {self.plugin_path} \n"
f" --- embeddedPythonPath: {self.embedded_python_path} \n"
f" --- embeddedPython: {self.embedded_python_binary} \n"
f" --- embeddedPythonLibPath: {self.embedded_python_libpath} \n"
f" --- embeddedPythonLib: {self.embedded_python_lib} \n"
f" --- embeddedPythonSiteLibs: {self.embedded_python_site_package} \n"
)
return res
def _query_python(self):
"""Query for python paths and configuration."""
self.python_version = f"{sys.version_info.major}.{sys.version_info.minor}"
self.base_python_binary = Path(sys.exec_prefix) / "bin" / "python"
self.base_python_libs = Path(sys.exec_prefix) / "lib" / f"python{self.python_version}"
class CCBundler:
config: CCAppBundleConfig
# dictionary of lib dependencies : key depends on (list of libs) (not recursive)
dependencies: dict[str, list[str]] = dict()
warnings: dict[str, list[str]] = dict()
def __init__(self, config: CCAppBundleConfig) -> None:
"""Construct a CCBundler object"""
self.config = config
def bundle(self) -> None:
"""Bundle the dependencies into the .app"""
if config.embed_python:
self._embed_python()
libs_found, libs_ex_found, libs_in_plugins = self._collect_dependencies()
self._embed_libraries(libs_found, libs_ex_found, libs_in_plugins)
# output debug files if needed
if self.config.output_dependencies:
logger.info("write debug files (macos_bundle_dependencies.json and macos_bundle_warnings.json)")
with open(
Path.cwd() / "macos_bundle_dependencies.json",
"w",
encoding="utf-8",
) as f:
json.dump(self.dependencies, f, sort_keys=True, indent=4)
with open(
Path.cwd() / "macos_bundle_warnings.json",
"w",
encoding="utf-8",
) as f:
json.dump(self.warnings, f, sort_keys=True, indent=4)
def _get_lib_dependencies(self, mainlib: Path) -> tuple[list[str], list[str]]:
"""List dependencies of mainlib (using otool -L).
We only look for dependencies with @rpath and @executable_path.
We consider @executable_path being relative to the CloudCompare executable.
We keep record and debug /usr and /System for debug purposes.
Args:
----
mainlib (Path): Path to a binary (lib, executable)
Returns:
-------
libs (list[Path]): lib @rpath or @executable_path
lib_ex (list[(Path, Path)]): lib @executable_path
"""
libs: list[Path] = []
lib_ex: list[Path] = []
warning_libs = []
with subprocess.Popen(["otool", "-L", str(mainlib)], stdout=subprocess.PIPE) as proc:
lines = proc.stdout.readlines()
logger.debug(mainlib)
lines.pop(0) # Drop the first line as it contains the name of the lib / binary
# now first line is LC_ID_DYLIB (should be @rpath/libname)
for line in lines:
vals = line.split()
if len(vals) < 2:
continue
pathlib = vals[0].decode()
logger.debug("->pathlib: %s", pathlib)
if pathlib == self.config.extra_pathlib:
logger.info("%s lib from additional extra pathlib", mainlib)
libs.append(Path(pathlib))
continue
dirs = pathlib.split("/")
# TODO: should be better with startswith
# we are likely to have only @rpath values
if dirs[0] == "@rpath":
libs.append(Path(dirs[1]))
elif dirs[0] == "@loader_path":
logger.warning("%s declares a dependencies with @loader_path, this won't be resolved", mainlib)
elif dirs[0] == "@executable_path":
logger.warning("%s declares a dependencies with @executable_path", mainlib)
# TODO: check if mainlib is in the bundle in order to be sure that
# the executable path is relative to the application
lib_ex.append(
(
mainlib.name,
Path(pathlib.removeprefix("@executable_path/")),
),
)
elif (dirs[1] != "usr") and (dirs[1] != "System"):
logger.warning("%s depends on undeclared pathlib: %s", mainlib, pathlib)
self.warnings[str(mainlib)] = str(warning_libs)
self.dependencies[mainlib.name] = str(libs)
return libs, lib_ex
@staticmethod
def _get_rpath(binary_path: Path) -> list[str]:
"""Retrieve paths stored in LC_RPATH part of the binary.
Paths are expected to be in the form @loader_path/xxx, @executable_path/xxx, or abs/relative paths
Args:
----
binary_path (Path): Path to a binary (lib, executable)
Returns:
-------
list[str]: rpath list (string representation)
"""
rpaths = []
with subprocess.Popen(["otool", "-l", str(binary_path)], stdout=subprocess.PIPE) as proc:
lines = proc.stdout.readlines()
for line in lines:
res = line.decode()
vals = res.split()
if len(vals) > 1 and vals[0] == "path":
rpaths.append(vals[1])
return rpaths
@staticmethod
def _convert_rpaths(binary_path: Path, rpaths: list[str]) -> list[Path]:
"""Convert rpaths to absolute paths.
Given a path to a binary (lib, executable) and a list of rpaths, resolve rpaths
and append binary_path to them in order to create putative absolute path to this binary
Args:
----
binary_path (Path): string representation of the path to a binary (lib, executable)
rpaths (list[str]): List of string representation of rpaths
Returns:
-------
list[Path]: list of putative full / absolute path to the binary
"""
dirname_binary = binary_path.parent
abs_paths = []
for rpath in rpaths:
if "@loader_path" in rpath:
vals = rpath.split("/")
abs_path = dirname_binary
if len(vals) > 1:
abs_path = (abs_path / Path("/".join(vals[1:]))).resolve()
else:
# TODO: test if it's an absolute path
abs_path = Path(rpath).resolve()
abs_paths.append(abs_path)
return abs_paths
def _copy_python_env(self) -> None:
"""Copy python environment.
Ideally this should be handled by CCPython-Runtime CMake script like in Windows.
"""
logger.info("Python: copy distribution in package")
try:
self.config.embedded_python_path.mkdir(parents=True)
self.config.embedded_python_libpath.mkdir()
except OSError:
logger.error(
"Python dir already exists in bundle, please clean your bundle and rerun this script",
)
sys.exit(1)
shutil.copytree(self.config.base_python_libs, self.config.embedded_python_lib)
shutil.copy2(self.config.base_python_binary, self.config.embedded_python_binary)
def _embed_python(self) -> None:
"""Embed python distribution dependencies in site-packages.
It copies the pyhton target distribution into the `.app` bundle
and then it collects dependencies and rewrites rpaths
of all the binaries/libraries found inside the distribution's tree.
"""
libs_to_check = [self.config.embedded_python_binary]
# results
libs_found = set()
lib_ex_found = set()
python_libs = set() # Lib in python dir
self._copy_python_env()
# --- enumerate all libs inside the dir
# Path.walk() is python 3.12+
for root, _, files in os.walk(self.config.embedded_python_lib):
for name in files:
ext = Path(name).suffix
if ext in (".dylib", ".so"):
library = self.config.embedded_python_lib / root / name
libs_to_check.append(library)
python_libs.add(library)
logger.info("number of libs (.so and .dylib) in embedded Python: %i", len(python_libs))
while len(libs_to_check):
lib2check = libs_to_check.pop(0)
if lib2check in libs_found:
continue
libs_found.add(lib2check)
libs, lib_ex = self._get_lib_dependencies(lib2check)
lib_ex_found.update(lib_ex)
rpaths = CCBundler._get_rpath(lib2check)
abs_rpaths = CCBundler._convert_rpaths(lib2check, rpaths)
if self.config.extra_pathlib not in abs_rpaths:
abs_rpaths.append(self.config.extra_pathlib)
for lib in libs:
if lib.is_absolute():
if lib not in libs_to_check and lib not in libs_found:
libs_to_check.append(lib)
else:
for abs_rp in abs_rpaths:
abs_lib = abs_rp / lib
if abs_lib.is_file():
if abs_lib not in libs_to_check and abs_lib not in libs_found:
libs_to_check.append(abs_lib)
break
logger.info("lib_ex_found to add to Frameworks: %i", len(lib_ex_found))
logger.info("libs_found to add to Frameworks: %i", len(libs_found))
libs_in_framework = set(self.config.frameworks_path.iterdir())
added_to_framework_count = 0
for lib in libs_found:
if lib == self.config.embedded_python_binary: # if it's the Python binary we continue
continue
base = self.config.frameworks_path / lib.name
if base not in libs_in_framework and lib not in python_libs:
shutil.copy2(
lib,
self.config.frameworks_path,
) # copy libs that are not in framework yet
added_to_framework_count = added_to_framework_count + 1
logger.info("libs added to Frameworks: %i", {added_to_framework_count})
logger.info(" --- Python libs: set rpath to Frameworks, nb libs: %i", len(python_libs))
# Set the rpath to the Frameworks path
# TODO: remove old rpath
deep_sp = len(self.config.embedded_python_lib.parents)
for file in python_libs:
deep_lib_sp = len(file.parents) - deep_sp
rpath = "@loader_path/../../../"
for _ in range(deep_lib_sp):
rpath += "../"
rpath += "Frameworks"
subprocess.run(
["install_name_tool", "-add_rpath", rpath, str(file)],
check=False,
)
def _collect_dependencies(self):
"""Collect dependencies of CloudCompare binary and QT libs
Returns
-------
set[Path]: Libs and binaries found in the collect process.
set[(Path, Path)]: Libs and binaries found with an @executable_path dependency.
set[Path]: Libs and binaries found in the plugin dir.
"""
# Searching for CC dependencies
libs_to_check = []
# results
libs_found = set() # Abs path of libs/binaries already checked, candidate for embedding in the bundle
lib_ex_found = set()
libs_in_plugins = set()
logger.info("Adding main executable to the libs to check")
libs_to_check.append(self.config.cc_bin_path)
logger.info("Adding lib already available in Frameworks to the libsToCheck")
for file_path in self.config.frameworks_path.iterdir():
libs_to_check.append(file_path)
logger.info("number of libs already in Frameworks directory: %i", len(libs_to_check))
logger.info("Adding plugins to the libs to check")
for plugin_dir in self.config.plugin_path.iterdir():
if plugin_dir.is_dir() and plugin_dir.suffix != ".app":
for file in plugin_dir.iterdir():
if file.is_file() and file.suffix in (".dylib", ".so"):
libs_to_check.append(file)
libs_in_plugins.add(file)
logger.info("number of libs in PlugIns directory: %i", len(libs_in_plugins))
logger.info("searching for dependencies...")
while len(libs_to_check):
# --- Unstack a binary/lib from the libs_to_check array
lib2check = libs_to_check.pop(0)
# If the lib was already processed we continue, of course
if lib2check in libs_found:
continue
# Add the current lib to the already processed libs
libs_found.add(lib2check)
# search for @rpath and @executable_path dependencies in the current lib
lib_deps, lib_ex = self._get_lib_dependencies(lib2check)
# @executable_path are handled in a seperate set
lib_ex_found.update(lib_ex)
# TODO: group these two functions since we do not need
# get all rpath for the current lib
rpaths_str = CCBundler._get_rpath(lib2check)
# get absolute path from found rpath
abs_search_paths = CCBundler._convert_rpaths(lib2check, rpaths_str)
# If the extra_pathlib is not already added, we ad it
# TODO:: there is no way it can be False
# maybe we should prefer to check for authorized lib_dir
# TODO: if rpath is @loader_path, LIB is either in frameworks (already embedded) or in extra_pathlib
# we can take advantage of that...
if self.config.extra_pathlib not in abs_search_paths:
abs_search_paths.append(self.config.extra_pathlib)
# TODO: check if exists, else throw and exception
for dependency in lib_deps:
for abs_rp in abs_search_paths:
abslib_path = abs_rp / dependency
if abslib_path.is_file():
if abslib_path not in libs_to_check and abslib_path not in libs_found:
# if this lib was not checked for dependencies yet, we append it to the list of lib to check
libs_to_check.append(abslib_path)
break
# TODO: handle lib_ex here
# for dependency in lib_ex:...
# TODO: add to libTOcheck executable_path/dep
return libs_found, lib_ex_found, libs_in_plugins
def _embed_libraries(
self,
libs_found: set[Path],
lib_ex_found: set[(Path, Path)],
libs_in_plugins: set[Path],
) -> None:
"""Embed collected libraries into the `.app` bundle.
rpath of embedded libs is modified to match their new location
Args:
----
libs_found (set[Path]): libs and binaries found in the collect process.
libs_ex_found (set[(Path, Path)]): libs and binaries found with an @executable_path dependency.
libs_found (set[Path]): libs and binaries found in the plugin dir.
"""
logger.info("Copying libraries")
logger.info("lib_ex_found to add to Frameworks: %i", len(lib_ex_found))
logger.info("libs_found to add to Frameworks: %i", len(libs_found))
libs_in_frameworks = set(self.config.frameworks_path.iterdir())
nb_libs_added = 0
for lib in libs_found:
if lib == self.config.cc_bin_path:
continue
base = self.config.frameworks_path / lib.name
if (base not in libs_in_frameworks) and (lib not in libs_in_plugins):
shutil.copy2(lib, self.config.frameworks_path)
nb_libs_added += 1
logger.info("number of libs added to Frameworks: %i", {nb_libs_added})
# --- ajout des rpath pour les libraries du framework : framework et ccPlugins
logger.info(" --- Frameworks libs: add rpath to Frameworks")
nb_frameworks_libs = 0
# TODO: purge old rpath
for file in self.config.frameworks_path.iterdir():
if file.is_file() and file.suffix in (".so", ".dylib"):
nb_frameworks_libs += 1
subprocess.run(
["install_name_tool", "-add_rpath", "@loader_path", str(file)],
stdout=subprocess.PIPE,
check=False,
)
logger.info("number of Frameworks libs with rpath modified: %i", nb_frameworks_libs)
logger.info(" --- PlugIns libs: add rpath to Frameworks, number of libs: %i", len(libs_in_plugins))
for file in libs_in_plugins:
if file.is_file():
subprocess.run(
["install_name_tool", "-add_rpath", "@loader_path/../../Frameworks", str(file)],
stdout=subprocess.PIPE,
check=False,
)
# TODO: make a function for this
# Embed libs with an @executable_path dependencies
for lib_ex in lib_ex_found:
base = lib_ex[0]
target = lib_ex[1]
framework_path = self.config.frameworks_path / base
plugin_path = self.config.plugin_path / "ccPlugins" / base
if framework_path.is_file():
base_path = framework_path
elif plugin_path.is_file():
base_path = plugin_path
else:
# This should not be possible
raise Exception("no base path")
sys.exit(1)
logger.info("modify : @executable_path -> @rpath: %s", base_path)
subprocess.run(
[
"install_name_tool",
"-change",
"@executable_path/" + str(target),
"@rpath/" + str(target),
str(base_path),
],
stdout=subprocess.PIPE,
check=False,
)
if __name__ == "__main__":
# configure logger
formatter = " BundleCC::%(levelname)-8s:: %(message)s"
logging.basicConfig(level=logging.INFO, format=formatter)
std_handler = logging.StreamHandler()
# CLI parser
parser = argparse.ArgumentParser("CCAppBundle")
parser.add_argument(
"install_path",
help="Path where the CC application is installed (CMake install dir)",
type=Path,
)
parser.add_argument(
"--extra_pathlib",
help="Extra path to find libraries (default to $CONDA_PREFIX/lib)",
type=Path,
)
parser.add_argument(
"--embed_python",
help="Whether embedding python or not",
action="store_true",
)
parser.add_argument(
"--output_dependencies",
help="Output a json files in order to debug dependency graph",
action="store_true",
)
arguments = parser.parse_args()
# convert extra_pathlib to absolute paths
if arguments.extra_pathlib is not None:
extra_pathlib = arguments.extra_pathlib.resolve()
else:
conda_prefix = os.environ.get("CONDA_PREFIX")
if conda_prefix is not None:
extra_pathlib = (Path(conda_prefix) / "lib").resolve()
else:
logger.error(
"Unable to find CONDA_PREFIX system variable, please run this script inside a conda environment or use the extra_pathlib option.",
)
sys.exit(1)
config = CCAppBundleConfig(
arguments.install_path,
extra_pathlib,
arguments.output_dependencies,
arguments.embed_python,
)
bundler = CCBundler(config)
bundler.bundle()
|