1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
|
# Copyright 2013 The Emscripten Authors. All rights reserved.
# Emscripten is available under two separate licenses, the MIT license and the
# University of Illinois/NCSA Open Source License. Both these licenses can be
# found in the LICENSE file.
import contextlib
import logging
import os
from . import tempfiles, filelock, config, utils
from .settings import settings
logger = logging.getLogger('cache')
# Permanent cache for system librarys and ports
class Cache:
# If EM_EXCLUSIVE_CACHE_ACCESS is true, this process is allowed to have direct
# access to the Emscripten cache without having to obtain an interprocess lock
# for it. Generally this is false, and this is used in the case that
# Emscripten process recursively calls to itself when building the cache, in
# which case the parent Emscripten process has already locked the cache.
# Essentially the env. var EM_EXCLUSIVE_CACHE_ACCESS signals from parent to
# child process that the child can reuse the lock that the parent already has
# acquired.
EM_EXCLUSIVE_CACHE_ACCESS = int(os.environ.get('EM_EXCLUSIVE_CACHE_ACCESS', '0'))
def __init__(self, dirname):
# figure out the root directory for all caching
dirname = os.path.normpath(dirname)
self.dirname = dirname
self.acquired_count = 0
# since the lock itself lives inside the cache directory we need to ensure it
# exists.
self.ensure()
self.filelock_name = os.path.join(dirname, 'cache.lock')
self.filelock = filelock.FileLock(self.filelock_name)
def acquire_cache_lock(self):
if config.FROZEN_CACHE:
# Raise an exception here rather than exit_with_error since in practice this
# should never happen
raise Exception('Attempt to lock the cache but FROZEN_CACHE is set')
if not self.EM_EXCLUSIVE_CACHE_ACCESS and self.acquired_count == 0:
logger.debug(f'PID {os.getpid()} acquiring multiprocess file lock to Emscripten cache at {self.dirname}')
try:
self.filelock.acquire(60)
except filelock.Timeout:
# The multiprocess cache locking can be disabled altogether by setting EM_EXCLUSIVE_CACHE_ACCESS=1 environment
# variable before building. (in that case, use "embuilder.py build ALL" to prepopulate the cache)
logger.warning(f'Accessing the Emscripten cache at "{self.dirname}" is taking a long time, another process should be writing to it. If there are none and you suspect this process has deadlocked, try deleting the lock file "{self.filelock_name}" and try again. If this occurs deterministically, consider filing a bug.')
self.filelock.acquire()
self.prev_EM_EXCLUSIVE_CACHE_ACCESS = os.environ.get('EM_EXCLUSIVE_CACHE_ACCESS')
os.environ['EM_EXCLUSIVE_CACHE_ACCESS'] = '1'
logger.debug('done')
self.acquired_count += 1
def release_cache_lock(self):
self.acquired_count -= 1
assert self.acquired_count >= 0, "Called release more times than acquire"
if not self.EM_EXCLUSIVE_CACHE_ACCESS and self.acquired_count == 0:
if self.prev_EM_EXCLUSIVE_CACHE_ACCESS:
os.environ['EM_EXCLUSIVE_CACHE_ACCESS'] = self.prev_EM_EXCLUSIVE_CACHE_ACCESS
else:
del os.environ['EM_EXCLUSIVE_CACHE_ACCESS']
self.filelock.release()
logger.debug(f'PID {os.getpid()} released multiprocess file lock to Emscripten cache at {self.dirname}')
@contextlib.contextmanager
def lock(self):
"""A context manager that performs actions in the given directory."""
self.acquire_cache_lock()
try:
yield
finally:
self.release_cache_lock()
def ensure(self):
utils.safe_ensure_dirs(self.dirname)
def erase(self):
with self.lock():
if os.path.exists(self.dirname):
for f in os.listdir(self.dirname):
tempfiles.try_delete(os.path.join(self.dirname, f))
def get_path(self, name):
return os.path.join(self.dirname, name)
def get_sysroot(self, absolute):
if absolute:
return os.path.join(self.dirname, 'sysroot')
return 'sysroot'
def get_include_dir(self, *parts):
return self.get_sysroot_dir('include', *parts)
def get_sysroot_dir(self, *parts):
return os.path.join(self.get_sysroot(absolute=True), *parts)
def get_lib_dir(self, absolute, varies=True):
path = os.path.join(self.get_sysroot(absolute=absolute), 'lib')
if settings.MEMORY64:
path = os.path.join(path, 'wasm64-emscripten')
else:
path = os.path.join(path, 'wasm32-emscripten')
if not varies:
return path
# if relevant, use a subdir of the cache
subdir = []
if settings.LTO:
if settings.LTO == 'thin':
subdir.append('thinlto')
else:
subdir.append('lto')
if settings.RELOCATABLE:
subdir.append('pic')
if subdir:
path = os.path.join(path, '-'.join(subdir))
return path
def get_lib_name(self, name, varies=True):
return os.path.join(self.get_lib_dir(absolute=False, varies=varies), name)
def erase_lib(self, name):
self.erase_file(self.get_lib_name(name))
def erase_file(self, shortname):
with self.lock():
name = os.path.join(self.dirname, shortname)
if os.path.exists(name):
logger.info(f'deleting cached file: {name}')
tempfiles.try_delete(name)
def get_lib(self, libname, *args, **kwargs):
name = self.get_lib_name(libname)
return self.get(name, *args, **kwargs)
# Request a cached file. If it isn't in the cache, it will be created with
# the given creator function
def get(self, shortname, creator, what=None, force=False):
cachename = os.path.join(self.dirname, shortname)
cachename = os.path.abspath(cachename)
# Check for existence before taking the lock in case we can avoid the
# lock completely.
if os.path.exists(cachename) and not force:
return cachename
if config.FROZEN_CACHE:
# Raise an exception here rather than exit_with_error since in practice this
# should never happen
raise Exception(f'FROZEN_CACHE is set, but cache file is missing: "{shortname}" (in cache root path "{self.dirname}")')
with self.lock():
if os.path.exists(cachename) and not force:
return cachename
if what is None:
if shortname.endswith(('.bc', '.so', '.a')):
what = 'system library'
else:
what = 'system asset'
message = f'generating {what}: {shortname}... (this will be cached in "{cachename}" for subsequent builds)'
logger.info(message)
utils.safe_ensure_dirs(os.path.dirname(cachename))
creator(cachename)
assert os.path.exists(cachename)
logger.info(' - ok')
return cachename
|
