1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220
|
#!/usr/bin/env python3
# Copyright 2014 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# This script computs the number of concurrent links we want to run in the build
# as a function of machine spec. It's based on GetDefaultConcurrentLinks in GYP.
import argparse
import multiprocessing
import os
import re
import subprocess
import sys
sys.path.insert(1, os.path.join(os.path.dirname(__file__), '..'))
import gn_helpers
def _GetMemoryMaxInCurrentCGroup(explanation):
with open("/proc/self/cgroup") as cgroup:
lines = cgroup.readlines()
if len(lines) >= 1:
cgroupname = lines[0].strip().split(':')[-1]
memmax = '/sys/fs/cgroup' + cgroupname + '/memory.max'
if os.path.exists(memmax):
with open(memmax) as f:
data = f.read().strip()
explanation.append(f'# cgroup {cgroupname} memory.max={data}')
try:
return int(data)
except ValueError as ex:
explanation.append(f'# cgroup memory.max exception {ex}')
return None
explanation.append(f'# cgroup memory.max not found')
return None
def _GetCPUCountFromCurrentCGroup(explanation):
with open("/proc/self/cgroup") as cgroup:
lines = cgroup.readlines()
if len(lines) >= 1:
cgroupname = lines[0].strip().split(':')[-1]
cpuset = '/sys/fs/cgroup' + cgroupname + '/cpuset.cpus'
if os.path.exists(cpuset):
with open(cpuset) as f:
data = f.read().strip()
explanation.append(f'# cgroup {cgroupname} cpuset.cpus={data}')
try:
return _CountCPUs(data)
except ValueError as ex:
explanation.append(f'# cgroup cpuset.cpus exception {ex}')
return None
explanation.append(f'# cgroup cpuset.cpus not found')
return None
def _CountCPUs(cpuset):
n = 0
for s in cpuset.split(','):
r = s.split('-')
if len(r) == 1 and int(r[0]) >= 0:
n += 1
continue
elif len(r) == 2:
n += int(r[1]) - int(r[0]) + 1
else:
# wrong range?
return 0
return n
def _GetTotalMemoryInBytes(explanation):
if sys.platform in ('win32', 'cygwin'):
import ctypes
class MEMORYSTATUSEX(ctypes.Structure):
_fields_ = [
("dwLength", ctypes.c_ulong),
("dwMemoryLoad", ctypes.c_ulong),
("ullTotalPhys", ctypes.c_ulonglong),
("ullAvailPhys", ctypes.c_ulonglong),
("ullTotalPageFile", ctypes.c_ulonglong),
("ullAvailPageFile", ctypes.c_ulonglong),
("ullTotalVirtual", ctypes.c_ulonglong),
("ullAvailVirtual", ctypes.c_ulonglong),
("sullAvailExtendedVirtual", ctypes.c_ulonglong),
]
stat = MEMORYSTATUSEX(dwLength=ctypes.sizeof(MEMORYSTATUSEX))
ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat))
return stat.ullTotalPhys
elif sys.platform.startswith('linux'):
if os.path.exists("/proc/self/cgroup"):
memmax = _GetMemoryMaxInCurrentCGroup(explanation)
if memmax:
return memmax
if os.path.exists("/proc/meminfo"):
with open("/proc/meminfo") as meminfo:
memtotal_re = re.compile(r'^MemTotal:\s*(\d*)\s*kB')
for line in meminfo:
match = memtotal_re.match(line)
if not match:
continue
return float(match.group(1)) * 2**10
elif sys.platform == 'darwin':
try:
return int(subprocess.check_output(['sysctl', '-n', 'hw.memsize']))
except Exception:
return 0
# TODO(scottmg): Implement this for other platforms.
return 0
def _GetDefaultConcurrentLinks(per_link_gb, reserve_gb, thin_lto_type,
secondary_per_link_gb, override_ram_in_gb):
explanation = []
explanation.append(
'per_link_gb={} reserve_gb={} secondary_per_link_gb={}'.format(
per_link_gb, reserve_gb, secondary_per_link_gb))
if override_ram_in_gb:
mem_total_gb = override_ram_in_gb
else:
mem_total_gb = float(_GetTotalMemoryInBytes(explanation)) / 2**30
adjusted_mem_total_gb = max(0, mem_total_gb - reserve_gb)
# Ensure that there is at least as many links allocated for the secondary as
# there is for the primary. The secondary link usually uses fewer gbs.
mem_cap = int(
max(1, adjusted_mem_total_gb / (per_link_gb + secondary_per_link_gb)))
cpu_count = None
if sys.platform.startswith('linux'):
try:
if os.path.exists('/proc/self/cgroup'):
cpu_count = _GetCPUCountFromCurrentCGroup(explanation)
except Exception as ex:
explanation.append(f'# cpu_count from cgroup exception {ex}')
if not cpu_count:
try:
cpu_count = multiprocessing.cpu_count()
explanation.append(f'# cpu_count from multiprocessing {cpu_count}')
except Exception as ex:
cpu_count = 1
explanation.append(f'# cpu_count from multiprocessing exception {ex}')
# A local LTO links saturate all cores, but only for some amount of the link.
cpu_cap = cpu_count
if thin_lto_type is not None:
assert thin_lto_type == 'local'
cpu_cap = min(cpu_count, 6)
explanation.append(f'cpu_count={cpu_count} cpu_cap={cpu_cap} ' +
f'mem_total_gb={mem_total_gb:.1f}GiB ' +
f'adjusted_mem_total_gb={adjusted_mem_total_gb:.1f}GiB')
num_links = min(mem_cap, cpu_cap)
if num_links == cpu_cap:
if cpu_cap == cpu_count:
reason = 'cpu_count'
else:
reason = 'cpu_cap (thinlto)'
else:
reason = 'RAM'
# static link see too many open files if we have many concurrent links.
# ref: http://b/233068481
if num_links > 30:
num_links = 30
reason = 'nofile'
explanation.append('concurrent_links={} (reason: {})'.format(
num_links, reason))
# Use remaining RAM for a secondary pool if needed.
if secondary_per_link_gb:
mem_remaining = adjusted_mem_total_gb - num_links * per_link_gb
secondary_size = int(max(0, mem_remaining / secondary_per_link_gb))
if secondary_size > cpu_count:
secondary_size = cpu_count
reason = 'cpu_count'
else:
reason = 'mem_remaining={:.1f}GiB'.format(mem_remaining)
explanation.append('secondary_size={} (reason: {})'.format(
secondary_size, reason))
else:
secondary_size = 0
return num_links, secondary_size, explanation
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--mem_per_link_gb', type=int, default=8)
parser.add_argument('--reserve_mem_gb', type=int, default=0)
parser.add_argument('--secondary_mem_per_link', type=int, default=0)
parser.add_argument('--override-ram-in-gb-for-testing', type=float, default=0)
parser.add_argument('--thin-lto')
options = parser.parse_args()
primary_pool_size, secondary_pool_size, explanation = (
_GetDefaultConcurrentLinks(options.mem_per_link_gb,
options.reserve_mem_gb, options.thin_lto,
options.secondary_mem_per_link,
options.override_ram_in_gb_for_testing))
if options.override_ram_in_gb_for_testing:
print('primary={} secondary={} explanation={}'.format(
primary_pool_size, secondary_pool_size, explanation))
else:
sys.stdout.write(
gn_helpers.ToGNString({
'primary_pool_size': primary_pool_size,
'secondary_pool_size': secondary_pool_size,
'explanation': explanation,
}))
return 0
if __name__ == '__main__':
sys.exit(main())
|