1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
|
#!/usr/bin/python3
import os
import sys
def parseArguments():
helpMessage = """
This sets up the Data directory required for a Shasta run.
The Data directory becomes the mount point of a huge page filesystem.
After this runs, you can use RunAssembly.py to start a Shasta run can be started in the current directory.
Invoke without arguments.
This script uses sudo to invoke commands that requires root privileges,
so depending on your sudo set up it may ask for your password.
"""
if not len(sys.argv) == 1:
print(helpMessage)
exit(1)
# This is used to check that we don't overwrite existing data.
def verifyDirectoryFiles(runDirectory = ''):
# If any of these is present in the run directory, don't do anything.
mustNotExist = ['Data']
for name in mustNotExist:
path = os.path.abspath(os.path.join(runDirectory, name))
if os.path.lexists(path):
print('%s must not exist. Remove it before running this script.' % name)
exit(1)
def setupRunDirectory(runDirectory = ''):
# Define the minimum and maximum amount of huge page memory.
# Leaving the minimum at zero has pro's and con's:
# Good: we don't need to worry about unmounting the
MB = 1024 * 1024
GB = 1024 * MB
totalSystemMemoryBytes = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')
minimumHugePageMemoryBytes = 0 # Consider increasing this
maximumHugePageMemoryBytes = totalSystemMemoryBytes - 8 * GB
maximumHugePageMemoryBytes = max(maximumHugePageMemoryBytes, minimumHugePageMemoryBytes)
hugePageSize = 2 * MB
minimumHugePageMemoryHugePages = int(minimumHugePageMemoryBytes / hugePageSize)
maximumHugePageMemoryHugePages = int(maximumHugePageMemoryBytes / hugePageSize)
os.system('sudo sh -c "echo %s > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages"'
% minimumHugePageMemoryHugePages)
os.system('sudo sh -c "echo %s > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_overcommit_hugepages"'
% maximumHugePageMemoryHugePages)
# Create the Data directory.
dataPath = os.path.abspath(os.path.join(runDirectory, 'Data'))
os.mkdir(dataPath)
# Mount the huge page filesystem.
command = 'sudo mount -t hugetlbfs -o uid=%s,gid=%s,pagesize=2M none %s' % (os.getuid(), os.getgid(), dataPath)
os.system(command)
def main():
parseArguments()
# Check that we don't overwrite existing data.
verifyDirectoryFiles()
# Create the Data and threadlogs directories.
setupRunDirectory()
if __name__ == "__main__":
main()
|
