1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
|
package kmer;
import java.util.Arrays;
import shared.Primes;
import shared.Shared;
import shared.Tools;
import structures.IntList;
public class ScheduleMaker {
public ScheduleMaker(int ways_, int bytesPerKmer_, boolean prealloc_, double memRatio_){
this(ways_, bytesPerKmer_, prealloc_, memRatio_, 0, 0, 0, 0);
}
public ScheduleMaker(int ways_, int bytesPerKmer_, boolean prealloc_, double memRatio_, int initialSize_){
this(ways_, bytesPerKmer_, prealloc_, memRatio_, initialSize_, 0, 0, 0);
}
public ScheduleMaker(int ways_, int bytesPerKmer_, boolean prealloc_, double memRatio_, int initialSize_,
int prepasses_, double prefilterFraction_, long filterMemoryOverride_){
bytesPerKmer=bytesPerKmer_;
prealloc=prealloc_;
memRatio=(float)(memRatio_<=0 ? 1 : memRatio_);
prepasses=prepasses_;
prefilter=(prepasses>0);
prefilterFraction=prefilter ? prefilterFraction_ : 0;
assert(prefilter==prefilterFraction>0) : prefilter+", "+prefilterFraction_+", "+prefilterFraction;
// assert(false && prefilter==prefilterFraction>0) : prefilter+", "+prefilterFraction_+", "+prefilterFraction;
if(prepasses<1){
filterMemory0=filterMemory1=0;
}else if(filterMemoryOverride>0){
filterMemory0=filterMemory1=filterMemoryOverride;
}else{
double low=Tools.min(prefilterFraction, 1-prefilterFraction);
double high=1-low;
if(prepasses<0 || (prepasses&1)==1){//odd passes
filterMemory0=(long)(usableMemory*low);
filterMemory1=(long)(usableMemory*high);
}else{//even passes
filterMemory0=(long)(usableMemory*high);
filterMemory1=(long)(usableMemory*low);
}
}
tableMemory=(long)(usableMemory*.95-Tools.min(filterMemory0, filterMemory1));
if(ways_<1){
long maxKmers=(2*tableMemory)/bytesPerKmer;
long minWays=Tools.min(10000, maxKmers/Integer.MAX_VALUE);
ways_=(int)Tools.max(31, (int)(Tools.min(96, Shared.threads())*2.5), minWays);
ways_=(int)Primes.primeAtLeast(ways_);
assert(ways_>0);
// System.err.println("ways="+ways_);
}
ways=ways_;
final double maxSize0=(tableMemory*0.95*memRatio)/(bytesPerKmer*ways);
assert(maxPrime>1 && maxSize0>2) :
"\nmaxPrime="+maxPrime+", maxSize0="+maxSize0+", tableMemory="+tableMemory+", usableMemory="+usableMemory+
", \nprepasses="+prepasses+", filterMemory0="+filterMemory0+", filterMemory1="+filterMemory1+", prefilterFraction="+prefilterFraction+
", \nmemRatio="+memRatio+", bytesPerKmer="+bytesPerKmer+", ways="+ways+
", \ninitialSize="+initialSize_+", initialSizeDefault="+initialSizeDefault+", prealloc="+prealloc;
lastSizeFraction=prealloc ? 1.0 : resizeMult/(1.0+resizeMult);
maxSize=Primes.primeAtMost((int)Tools.min(maxPrime, maxSize0*lastSizeFraction));
initialSize=(prealloc ? maxSize : Primes.primeAtMost(initialSize_>0 ? initialSize_ : initialSizeDefault));
estimatedKmerCapacity=(long)(maxSize*HashArray.maxLoadFactorFinal*0.97*ways);
// System.err.println(Arrays.toString(makeSchedule()));
// System.err.println("ways="+ways+", maxSize="+maxSize+", estimatedKmerCapacity="+estimatedKmerCapacity+", "+Arrays.toString(makeSchedule()));
//
// assert(false) :
// "\nmaxPrime="+maxPrime+", maxSize0="+maxSize0+", tableMemory="+tableMemory+", usableMemory="+usableMemory+
// ", \nprepasses="+prepasses+", filterMemory0="+filterMemory0+", filterMemory1="+filterMemory1+", prefilterFraction="+prefilterFraction+
// ", \nmemRatio="+memRatio+", bytesPerKmer="+bytesPerKmer+", ways="+ways+
// ", \ninitialSize="+initialSize_+", initialSizeDefault="+initialSizeDefault+", prealloc="+prealloc+
// ", \nmaxSize="+maxSize+", initialSize="+initialSize+", estimatedKmerCapacity="+estimatedKmerCapacity+
// ", \n"+Arrays.toString(makeSchedule());
// assert(false) : Arrays.toString(makeSchedule());
}
public int[] makeSchedule(){
if(prealloc || maxSize<2L*initialSize){return new int[] {maxSize};}
IntList list=new IntList(10);
list.add(maxSize);
for(double x=maxSize*invResizeMult; x>=initialSize; x=x*invResizeMult2){
list.add((int)x);
}
if(list.size()>1 && list.lastElement()>=2*initialSize){
list.add(initialSize);
}
list.reverse();
// if(list.lastElement()*2L<maxPrime){list.add(2*maxSize);}//This ensures that the program will crash rather than garbage-collecting for a long time
int[] array=list.toArray();
// if(initialSize>2 && array.length>2 && array[0]>initialSize){array[0]=initialSize;}
assert(Tools.isSorted(array)) : Arrays.toString(array);
for(int i=0; i<array.length; i++){
array[i]=array[i]==1 ? 1 : (int)Tools.min(maxPrime, Primes.primeAtLeast(array[i]));
}
return array;
}
// public static int[] makeScheduleStatic(int initialSize, int maxSize, boolean autoResize){
// if(!autoResize || initialSize>=maxSize){return null;}
// IntList list=new IntList(10);
// list.add((int)(maxSize*0.8));
// for(long x=maxSize; x>=initialSize; x=x/5){
// list.add((int)x);
// }
// if(list.size()>1 && list.lastElement()>=2*initialSize){
// list.add(initialSize);
// }
// list.reverse();
// int[] array=list.toArray();
// for(int i=0; i<array.length; i++){
// array[i]=array[i]==1 ? 1 : (int)Tools.min(maxPrime, Primes.primeAtLeast(array[i]));
// }
// return array;
// }
final double resizeMult=5.0;
final double resizeMult2=3.0;
final double invResizeMult=1/resizeMult;
final double invResizeMult2=1/resizeMult2;
final double lastSizeFraction;
final long memory=Runtime.getRuntime().maxMemory();
final double xmsRatio=Shared.xmsRatio();
//TODO: Add term for JDK (Oracle/Open) and version.
final long usableMemory=(long)Tools.max(((memory-96000000)*
(xmsRatio>0.97 ? 0.82 : 0.72)), memory*0.45);
final long filterMemoryOverride=0;
final long filterMemory0, filterMemory1;
final long tableMemory;
final double prefilterFraction;
final int prepasses;
final boolean prefilter;
final int bytesPerKmer;
public final long estimatedKmerCapacity;
public final int ways;
final int initialSize;
final int maxSize;
final boolean prealloc;
final float memRatio;
static final int initialSizeDefault=128000;
static final int maxPrime=(int)Primes.primeAtMost(Integer.MAX_VALUE-100-20);
}
|
