1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222
|
package cardinality;
import java.util.concurrent.atomic.AtomicIntegerArray;
import shared.Parser;
import shared.Tools;
import structures.LongList;
/**
* @author Brian Bushnell
* @date Sep 30, 2015
*
*/
public final class LogLog extends CardinalityTracker {
/*--------------------------------------------------------------*/
/*---------------- Initialization ----------------*/
/*--------------------------------------------------------------*/
/** Create a LogLog with default parameters */
LogLog(){
this(2048, 31, -1, 0);
}
/** Create a LogLog with parsed parameters */
LogLog(Parser p){
super(p);
//assert(atomic);
maxArrayA=(atomic ? new AtomicIntegerArray(buckets) : null);
maxArray=(atomic ? null : new int[buckets]);
}
/**
* Create a LogLog with specified parameters
* @param buckets_ Number of buckets (counters)
* @param k_ Kmer length
* @param seed Random number generator seed; -1 for a random seed
* @param minProb_ Ignore kmers with under this probability of being correct
*/
LogLog(int buckets_, int k_, long seed, float minProb_){
super(buckets_, k_, seed, minProb_);
//assert(atomic);
maxArrayA=(atomic ? new AtomicIntegerArray(buckets) : null);
maxArray=(atomic ? null : new int[buckets]);
}
/*--------------------------------------------------------------*/
/*---------------- Methods ----------------*/
/*--------------------------------------------------------------*/
//Restores floating point to integer
private long restore(int score){
int leading=score;
long mantissa=1;
int shift=64-leading-1;
long original=mantissa<<shift;
return original;
}
@Override
public final long cardinality(){
double difSum=0;
double estLogSum=0;
int count=0;
LongList list=new LongList(buckets);
//assert(atomic);
if(atomic){
for(int i=0; i<maxArrayA.length(); i++){
int max=maxArrayA.get(i);
long val=restore(max);
if(max>0 && val>0){
// long val=restore(max);
// System.err.println("val="+val);
final long dif=val;
difSum+=dif;
count++;
double est=2*(Long.MAX_VALUE/(double)dif)*SKIPMOD;
estLogSum+=Math.log(est);
list.add(dif);
}
}
}else{
for(int i=0; i<maxArray.length; i++){
int max=maxArray[i];
long val=restore(max);
if(max>0 && val>0){
// long val=restore(max);
final long dif=val;
difSum+=dif;
count++;
double est=2*(Long.MAX_VALUE/(double)dif)*SKIPMOD;
estLogSum+=Math.log(est);
list.add(dif);
}
}
}
final int div=count;//Could also be count be that causes problems
final double mean=difSum/Tools.max(div, 1);
list.sort();
final long median=list.median();
final double mwa=list.medianWeightedAverage();
//What to use as the value from the counters
final double proxy=mean;
// assert(false) : mean+", "+median+", "+difSum+", "+list;
final double estimatePerSet=2*(Long.MAX_VALUE/proxy)*SKIPMOD;
final double conversionFactor=0.7213428177;
final double total=conversionFactor*estimatePerSet*div*((count+buckets)/(float)(buckets+buckets));
// final double estSum=div*Math.exp(estLogSum/(Tools.max(div, 1)));
// double medianEst=2*(Long.MAX_VALUE/(double)median)*SKIPMOD*div;
// new Exception().printStackTrace();
// System.err.println(maxArray);
//// Overall, it looks like "total" is the best, then "estSum", then "medianEst" is the worst, in terms of variance.
// System.err.println("difSum="+difSum+", count="+count+", mean="+mean+", est="+estimatePerSet+", total="+(long)total);
// System.err.println("estSum="+(long)estSum+", median="+median+", medianEst="+(long)medianEst);
long cardinality=(long)(total);
lastCardinality=cardinality;
return cardinality;
}
// @Override
// public final long cardinality(){
// long sum=0;
// //assert(atomic);
// if(atomic){
// for(int i=0; i<maxArray.length(); i++){
// sum+=maxArray.get(i);
// }
// }else{
// for(int i=0; i<maxArray2.length; i++){
// sum+=maxArray2[i];
// }
// }
// double mean=sum/(double)buckets;
// long cardinality=(long)((((Math.pow(2, mean)-1)*buckets*SKIPMOD))/1.258275);
// lastCardinality=cardinality;
// return cardinality;
// }
public final long cardinalityH(){
double sum=0;
for(int i=0; i<maxArrayA.length(); i++){
int x=Tools.max(1, maxArrayA.get(i));
sum+=1.0/x;
}
double mean=buckets/sum;
return (long)((Math.pow(2, mean)*buckets*SKIPMOD));
}
@Override
public final void add(CardinalityTracker log){
assert(log.getClass()==this.getClass());
add((LogLog)log);
}
public void add(LogLog log){
if(atomic && maxArrayA!=log.maxArrayA){
for(int i=0; i<buckets; i++){
maxArrayA.set(i, Tools.max(maxArrayA.get(i), log.maxArrayA.get(i)));
}
}else if(maxArray!=log.maxArray){
for(int i=0; i<buckets; i++){
maxArray[i]=Tools.max(maxArray[i], log.maxArray[i]);
}
}
}
@Override
public void hashAndStore(final long number){
// if(number%SKIPMOD!=0){return;} //Slows down moderately
long key=number;
// key=hash(key, tables[((int)number)&numTablesMask]);
key=Tools.hash64shift(key);
// if(key<0 || key>maxHashedValue){return;}//Slows things down by 50% lot, mysteriously
int leading=Long.numberOfLeadingZeros(key);
// counts[leading]++;
// if(leading<3){return;}//Slows things down slightly
// final int bucket=(int)((number&Integer.MAX_VALUE)%buckets);
final int bucket=(int)(key&bucketMask);
if(atomic){
int x=maxArrayA.get(bucket);
while(leading>x){
boolean b=maxArrayA.compareAndSet(bucket, x, leading);
if(b){x=leading;}
else{x=maxArrayA.get(bucket);}
}
}else{
maxArray[bucket]=Tools.max(leading, maxArray[bucket]);
}
}
@Override
public final float[] compensationFactorLogBucketsArray(){
return compensationFactorLogBucketsArray;
}
/*--------------------------------------------------------------*/
/*---------------- Fields ----------------*/
/*--------------------------------------------------------------*/
/** Maintains state. These are the actual buckets. */
private final int[] maxArray;
/** Atomic version of maxArray. */
private final AtomicIntegerArray maxArrayA;
private static final float[] compensationFactorLogBucketsArray={
0.053699781f, 0.49556874f, 0.742263622f, 0.861204899f, 0.926038294f,
0.967001269f, 0.982949748f, 0.992495155f, 0.996185775f, 0.998077246f
};
}
|