File: heap-checker.cc

package info (click to toggle)
google-perftools 2.7-1
  • links: PTS, VCS
  • area: main
  • in suites: bullseye, buster, sid
  • size: 7,284 kB
  • sloc: cpp: 27,869; ansic: 9,534; sh: 4,799; perl: 4,116; makefile: 1,101; asm: 128
file content (2388 lines) | stat: -rwxr-xr-x 100,182 bytes parent folder | download | duplicates (3)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*-
// Copyright (c) 2005, Google Inc.
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//     * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//     * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

// ---
// All Rights Reserved.
//
// Author: Maxim Lifantsev
//

#include "config.h"

#include <fcntl.h>    // for O_RDONLY (we use syscall to do actual reads)
#include <string.h>
#include <errno.h>
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#ifdef HAVE_MMAP
#include <sys/mman.h>
#endif
#ifdef HAVE_PTHREAD
#include <pthread.h>
#endif
#include <sys/stat.h>
#include <sys/types.h>
#include <time.h>
#include <assert.h>

#if defined(HAVE_LINUX_PTRACE_H)
#include <linux/ptrace.h>
#endif
#ifdef HAVE_SYS_SYSCALL_H
#include <sys/syscall.h>
#endif
#if defined(_WIN32) || defined(__CYGWIN__) || defined(__CYGWIN32__) || defined(__MINGW32__)
#include <wtypes.h>
#include <winbase.h>
#undef ERROR     // windows defines these as macros, which can cause trouble
#undef max
#undef min
#endif

#include <string>
#include <vector>
#include <map>
#include <set>
#include <algorithm>
#include <functional>

#include <gperftools/heap-checker.h>

#include "base/basictypes.h"
#include "base/googleinit.h"
#include "base/logging.h"
#include <gperftools/stacktrace.h>
#include "base/commandlineflags.h"
#include "base/elfcore.h"              // for i386_regs
#include "base/thread_lister.h"
#include "heap-profile-table.h"
#include "base/low_level_alloc.h"
#include "malloc_hook-inl.h"
#include <gperftools/malloc_hook.h>
#include <gperftools/malloc_extension.h>
#include "maybe_threads.h"
#include "memory_region_map.h"
#include "base/spinlock.h"
#include "base/sysinfo.h"
#include "base/stl_allocator.h"

using std::string;
using std::basic_string;
using std::pair;
using std::map;
using std::set;
using std::vector;
using std::swap;
using std::make_pair;
using std::min;
using std::max;
using std::less;
using std::char_traits;

// If current process is being ptrace()d, 'TracerPid' in /proc/self/status
// will be non-zero.
static bool IsDebuggerAttached(void) {    // only works under linux, probably
  char buf[256];   // TracerPid comes relatively earlier in status output
  int fd = open("/proc/self/status", O_RDONLY);
  if (fd == -1) {
    return false;  // Can't tell for sure.
  }
  const int len = read(fd, buf, sizeof(buf));
  bool rc = false;
  if (len > 0) {
    const char *const kTracerPid = "TracerPid:\t";
    buf[len - 1] = '\0';
    const char *p = strstr(buf, kTracerPid);
    if (p != NULL) {
      rc = (strncmp(p + strlen(kTracerPid), "0\n", 2) != 0);
    }
  }
  close(fd);
  return rc;
}

// This is the default if you don't link in -lprofiler
extern "C" {
ATTRIBUTE_WEAK PERFTOOLS_DLL_DECL int ProfilingIsEnabledForAllThreads();
int ProfilingIsEnabledForAllThreads() { return false; }
}

//----------------------------------------------------------------------
// Flags that control heap-checking
//----------------------------------------------------------------------

DEFINE_string(heap_check,
              EnvToString("HEAPCHECK", ""),
              "The heap leak checking to be done over the whole executable: "
              "\"minimal\", \"normal\", \"strict\", "
              "\"draconian\", \"as-is\", and \"local\" "
              " or the empty string are the supported choices. "
              "(See HeapLeakChecker_InternalInitStart for details.)");

DEFINE_bool(heap_check_report, true, "Obsolete");

DEFINE_bool(heap_check_before_constructors,
            true,
            "deprecated; pretty much always true now");

DEFINE_bool(heap_check_after_destructors,
            EnvToBool("HEAP_CHECK_AFTER_DESTRUCTORS", false),
            "If overall heap check is to end after global destructors "
            "or right after all REGISTER_HEAPCHECK_CLEANUP's");

DEFINE_bool(heap_check_strict_check, true, "Obsolete");

DEFINE_bool(heap_check_ignore_global_live,
            EnvToBool("HEAP_CHECK_IGNORE_GLOBAL_LIVE", true),
            "If overall heap check is to ignore heap objects reachable "
            "from the global data");

DEFINE_bool(heap_check_identify_leaks,
            EnvToBool("HEAP_CHECK_IDENTIFY_LEAKS", false),
            "If heap check should generate the addresses of the leaked "
            "objects in the memory leak profiles.  This may be useful "
            "in tracking down leaks where only a small fraction of "
            "objects allocated at the same stack trace are leaked.");

DEFINE_bool(heap_check_ignore_thread_live,
            EnvToBool("HEAP_CHECK_IGNORE_THREAD_LIVE", true),
            "If set to true, objects reachable from thread stacks "
            "and registers are not reported as leaks");

DEFINE_bool(heap_check_test_pointer_alignment,
            EnvToBool("HEAP_CHECK_TEST_POINTER_ALIGNMENT", false),
            "Set to true to check if the found leak can be due to "
            "use of unaligned pointers");

// Alignment at which all pointers in memory are supposed to be located;
// use 1 if any alignment is ok.
// heap_check_test_pointer_alignment flag guides if we try the value of 1.
// The larger it can be, the lesser is the chance of missing real leaks.
static const size_t kPointerSourceAlignment = sizeof(void*);
DEFINE_int32(heap_check_pointer_source_alignment,
	     EnvToInt("HEAP_CHECK_POINTER_SOURCE_ALIGNMENT",
                      kPointerSourceAlignment),
             "Alignment at which all pointers in memory are supposed to be "
             "located.  Use 1 if any alignment is ok.");

// A reasonable default to handle pointers inside of typical class objects:
// Too low and we won't be able to traverse pointers to normally-used
// nested objects and base parts of multiple-inherited objects.
// Too high and it will both slow down leak checking (FindInsideAlloc
// in HaveOnHeapLocked will get slower when there are large on-heap objects)
// and make it probabilistically more likely to miss leaks
// of large-sized objects.
static const int64 kHeapCheckMaxPointerOffset = 1024;
DEFINE_int64(heap_check_max_pointer_offset,
	     EnvToInt("HEAP_CHECK_MAX_POINTER_OFFSET",
                      kHeapCheckMaxPointerOffset),
             "Largest pointer offset for which we traverse "
             "pointers going inside of heap allocated objects. "
             "Set to -1 to use the actual largest heap object size.");

DEFINE_bool(heap_check_run_under_gdb,
            EnvToBool("HEAP_CHECK_RUN_UNDER_GDB", false),
            "If false, turns off heap-checking library when running under gdb "
            "(normally, set to 'true' only when debugging the heap-checker)");

DEFINE_int32(heap_check_delay_seconds, 0,
             "Number of seconds to delay on-exit heap checking."
             " If you set this flag,"
             " you may also want to set exit_timeout_seconds in order to"
             " avoid exit timeouts.\n"
             "NOTE: This flag is to be used only to help diagnose issues"
             " where it is suspected that the heap checker is reporting"
             " false leaks that will disappear if the heap checker delays"
             " its checks. Report any such issues to the heap-checker"
             " maintainer(s).");

//----------------------------------------------------------------------

DEFINE_string(heap_profile_pprof,
              EnvToString("PPROF_PATH", "pprof"),
              "OBSOLETE; not used");

DEFINE_string(heap_check_dump_directory,
              EnvToString("HEAP_CHECK_DUMP_DIRECTORY", "/tmp"),
              "Directory to put heap-checker leak dump information");


//----------------------------------------------------------------------
// HeapLeakChecker global data
//----------------------------------------------------------------------

// Global lock for all the global data of this module.
static SpinLock heap_checker_lock(SpinLock::LINKER_INITIALIZED);

//----------------------------------------------------------------------

// Heap profile prefix for leak checking profiles.
// Gets assigned once when leak checking is turned on, then never modified.
static const string* profile_name_prefix = NULL;

// Whole-program heap leak checker.
// Gets assigned once when leak checking is turned on,
// then main_heap_checker is never deleted.
static HeapLeakChecker* main_heap_checker = NULL;

// Whether we will use main_heap_checker to do a check at program exit
// automatically. In any case user can ask for more checks on main_heap_checker
// via GlobalChecker().
static bool do_main_heap_check = false;

// The heap profile we use to collect info about the heap.
// This is created in HeapLeakChecker::BeforeConstructorsLocked
// together with setting heap_checker_on (below) to true
// and registering our new/delete malloc hooks;
// similarly all are unset in HeapLeakChecker::TurnItselfOffLocked.
static HeapProfileTable* heap_profile = NULL;

// If we are doing (or going to do) any kind of heap-checking.
static bool heap_checker_on = false;

// pid of the process that does whole-program heap leak checking
static pid_t heap_checker_pid = 0;

// If we did heap profiling during global constructors execution
static bool constructor_heap_profiling = false;

// RAW_VLOG level we dump key INFO messages at.  If you want to turn
// off these messages, set the environment variable PERFTOOLS_VERBOSE=-1.
static const int heap_checker_info_level = 0;

//----------------------------------------------------------------------
// HeapLeakChecker's own memory allocator that is
// independent of the normal program allocator.
//----------------------------------------------------------------------

// Wrapper of LowLevelAlloc for STL_Allocator and direct use.
// We always access this class under held heap_checker_lock,
// this allows us to in particular protect the period when threads are stopped
// at random spots with TCMalloc_ListAllProcessThreads by heap_checker_lock,
// w/o worrying about the lock in LowLevelAlloc::Arena.
// We rely on the fact that we use an own arena with an own lock here.
class HeapLeakChecker::Allocator {
 public:
  static void Init() {
    RAW_DCHECK(heap_checker_lock.IsHeld(), "");
    RAW_DCHECK(arena_ == NULL, "");
    arena_ = LowLevelAlloc::NewArena(0, LowLevelAlloc::DefaultArena());
  }
  static void Shutdown() {
    RAW_DCHECK(heap_checker_lock.IsHeld(), "");
    if (!LowLevelAlloc::DeleteArena(arena_)  ||  alloc_count_ != 0) {
      RAW_LOG(FATAL, "Internal heap checker leak of %d objects", alloc_count_);
    }
  }
  static int alloc_count() {
    RAW_DCHECK(heap_checker_lock.IsHeld(), "");
    return alloc_count_;
  }
  static void* Allocate(size_t n) {
    RAW_DCHECK(arena_  &&  heap_checker_lock.IsHeld(), "");
    void* p = LowLevelAlloc::AllocWithArena(n, arena_);
    if (p) alloc_count_ += 1;
    return p;
  }
  static void Free(void* p) {
    RAW_DCHECK(heap_checker_lock.IsHeld(), "");
    if (p) alloc_count_ -= 1;
    LowLevelAlloc::Free(p);
  }
  static void Free(void* p, size_t /* n */) {
    Free(p);
  }
  // destruct, free, and make *p to be NULL
  template<typename T> static void DeleteAndNull(T** p) {
    (*p)->~T();
    Free(*p);
    *p = NULL;
  }
  template<typename T> static void DeleteAndNullIfNot(T** p) {
    if (*p != NULL) DeleteAndNull(p);
  }
 private:
  static LowLevelAlloc::Arena* arena_;
  static int alloc_count_;
};

LowLevelAlloc::Arena* HeapLeakChecker::Allocator::arena_ = NULL;
int HeapLeakChecker::Allocator::alloc_count_ = 0;

//----------------------------------------------------------------------
// HeapLeakChecker live object tracking components
//----------------------------------------------------------------------

// Cases of live object placement we distinguish
enum ObjectPlacement {
  MUST_BE_ON_HEAP,   // Must point to a live object of the matching size in the
                     // heap_profile map of the heap when we get to it
  IGNORED_ON_HEAP,   // Is a live (ignored) object on heap
  MAYBE_LIVE,        // Is a piece of writable memory from /proc/self/maps
  IN_GLOBAL_DATA,    // Is part of global data region of the executable
  THREAD_DATA,       // Part of a thread stack and a thread descriptor with TLS
  THREAD_REGISTERS,  // Values in registers of some thread
};

// Information about an allocated object
struct AllocObject {
  const void* ptr;        // the object
  uintptr_t size;         // its size
  ObjectPlacement place;  // where ptr points to

  AllocObject(const void* p, size_t s, ObjectPlacement l)
    : ptr(p), size(s), place(l) { }
};

// All objects (memory ranges) ignored via HeapLeakChecker::IgnoreObject
// Key is the object's address; value is its size.
typedef map<uintptr_t, size_t, less<uintptr_t>,
            STL_Allocator<pair<const uintptr_t, size_t>,
                          HeapLeakChecker::Allocator>
           > IgnoredObjectsMap;
static IgnoredObjectsMap* ignored_objects = NULL;

// All objects (memory ranges) that we consider to be the sources of pointers
// to live (not leaked) objects.
// At different times this holds (what can be reached from) global data regions
// and the objects we've been told to ignore.
// For any AllocObject::ptr "live_objects" is supposed to contain at most one
// record at any time. We maintain this by checking with the heap_profile map
// of the heap and removing the live heap objects we've handled from it.
// This vector is maintained as a stack and the frontier of reachable
// live heap objects in our flood traversal of them.
typedef vector<AllocObject,
               STL_Allocator<AllocObject, HeapLeakChecker::Allocator>
              > LiveObjectsStack;
static LiveObjectsStack* live_objects = NULL;

// A special string type that uses my allocator
typedef basic_string<char, char_traits<char>,
                     STL_Allocator<char, HeapLeakChecker::Allocator>
                    > HCL_string;

// A placeholder to fill-in the starting values for live_objects
// for each library so we can keep the library-name association for logging.
typedef map<HCL_string, LiveObjectsStack, less<HCL_string>,
            STL_Allocator<pair<const HCL_string, LiveObjectsStack>,
                          HeapLeakChecker::Allocator>
           > LibraryLiveObjectsStacks;
static LibraryLiveObjectsStacks* library_live_objects = NULL;

// Value stored in the map of disabled address ranges;
// its key is the end of the address range.
// We'll ignore allocations with a return address in a disabled range
// if the address occurs at 'max_depth' or less in the stack trace.
struct HeapLeakChecker::RangeValue {
  uintptr_t start_address;  // the start of the range
  int       max_depth;      // the maximal stack depth to disable at
};
typedef map<uintptr_t, HeapLeakChecker::RangeValue, less<uintptr_t>,
            STL_Allocator<pair<const uintptr_t, HeapLeakChecker::RangeValue>,
                          HeapLeakChecker::Allocator>
           > DisabledRangeMap;
// The disabled program counter address ranges for profile dumping
// that are registered with HeapLeakChecker::DisableChecksFromToLocked.
static DisabledRangeMap* disabled_ranges = NULL;

// Set of stack tops.
// These are used to consider live only appropriate chunks of the memory areas
// that are used for stacks (and maybe thread-specific data as well)
// so that we do not treat pointers from outdated stack frames as live.
typedef set<uintptr_t, less<uintptr_t>,
            STL_Allocator<uintptr_t, HeapLeakChecker::Allocator>
           > StackTopSet;
static StackTopSet* stack_tops = NULL;

// A map of ranges of code addresses for the system libraries
// that can mmap/mremap/sbrk-allocate memory regions for stacks
// and thread-local storage that we want to consider as live global data.
// Maps from the end address to the start address.
typedef map<uintptr_t, uintptr_t, less<uintptr_t>,
            STL_Allocator<pair<const uintptr_t, uintptr_t>,
                          HeapLeakChecker::Allocator>
           > GlobalRegionCallerRangeMap;
static GlobalRegionCallerRangeMap* global_region_caller_ranges = NULL;

// TODO(maxim): make our big data structs into own modules

// Disabler is implemented by keeping track of a per-thread count
// of active Disabler objects.  Any objects allocated while the
// count > 0 are not reported.

#ifdef HAVE_TLS

static __thread int thread_disable_counter
// The "inital exec" model is faster than the default TLS model, at
// the cost you can't dlopen this library.  But dlopen on heap-checker
// doesn't work anyway -- it must run before main -- so this is a good
// trade-off.
# ifdef HAVE___ATTRIBUTE__
   __attribute__ ((tls_model ("initial-exec")))
# endif
    ;
inline int get_thread_disable_counter() {
  return thread_disable_counter;
}
inline void set_thread_disable_counter(int value) {
  thread_disable_counter = value;
}

#else  // #ifdef HAVE_TLS

static pthread_key_t thread_disable_counter_key;
static int main_thread_counter;   // storage for use before main()
static bool use_main_thread_counter = true;

// TODO(csilvers): this is called from NewHook, in the middle of malloc().
// If perftools_pthread_getspecific calls malloc, that will lead to an
// infinite loop.  I don't know how to fix that, so I hope it never happens!
inline int get_thread_disable_counter() {
  if (use_main_thread_counter)  // means we're running really early
    return main_thread_counter;
  void* p = perftools_pthread_getspecific(thread_disable_counter_key);
  return (intptr_t)p;   // kinda evil: store the counter directly in the void*
}

inline void set_thread_disable_counter(int value) {
  if (use_main_thread_counter) {   // means we're running really early
    main_thread_counter = value;
    return;
  }
  intptr_t pointer_sized_value = value;
  // kinda evil: store the counter directly in the void*
  void* p = (void*)pointer_sized_value;
  // NOTE: this may call malloc, which will call NewHook which will call
  // get_thread_disable_counter() which will call pthread_getspecific().  I
  // don't know if anything bad can happen if we call getspecific() in the
  // middle of a setspecific() call.  It seems to work ok in practice...
  perftools_pthread_setspecific(thread_disable_counter_key, p);
}

// The idea here is that this initializer will run pretty late: after
// pthreads have been totally set up.  At this point we can call
// pthreads routines, so we set those up.
class InitThreadDisableCounter {
 public:
  InitThreadDisableCounter() {
    perftools_pthread_key_create(&thread_disable_counter_key, NULL);
    // Set up the main thread's value, which we have a special variable for.
    void* p = (void*)(intptr_t)main_thread_counter;   // store the counter directly
    perftools_pthread_setspecific(thread_disable_counter_key, p);
    use_main_thread_counter = false;
  }
};
InitThreadDisableCounter init_thread_disable_counter;

#endif  // #ifdef HAVE_TLS

HeapLeakChecker::Disabler::Disabler() {
  // It is faster to unconditionally increment the thread-local
  // counter than to check whether or not heap-checking is on
  // in a thread-safe manner.
  int counter = get_thread_disable_counter();
  set_thread_disable_counter(counter + 1);
  RAW_VLOG(10, "Increasing thread disable counter to %d", counter + 1);
}

HeapLeakChecker::Disabler::~Disabler() {
  int counter = get_thread_disable_counter();
  RAW_DCHECK(counter > 0, "");
  if (counter > 0) {
    set_thread_disable_counter(counter - 1);
    RAW_VLOG(10, "Decreasing thread disable counter to %d", counter);
  } else {
    RAW_VLOG(0, "Thread disable counter underflow : %d", counter);
  }
}

//----------------------------------------------------------------------

// The size of the largest heap object allocated so far.
static size_t max_heap_object_size = 0;
// The possible range of addresses that can point
// into one of the elements of heap_objects.
static uintptr_t min_heap_address = uintptr_t(-1LL);
static uintptr_t max_heap_address = 0;

//----------------------------------------------------------------------

// Simple casting helpers for uintptr_t and void*:
template<typename T>
inline static const void* AsPtr(T addr) {
  return reinterpret_cast<void*>(addr);
}
inline static uintptr_t AsInt(const void* ptr) {
  return reinterpret_cast<uintptr_t>(ptr);
}

//----------------------------------------------------------------------

// We've seen reports that strstr causes heap-checker crashes in some
// libc's (?):
//    http://code.google.com/p/gperftools/issues/detail?id=263
// It's simple enough to use our own.  This is not in time-critical code.
static const char* hc_strstr(const char* s1, const char* s2) {
  const size_t len = strlen(s2);
  RAW_CHECK(len > 0, "Unexpected empty string passed to strstr()");
  for (const char* p = strchr(s1, *s2); p != NULL; p = strchr(p+1, *s2)) {
    if (strncmp(p, s2, len) == 0) {
      return p;
    }
  }
  return NULL;
}

//----------------------------------------------------------------------

// Our hooks for MallocHook
static void NewHook(const void* ptr, size_t size) {
  if (ptr != NULL) {
    const int counter = get_thread_disable_counter();
    const bool ignore = (counter > 0);
    RAW_VLOG(16, "Recording Alloc: %p of %" PRIuS "; %d", ptr, size,
             int(counter));

    // Fetch the caller's stack trace before acquiring heap_checker_lock.
    void* stack[HeapProfileTable::kMaxStackDepth];
    int depth = HeapProfileTable::GetCallerStackTrace(0, stack);

    { SpinLockHolder l(&heap_checker_lock);
      if (size > max_heap_object_size) max_heap_object_size = size;
      uintptr_t addr = AsInt(ptr);
      if (addr < min_heap_address) min_heap_address = addr;
      addr += size;
      if (addr > max_heap_address) max_heap_address = addr;
      if (heap_checker_on) {
        heap_profile->RecordAlloc(ptr, size, depth, stack);
        if (ignore) {
          heap_profile->MarkAsIgnored(ptr);
        }
      }
    }
    RAW_VLOG(17, "Alloc Recorded: %p of %" PRIuS "", ptr, size);
  }
}

static void DeleteHook(const void* ptr) {
  if (ptr != NULL) {
    RAW_VLOG(16, "Recording Free %p", ptr);
    { SpinLockHolder l(&heap_checker_lock);
      if (heap_checker_on) heap_profile->RecordFree(ptr);
    }
    RAW_VLOG(17, "Free Recorded: %p", ptr);
  }
}

//----------------------------------------------------------------------

enum StackDirection {
  GROWS_TOWARDS_HIGH_ADDRESSES,
  GROWS_TOWARDS_LOW_ADDRESSES,
  UNKNOWN_DIRECTION
};

// Determine which way the stack grows:

static StackDirection ATTRIBUTE_NOINLINE GetStackDirection(
    const uintptr_t *const ptr) {
  uintptr_t x;
  if (&x < ptr)
    return GROWS_TOWARDS_LOW_ADDRESSES;
  if (ptr < &x)
    return GROWS_TOWARDS_HIGH_ADDRESSES;

  RAW_CHECK(0, "");  // Couldn't determine the stack direction.

  return UNKNOWN_DIRECTION;
}

// Direction of stack growth (will initialize via GetStackDirection())
static StackDirection stack_direction = UNKNOWN_DIRECTION;

// This routine is called for every thread stack we know about to register it.
static void RegisterStackLocked(const void* top_ptr) {
  RAW_DCHECK(heap_checker_lock.IsHeld(), "");
  RAW_DCHECK(MemoryRegionMap::LockIsHeld(), "");
  RAW_VLOG(10, "Thread stack at %p", top_ptr);
  uintptr_t top = AsInt(top_ptr);
  stack_tops->insert(top);  // add for later use

  // make sure stack_direction is initialized
  if (stack_direction == UNKNOWN_DIRECTION) {
    stack_direction = GetStackDirection(&top);
  }

  // Find memory region with this stack
  MemoryRegionMap::Region region;
  if (MemoryRegionMap::FindAndMarkStackRegion(top, &region)) {
    // Make the proper portion of the stack live:
    if (stack_direction == GROWS_TOWARDS_LOW_ADDRESSES) {
      RAW_VLOG(11, "Live stack at %p of %" PRIuPTR " bytes",
                  top_ptr, region.end_addr - top);
      live_objects->push_back(AllocObject(top_ptr, region.end_addr - top,
                                          THREAD_DATA));
    } else {  // GROWS_TOWARDS_HIGH_ADDRESSES
      RAW_VLOG(11, "Live stack at %p of %" PRIuPTR " bytes",
                  AsPtr(region.start_addr),
                  top - region.start_addr);
      live_objects->push_back(AllocObject(AsPtr(region.start_addr),
                                          top - region.start_addr,
                                          THREAD_DATA));
    }
  // not in MemoryRegionMap, look in library_live_objects:
  } else if (FLAGS_heap_check_ignore_global_live) {
    for (LibraryLiveObjectsStacks::iterator lib = library_live_objects->begin();
         lib != library_live_objects->end(); ++lib) {
      for (LiveObjectsStack::iterator span = lib->second.begin();
           span != lib->second.end(); ++span) {
        uintptr_t start = AsInt(span->ptr);
        uintptr_t end = start + span->size;
        if (start <= top  &&  top < end) {
          RAW_VLOG(11, "Stack at %p is inside /proc/self/maps chunk %p..%p",
                      top_ptr, AsPtr(start), AsPtr(end));
          // Shrink start..end region by chopping away the memory regions in
          // MemoryRegionMap that land in it to undo merging of regions
          // in /proc/self/maps, so that we correctly identify what portion
          // of start..end is actually the stack region.
          uintptr_t stack_start = start;
          uintptr_t stack_end = end;
          // can optimize-away this loop, but it does not run often
          RAW_DCHECK(MemoryRegionMap::LockIsHeld(), "");
          for (MemoryRegionMap::RegionIterator r =
                 MemoryRegionMap::BeginRegionLocked();
               r != MemoryRegionMap::EndRegionLocked(); ++r) {
            if (top < r->start_addr  &&  r->start_addr < stack_end) {
              stack_end = r->start_addr;
            }
            if (stack_start < r->end_addr  &&  r->end_addr <= top) {
              stack_start = r->end_addr;
            }
          }
          if (stack_start != start  ||  stack_end != end) {
            RAW_VLOG(11, "Stack at %p is actually inside memory chunk %p..%p",
                        top_ptr, AsPtr(stack_start), AsPtr(stack_end));
          }
          // Make the proper portion of the stack live:
          if (stack_direction == GROWS_TOWARDS_LOW_ADDRESSES) {
            RAW_VLOG(11, "Live stack at %p of %" PRIuPTR " bytes",
                        top_ptr, stack_end - top);
            live_objects->push_back(
              AllocObject(top_ptr, stack_end - top, THREAD_DATA));
          } else {  // GROWS_TOWARDS_HIGH_ADDRESSES
            RAW_VLOG(11, "Live stack at %p of %" PRIuPTR " bytes",
                        AsPtr(stack_start), top - stack_start);
            live_objects->push_back(
              AllocObject(AsPtr(stack_start), top - stack_start, THREAD_DATA));
          }
          lib->second.erase(span);  // kill the rest of the region
          // Put the non-stack part(s) of the region back:
          if (stack_start != start) {
            lib->second.push_back(AllocObject(AsPtr(start), stack_start - start,
                                  MAYBE_LIVE));
          }
          if (stack_end != end) {
            lib->second.push_back(AllocObject(AsPtr(stack_end), end - stack_end,
                                  MAYBE_LIVE));
          }
          return;
        }
      }
    }
    RAW_LOG(ERROR, "Memory region for stack at %p not found. "
                   "Will likely report false leak positives.", top_ptr);
  }
}

// Iterator for heap allocation map data to make ignored objects "live"
// (i.e., treated as roots for the mark-and-sweep phase)
static void MakeIgnoredObjectsLiveCallbackLocked(
    const void* ptr, const HeapProfileTable::AllocInfo& info) {
  RAW_DCHECK(heap_checker_lock.IsHeld(), "");
  if (info.ignored) {
    live_objects->push_back(AllocObject(ptr, info.object_size,
                                        MUST_BE_ON_HEAP));
  }
}

// Iterator for heap allocation map data to make objects allocated from
// disabled regions of code to be live.
static void MakeDisabledLiveCallbackLocked(
    const void* ptr, const HeapProfileTable::AllocInfo& info) {
  RAW_DCHECK(heap_checker_lock.IsHeld(), "");
  bool stack_disable = false;
  bool range_disable = false;
  for (int depth = 0; depth < info.stack_depth; depth++) {
    uintptr_t addr = AsInt(info.call_stack[depth]);
    if (disabled_ranges) {
      DisabledRangeMap::const_iterator iter
        = disabled_ranges->upper_bound(addr);
      if (iter != disabled_ranges->end()) {
        RAW_DCHECK(iter->first > addr, "");
        if (iter->second.start_address < addr  &&
            iter->second.max_depth > depth) {
          range_disable = true;  // in range; dropping
          break;
        }
      }
    }
  }
  if (stack_disable || range_disable) {
    uintptr_t start_address = AsInt(ptr);
    uintptr_t end_address = start_address + info.object_size;
    StackTopSet::const_iterator iter
      = stack_tops->lower_bound(start_address);
    if (iter != stack_tops->end()) {
      RAW_DCHECK(*iter >= start_address, "");
      if (*iter < end_address) {
        // We do not disable (treat as live) whole allocated regions
        // if they are used to hold thread call stacks
        // (i.e. when we find a stack inside).
        // The reason is that we'll treat as live the currently used
        // stack portions anyway (see RegisterStackLocked),
        // and the rest of the region where the stack lives can well
        // contain outdated stack variables which are not live anymore,
        // hence should not be treated as such.
        RAW_VLOG(11, "Not %s-disabling %" PRIuS " bytes at %p"
                    ": have stack inside: %p",
                    (stack_disable ? "stack" : "range"),
                    info.object_size, ptr, AsPtr(*iter));
        return;
      }
    }
    RAW_VLOG(11, "%s-disabling %" PRIuS " bytes at %p",
                (stack_disable ? "Stack" : "Range"), info.object_size, ptr);
    live_objects->push_back(AllocObject(ptr, info.object_size,
                                        MUST_BE_ON_HEAP));
  }
}

static const char kUnnamedProcSelfMapEntry[] = "UNNAMED";

// This function takes some fields from a /proc/self/maps line:
//
//   start_address  start address of a memory region.
//   end_address    end address of a memory region
//   permissions    rwx + private/shared bit
//   filename       filename of the mapped file
//
// If the region is not writeable, then it cannot have any heap
// pointers in it, otherwise we record it as a candidate live region
// to get filtered later.
static void RecordGlobalDataLocked(uintptr_t start_address,
                                   uintptr_t end_address,
                                   const char* permissions,
                                   const char* filename) {
  RAW_DCHECK(heap_checker_lock.IsHeld(), "");
  // Ignore non-writeable regions.
  if (strchr(permissions, 'w') == NULL) return;
  if (filename == NULL  ||  *filename == '\0') {
    filename = kUnnamedProcSelfMapEntry;
  }
  RAW_VLOG(11, "Looking into %s: 0x%" PRIxPTR "..0x%" PRIxPTR,
              filename, start_address, end_address);
  (*library_live_objects)[filename].
    push_back(AllocObject(AsPtr(start_address),
                          end_address - start_address,
                          MAYBE_LIVE));
}

// See if 'library' from /proc/self/maps has base name 'library_base'
// i.e. contains it and has '.' or '-' after it.
static bool IsLibraryNamed(const char* library, const char* library_base) {
  const char* p = hc_strstr(library, library_base);
  size_t sz = strlen(library_base);
  return p != NULL  &&  (p[sz] == '.'  ||  p[sz] == '-');
}

// static
void HeapLeakChecker::DisableLibraryAllocsLocked(const char* library,
                                                 uintptr_t start_address,
                                                 uintptr_t end_address) {
  RAW_DCHECK(heap_checker_lock.IsHeld(), "");
  int depth = 0;
  // TODO(maxim): maybe this should be extended to also use objdump
  //              and pick the text portion of the library more precisely.
  if (IsLibraryNamed(library, "/libpthread")  ||
        // libpthread has a lot of small "system" leaks we don't care about.
        // In particular it allocates memory to store data supplied via
        // pthread_setspecific (which can be the only pointer to a heap object).
      IsLibraryNamed(library, "/libdl")  ||
        // library loaders leak some "system" heap that we don't care about
      IsLibraryNamed(library, "/libcrypto")  ||
        // Sometimes libcrypto of OpenSSH is compiled with -fomit-frame-pointer
        // (any library can be, of course, but this one often is because speed
        // is so important for making crypto usable).  We ignore all its
        // allocations because we can't see the call stacks.  We'd prefer
        // to ignore allocations done in files/symbols that match
        // "default_malloc_ex|default_realloc_ex"
        // but that doesn't work when the end-result binary is stripped.
      IsLibraryNamed(library, "/libjvm")  ||
        // JVM has a lot of leaks we don't care about.
      IsLibraryNamed(library, "/libzip")
        // The JVM leaks java.util.zip.Inflater after loading classes.
     ) {
    depth = 1;  // only disable allocation calls directly from the library code
  } else if (IsLibraryNamed(library, "/ld")
               // library loader leaks some "system" heap
               // (e.g. thread-local storage) that we don't care about
            ) {
    depth = 2;  // disable allocation calls directly from the library code
                // and at depth 2 from it.
    // We need depth 2 here solely because of a libc bug that
    // forces us to jump through __memalign_hook and MemalignOverride hoops
    // in tcmalloc.cc.
    // Those buggy __libc_memalign() calls are in ld-linux.so and happen for
    // thread-local storage allocations that we want to ignore here.
    // We go with the depth-2 hack as a workaround for this libc bug:
    // otherwise we'd need to extend MallocHook interface
    // so that correct stack depth adjustment can be propagated from
    // the exceptional case of MemalignOverride.
    // Using depth 2 here should not mask real leaks because ld-linux.so
    // does not call user code.
  }
  if (depth) {
    RAW_VLOG(10, "Disabling allocations from %s at depth %d:", library, depth);
    DisableChecksFromToLocked(AsPtr(start_address), AsPtr(end_address), depth);
    if (IsLibraryNamed(library, "/libpthread")  ||
        IsLibraryNamed(library, "/libdl")  ||
        IsLibraryNamed(library, "/ld")) {
      RAW_VLOG(10, "Global memory regions made by %s will be live data",
                  library);
      if (global_region_caller_ranges == NULL) {
        global_region_caller_ranges =
          new(Allocator::Allocate(sizeof(GlobalRegionCallerRangeMap)))
            GlobalRegionCallerRangeMap;
      }
      global_region_caller_ranges
        ->insert(make_pair(end_address, start_address));
    }
  }
}

// static
HeapLeakChecker::ProcMapsResult HeapLeakChecker::UseProcMapsLocked(
                                  ProcMapsTask proc_maps_task) {
  RAW_DCHECK(heap_checker_lock.IsHeld(), "");
  // Need to provide own scratch memory to ProcMapsIterator:
  ProcMapsIterator::Buffer buffer;
  ProcMapsIterator it(0, &buffer);
  if (!it.Valid()) {
    int errsv = errno;
    RAW_LOG(ERROR, "Could not open /proc/self/maps: errno=%d. "
                   "Libraries will not be handled correctly.", errsv);
    return CANT_OPEN_PROC_MAPS;
  }
  uint64 start_address, end_address, file_offset;
  int64 inode;
  char *permissions, *filename;
  bool saw_shared_lib = false;
  bool saw_nonzero_inode = false;
  bool saw_shared_lib_with_nonzero_inode = false;
  while (it.Next(&start_address, &end_address, &permissions,
                 &file_offset, &inode, &filename)) {
    if (start_address >= end_address) {
      // Warn if a line we can be interested in is ill-formed:
      if (inode != 0) {
        RAW_LOG(ERROR, "Errors reading /proc/self/maps. "
                       "Some global memory regions will not "
                       "be handled correctly.");
      }
      // Silently skip other ill-formed lines: some are possible
      // probably due to the interplay of how /proc/self/maps is updated
      // while we read it in chunks in ProcMapsIterator and
      // do things in this loop.
      continue;
    }
    // Determine if any shared libraries are present (this is the same
    // list of extensions as is found in pprof).  We want to ignore
    // 'fake' libraries with inode 0 when determining.  However, some
    // systems don't share inodes via /proc, so we turn off this check
    // if we don't see any evidence that we're getting inode info.
    if (inode != 0) {
      saw_nonzero_inode = true;
    }
    if ((hc_strstr(filename, "lib") && hc_strstr(filename, ".so")) ||
        hc_strstr(filename, ".dll") ||
        // not all .dylib filenames start with lib. .dylib is big enough
        // that we are unlikely to get false matches just checking that.
        hc_strstr(filename, ".dylib") || hc_strstr(filename, ".bundle")) {
      saw_shared_lib = true;
      if (inode != 0) {
        saw_shared_lib_with_nonzero_inode = true;
      }
    }

    switch (proc_maps_task) {
      case DISABLE_LIBRARY_ALLOCS:
        // All lines starting like
        // "401dc000-4030f000 r??p 00132000 03:01 13991972  lib/bin"
        // identify a data and code sections of a shared library or our binary
        if (inode != 0 && strncmp(permissions, "r-xp", 4) == 0) {
          DisableLibraryAllocsLocked(filename, start_address, end_address);
        }
        break;
      case RECORD_GLOBAL_DATA:
        RecordGlobalDataLocked(start_address, end_address,
                               permissions, filename);
        break;
      default:
        RAW_CHECK(0, "");
    }
  }
  // If /proc/self/maps is reporting inodes properly (we saw a
  // non-zero inode), then we only say we saw a shared lib if we saw a
  // 'real' one, with a non-zero inode.
  if (saw_nonzero_inode) {
    saw_shared_lib = saw_shared_lib_with_nonzero_inode;
  }
  if (!saw_shared_lib) {
    RAW_LOG(ERROR, "No shared libs detected. Will likely report false leak "
                   "positives for statically linked executables.");
    return NO_SHARED_LIBS_IN_PROC_MAPS;
  }
  return PROC_MAPS_USED;
}

// Total number and size of live objects dropped from the profile;
// (re)initialized in IgnoreAllLiveObjectsLocked.
static int64 live_objects_total;
static int64 live_bytes_total;

// pid of the thread that is doing the current leak check
// (protected by our lock; IgnoreAllLiveObjectsLocked sets it)
static pid_t self_thread_pid = 0;

// Status of our thread listing callback execution
// (protected by our lock; used from within IgnoreAllLiveObjectsLocked)
static enum {
  CALLBACK_NOT_STARTED,
  CALLBACK_STARTED,
  CALLBACK_COMPLETED,
} thread_listing_status = CALLBACK_NOT_STARTED;

// Ideally to avoid deadlocks this function should not result in any libc
// or other function calls that might need to lock a mutex:
// It is called when all threads of a process are stopped
// at arbitrary points thus potentially holding those locks.
//
// In practice we are calling some simple i/o and sprintf-type library functions
// for logging messages, but use only our own LowLevelAlloc::Arena allocator.
//
// This is known to be buggy: the library i/o function calls are able to cause
// deadlocks when they request a lock that a stopped thread happens to hold.
// This issue as far as we know have so far not resulted in any deadlocks
// in practice, so for now we are taking our chance that the deadlocks
// have insignificant frequency.
//
// If such deadlocks become a problem we should make the i/o calls
// into appropriately direct system calls (or eliminate them),
// in particular write() is not safe and vsnprintf() is potentially dangerous
// due to reliance on locale functions (these are called through RAW_LOG
// and in other ways).
//

#if defined(HAVE_LINUX_PTRACE_H) && defined(HAVE_SYS_SYSCALL_H) && defined(DUMPER)
# if (defined(__i386__) || defined(__x86_64))
#  define THREAD_REGS i386_regs
# elif defined(__PPC__)
#  define THREAD_REGS ppc_regs
# endif
#endif

/*static*/ int HeapLeakChecker::IgnoreLiveThreadsLocked(void* parameter,
                                                        int num_threads,
                                                        pid_t* thread_pids,
                                                        va_list /*ap*/) {
  RAW_DCHECK(heap_checker_lock.IsHeld(), "");
  thread_listing_status = CALLBACK_STARTED;
  RAW_VLOG(11, "Found %d threads (from pid %d)", num_threads, getpid());

  if (FLAGS_heap_check_ignore_global_live) {
    UseProcMapsLocked(RECORD_GLOBAL_DATA);
  }

  // We put the registers from other threads here
  // to make pointers stored in them live.
  vector<void*, STL_Allocator<void*, Allocator> > thread_registers;

  int failures = 0;
  for (int i = 0; i < num_threads; ++i) {
    // the leak checking thread itself is handled
    // specially via self_thread_stack, not here:
    if (thread_pids[i] == self_thread_pid) continue;
    RAW_VLOG(11, "Handling thread with pid %d", thread_pids[i]);
#ifdef THREAD_REGS
    THREAD_REGS thread_regs;
#define sys_ptrace(r, p, a, d)  syscall(SYS_ptrace, (r), (p), (a), (d))
    // We use sys_ptrace to avoid thread locking
    // because this is called from TCMalloc_ListAllProcessThreads
    // when all but this thread are suspended.
    if (sys_ptrace(PTRACE_GETREGS, thread_pids[i], NULL, &thread_regs) == 0) {
      // Need to use SP to get all the data from the very last stack frame:
      COMPILE_ASSERT(sizeof(thread_regs.SP) == sizeof(void*),
                     SP_register_does_not_look_like_a_pointer);
      RegisterStackLocked(reinterpret_cast<void*>(thread_regs.SP));
      // Make registers live (just in case PTRACE_ATTACH resulted in some
      // register pointers still being in the registers and not on the stack):
      for (void** p = reinterpret_cast<void**>(&thread_regs);
           p < reinterpret_cast<void**>(&thread_regs + 1); ++p) {
        RAW_VLOG(12, "Thread register %p", *p);
        thread_registers.push_back(*p);
      }
    } else {
      failures += 1;
    }
#else
    failures += 1;
#endif
  }
  // Use all the collected thread (stack) liveness sources:
  IgnoreLiveObjectsLocked("threads stack data", "");
  if (thread_registers.size()) {
    // Make thread registers be live heap data sources.
    // we rely here on the fact that vector is in one memory chunk:
    RAW_VLOG(11, "Live registers at %p of %" PRIuS " bytes",
                &thread_registers[0], thread_registers.size() * sizeof(void*));
    live_objects->push_back(AllocObject(&thread_registers[0],
                                        thread_registers.size() * sizeof(void*),
                                        THREAD_REGISTERS));
    IgnoreLiveObjectsLocked("threads register data", "");
  }
  // Do all other liveness walking while all threads are stopped:
  IgnoreNonThreadLiveObjectsLocked();
  // Can now resume the threads:
  TCMalloc_ResumeAllProcessThreads(num_threads, thread_pids);
  thread_listing_status = CALLBACK_COMPLETED;
  return failures;
}

// Stack top of the thread that is doing the current leak check
// (protected by our lock; IgnoreAllLiveObjectsLocked sets it)
static const void* self_thread_stack_top;

// static
void HeapLeakChecker::IgnoreNonThreadLiveObjectsLocked() {
  RAW_DCHECK(heap_checker_lock.IsHeld(), "");
  RAW_DCHECK(MemoryRegionMap::LockIsHeld(), "");
  RAW_VLOG(11, "Handling self thread with pid %d", self_thread_pid);
  // Register our own stack:

  // Important that all stack ranges (including the one here)
  // are known before we start looking at them
  // in MakeDisabledLiveCallbackLocked:
  RegisterStackLocked(self_thread_stack_top);
  IgnoreLiveObjectsLocked("stack data", "");

  // Make objects we were told to ignore live:
  if (ignored_objects) {
    for (IgnoredObjectsMap::const_iterator object = ignored_objects->begin();
         object != ignored_objects->end(); ++object) {
      const void* ptr = AsPtr(object->first);
      RAW_VLOG(11, "Ignored live object at %p of %" PRIuS " bytes",
                  ptr, object->second);
      live_objects->
        push_back(AllocObject(ptr, object->second, MUST_BE_ON_HEAP));
      // we do this liveness check for ignored_objects before doing any
      // live heap walking to make sure it does not fail needlessly:
      size_t object_size;
      if (!(heap_profile->FindAlloc(ptr, &object_size)  &&
            object->second == object_size)) {
        RAW_LOG(FATAL, "Object at %p of %" PRIuS " bytes from an"
                       " IgnoreObject() has disappeared", ptr, object->second);
      }
    }
    IgnoreLiveObjectsLocked("ignored objects", "");
  }

  // Treat objects that were allocated when a Disabler was live as
  // roots.  I.e., if X was allocated while a Disabler was active,
  // and Y is reachable from X, arrange that neither X nor Y are
  // treated as leaks.
  heap_profile->IterateAllocs(MakeIgnoredObjectsLiveCallbackLocked);
  IgnoreLiveObjectsLocked("disabled objects", "");

  // Make code-address-disabled objects live and ignored:
  // This in particular makes all thread-specific data live
  // because the basic data structure to hold pointers to thread-specific data
  // is allocated from libpthreads and we have range-disabled that
  // library code with UseProcMapsLocked(DISABLE_LIBRARY_ALLOCS);
  // so now we declare all thread-specific data reachable from there as live.
  heap_profile->IterateAllocs(MakeDisabledLiveCallbackLocked);
  IgnoreLiveObjectsLocked("disabled code", "");

  // Actually make global data live:
  if (FLAGS_heap_check_ignore_global_live) {
    bool have_null_region_callers = false;
    for (LibraryLiveObjectsStacks::iterator l = library_live_objects->begin();
         l != library_live_objects->end(); ++l) {
      RAW_CHECK(live_objects->empty(), "");
      // Process library_live_objects in l->second
      // filtering them by MemoryRegionMap:
      // It's safe to iterate over MemoryRegionMap
      // w/o locks here as we are inside MemoryRegionMap::Lock():
      RAW_DCHECK(MemoryRegionMap::LockIsHeld(), "");
      // The only change to MemoryRegionMap possible in this loop
      // is region addition as a result of allocating more memory
      // for live_objects. This won't invalidate the RegionIterator
      // or the intent of the loop.
      // --see the comment by MemoryRegionMap::BeginRegionLocked().
      for (MemoryRegionMap::RegionIterator region =
             MemoryRegionMap::BeginRegionLocked();
           region != MemoryRegionMap::EndRegionLocked(); ++region) {
        // "region" from MemoryRegionMap is to be subtracted from
        // (tentatively live) regions in l->second
        // if it has a stack inside or it was allocated by
        // a non-special caller (not one covered by a range
        // in global_region_caller_ranges).
        // This will in particular exclude all memory chunks used
        // by the heap itself as well as what's been allocated with
        // any allocator on top of mmap.
        bool subtract = true;
        if (!region->is_stack  &&  global_region_caller_ranges) {
          if (region->caller() == static_cast<uintptr_t>(NULL)) {
            have_null_region_callers = true;
          } else {
            GlobalRegionCallerRangeMap::const_iterator iter
              = global_region_caller_ranges->upper_bound(region->caller());
            if (iter != global_region_caller_ranges->end()) {
              RAW_DCHECK(iter->first > region->caller(), "");
              if (iter->second < region->caller()) {  // in special region
                subtract = false;
              }
            }
          }
        }
        if (subtract) {
          // The loop puts the result of filtering l->second into live_objects:
          for (LiveObjectsStack::const_iterator i = l->second.begin();
               i != l->second.end(); ++i) {
            // subtract *region from *i
            uintptr_t start = AsInt(i->ptr);
            uintptr_t end = start + i->size;
            if (region->start_addr <= start  &&  end <= region->end_addr) {
              // full deletion due to subsumption
            } else if (start < region->start_addr  &&
                       region->end_addr < end) {  // cutting-out split
              live_objects->push_back(AllocObject(i->ptr,
                                                  region->start_addr - start,
                                                  IN_GLOBAL_DATA));
              live_objects->push_back(AllocObject(AsPtr(region->end_addr),
                                                  end - region->end_addr,
                                                  IN_GLOBAL_DATA));
            } else if (region->end_addr > start  &&
                       region->start_addr <= start) {  // cut from start
              live_objects->push_back(AllocObject(AsPtr(region->end_addr),
                                                  end - region->end_addr,
                                                  IN_GLOBAL_DATA));
            } else if (region->start_addr > start  &&
                       region->start_addr < end) {  // cut from end
              live_objects->push_back(AllocObject(i->ptr,
                                                  region->start_addr - start,
                                                  IN_GLOBAL_DATA));
            } else {  // pass: no intersection
              live_objects->push_back(AllocObject(i->ptr, i->size,
                                                  IN_GLOBAL_DATA));
            }
          }
          // Move live_objects back into l->second
          // for filtering by the next region.
          live_objects->swap(l->second);
          live_objects->clear();
        }
      }
      // Now get and use live_objects from the final version of l->second:
      if (VLOG_IS_ON(11)) {
        for (LiveObjectsStack::const_iterator i = l->second.begin();
             i != l->second.end(); ++i) {
          RAW_VLOG(11, "Library live region at %p of %" PRIuPTR " bytes",
                      i->ptr, i->size);
        }
      }
      live_objects->swap(l->second);
      IgnoreLiveObjectsLocked("in globals of\n  ", l->first.c_str());
    }
    if (have_null_region_callers) {
      RAW_LOG(ERROR, "Have memory regions w/o callers: "
                     "might report false leaks");
    }
    Allocator::DeleteAndNull(&library_live_objects);
  }
}

// Callback for TCMalloc_ListAllProcessThreads in IgnoreAllLiveObjectsLocked below
// to test/verify that we have just the one main thread, in which case
// we can do everything in that main thread,
// so that CPU profiler can collect all its samples.
// Returns the number of threads in the process.
static int IsOneThread(void* parameter, int num_threads,
                       pid_t* thread_pids, va_list ap) {
  if (num_threads != 1) {
    RAW_LOG(WARNING, "Have threads: Won't CPU-profile the bulk of leak "
                     "checking work happening in IgnoreLiveThreadsLocked!");
  }
  TCMalloc_ResumeAllProcessThreads(num_threads, thread_pids);
  return num_threads;
}

// Dummy for IgnoreAllLiveObjectsLocked below.
// Making it global helps with compiler warnings.
static va_list dummy_ap;

// static
void HeapLeakChecker::IgnoreAllLiveObjectsLocked(const void* self_stack_top) {
  RAW_DCHECK(heap_checker_lock.IsHeld(), "");
  RAW_CHECK(live_objects == NULL, "");
  live_objects = new(Allocator::Allocate(sizeof(LiveObjectsStack)))
                   LiveObjectsStack;
  stack_tops = new(Allocator::Allocate(sizeof(StackTopSet))) StackTopSet;
  // reset the counts
  live_objects_total = 0;
  live_bytes_total = 0;
  // Reduce max_heap_object_size to FLAGS_heap_check_max_pointer_offset
  // for the time of leak check.
  // FLAGS_heap_check_max_pointer_offset caps max_heap_object_size
  // to manage reasonably low chances of random bytes
  // appearing to be pointing into large actually leaked heap objects.
  const size_t old_max_heap_object_size = max_heap_object_size;
  max_heap_object_size = (
    FLAGS_heap_check_max_pointer_offset != -1
    ? min(size_t(FLAGS_heap_check_max_pointer_offset), max_heap_object_size)
    : max_heap_object_size);
  // Record global data as live:
  if (FLAGS_heap_check_ignore_global_live) {
    library_live_objects =
      new(Allocator::Allocate(sizeof(LibraryLiveObjectsStacks)))
        LibraryLiveObjectsStacks;
  }
  // Ignore all thread stacks:
  thread_listing_status = CALLBACK_NOT_STARTED;
  bool need_to_ignore_non_thread_objects = true;
  self_thread_pid = getpid();
  self_thread_stack_top = self_stack_top;
  if (FLAGS_heap_check_ignore_thread_live) {
    // In case we are doing CPU profiling we'd like to do all the work
    // in the main thread, not in the special thread created by
    // TCMalloc_ListAllProcessThreads, so that CPU profiler can
    // collect all its samples.  The machinery of
    // TCMalloc_ListAllProcessThreads conflicts with the CPU profiler
    // by also relying on signals and ::sigaction.  We can do this
    // (run everything in the main thread) safely only if there's just
    // the main thread itself in our process.  This variable reflects
    // these two conditions:
    bool want_and_can_run_in_main_thread =
      ProfilingIsEnabledForAllThreads()  &&
      TCMalloc_ListAllProcessThreads(NULL, IsOneThread) == 1;
    // When the normal path of TCMalloc_ListAllProcessThreads below is taken,
    // we fully suspend the threads right here before any liveness checking
    // and keep them suspended for the whole time of liveness checking
    // inside of the IgnoreLiveThreadsLocked callback.
    // (The threads can't (de)allocate due to lock on the delete hook but
    //  if not suspended they could still mess with the pointer
    //  graph while we walk it).
    int r = want_and_can_run_in_main_thread
            ? IgnoreLiveThreadsLocked(NULL, 1, &self_thread_pid, dummy_ap)
            : TCMalloc_ListAllProcessThreads(NULL, IgnoreLiveThreadsLocked);
    need_to_ignore_non_thread_objects = r < 0;
    if (r < 0) {
      RAW_LOG(WARNING, "Thread finding failed with %d errno=%d", r, errno);
      if (thread_listing_status == CALLBACK_COMPLETED) {
        RAW_LOG(INFO, "Thread finding callback "
                      "finished ok; hopefully everything is fine");
        need_to_ignore_non_thread_objects = false;
      } else if (thread_listing_status == CALLBACK_STARTED) {
        RAW_LOG(FATAL, "Thread finding callback was "
                       "interrupted or crashed; can't fix this");
      } else {  // CALLBACK_NOT_STARTED
        RAW_LOG(ERROR, "Could not find thread stacks. "
                       "Will likely report false leak positives.");
      }
    } else if (r != 0) {
      RAW_LOG(ERROR, "Thread stacks not found for %d threads. "
                     "Will likely report false leak positives.", r);
    } else {
      RAW_VLOG(11, "Thread stacks appear to be found for all threads");
    }
  } else {
    RAW_LOG(WARNING, "Not looking for thread stacks; "
                     "objects reachable only from there "
                     "will be reported as leaks");
  }
  // Do all other live data ignoring here if we did not do it
  // within thread listing callback with all threads stopped.
  if (need_to_ignore_non_thread_objects) {
    if (FLAGS_heap_check_ignore_global_live) {
      UseProcMapsLocked(RECORD_GLOBAL_DATA);
    }
    IgnoreNonThreadLiveObjectsLocked();
  }
  if (live_objects_total) {
    RAW_VLOG(10, "Ignoring %" PRId64 " reachable objects of %" PRId64 " bytes",
                live_objects_total, live_bytes_total);
  }
  // Free these: we made them here and heap_profile never saw them
  Allocator::DeleteAndNull(&live_objects);
  Allocator::DeleteAndNull(&stack_tops);
  max_heap_object_size = old_max_heap_object_size;  // reset this var
}

// Alignment at which we should consider pointer positions
// in IgnoreLiveObjectsLocked. Will normally use the value of
// FLAGS_heap_check_pointer_source_alignment.
static size_t pointer_source_alignment = kPointerSourceAlignment;
// Global lock for HeapLeakChecker::DoNoLeaks
// to protect pointer_source_alignment.
static SpinLock alignment_checker_lock(SpinLock::LINKER_INITIALIZED);

// This function changes the live bits in the heap_profile-table's state:
// we only record the live objects to be skipped.
//
// When checking if a byte sequence points to a heap object we use
// HeapProfileTable::FindInsideAlloc to handle both pointers to
// the start and inside of heap-allocated objects.
// The "inside" case needs to be checked to support
// at least the following relatively common cases:
// - C++ arrays allocated with new FooClass[size] for classes
//   with destructors have their size recorded in a sizeof(int) field
//   before the place normal pointers point to.
// - basic_string<>-s for e.g. the C++ library of gcc 3.4
//   have the meta-info in basic_string<...>::_Rep recorded
//   before the place normal pointers point to.
// - Multiple-inherited objects have their pointers when cast to
//   different base classes pointing inside of the actually
//   allocated object.
// - Sometimes reachability pointers point to member objects of heap objects,
//   and then those member objects point to the full heap object.
// - Third party UnicodeString: it stores a 32-bit refcount
//   (in both 32-bit and 64-bit binaries) as the first uint32
//   in the allocated memory and a normal pointer points at
//   the second uint32 behind the refcount.
// By finding these additional objects here
// we slightly increase the chance to mistake random memory bytes
// for a pointer and miss a leak in a particular run of a binary.
//
/*static*/ void HeapLeakChecker::IgnoreLiveObjectsLocked(const char* name,
                                                         const char* name2) {
  RAW_DCHECK(heap_checker_lock.IsHeld(), "");
  int64 live_object_count = 0;
  int64 live_byte_count = 0;
  while (!live_objects->empty()) {
    const char* object =
      reinterpret_cast<const char*>(live_objects->back().ptr);
    size_t size = live_objects->back().size;
    const ObjectPlacement place = live_objects->back().place;
    live_objects->pop_back();
    if (place == MUST_BE_ON_HEAP  &&  heap_profile->MarkAsLive(object)) {
      live_object_count += 1;
      live_byte_count += size;
    }
    RAW_VLOG(13, "Looking for heap pointers in %p of %" PRIuS " bytes",
                object, size);
    const char* const whole_object = object;
    size_t const whole_size = size;
    // Try interpretting any byte sequence in object,size as a heap pointer:
    const size_t remainder = AsInt(object) % pointer_source_alignment;
    if (remainder) {
      object += pointer_source_alignment - remainder;
      if (size >= pointer_source_alignment - remainder) {
        size -= pointer_source_alignment - remainder;
      } else {
        size = 0;
      }
    }
    if (size < sizeof(void*)) continue;

#ifdef NO_FRAME_POINTER
    // Frame pointer omission requires us to use libunwind, which uses direct
    // mmap and munmap system calls, and that needs special handling.
    if (name2 == kUnnamedProcSelfMapEntry) {
      static const uintptr_t page_mask = ~(getpagesize() - 1);
      const uintptr_t addr = reinterpret_cast<uintptr_t>(object);
      if ((addr & page_mask) == 0 && (size & page_mask) == 0) {
        // This is an object we slurped from /proc/self/maps.
        // It may or may not be readable at this point.
        //
        // In case all the above conditions made a mistake, and the object is
        // not related to libunwind, we also verify that it's not readable
        // before ignoring it.
        if (msync(const_cast<char*>(object), size, MS_ASYNC) != 0) {
          // Skip unreadable object, so we don't crash trying to sweep it.
          RAW_VLOG(0, "Ignoring inaccessible object [%p, %p) "
                   "(msync error %d (%s))",
                   object, object + size, errno, strerror(errno));
          continue;
        }
      }
    }
#endif

    const char* const max_object = object + size - sizeof(void*);
    while (object <= max_object) {
      // potentially unaligned load:
      const uintptr_t addr = *reinterpret_cast<const uintptr_t*>(object);
      // Do fast check before the more expensive HaveOnHeapLocked lookup:
      // this code runs for all memory words that are potentially pointers:
      const bool can_be_on_heap =
        // Order tests by the likelyhood of the test failing in 64/32 bit modes.
        // Yes, this matters: we either lose 5..6% speed in 32 bit mode
        // (which is already slower) or by a factor of 1.5..1.91 in 64 bit mode.
        // After the alignment test got dropped the above performance figures
        // must have changed; might need to revisit this.
#if defined(__x86_64__)
        addr <= max_heap_address  &&  // <= is for 0-sized object with max addr
        min_heap_address <= addr;
#else
        min_heap_address <= addr  &&
        addr <= max_heap_address;  // <= is for 0-sized object with max addr
#endif
      if (can_be_on_heap) {
        const void* ptr = reinterpret_cast<const void*>(addr);
        // Too expensive (inner loop): manually uncomment when debugging:
        // RAW_VLOG(17, "Trying pointer to %p at %p", ptr, object);
        size_t object_size;
        if (HaveOnHeapLocked(&ptr, &object_size)  &&
            heap_profile->MarkAsLive(ptr)) {
          // We take the (hopefully low) risk here of encountering by accident
          // a byte sequence in memory that matches an address of
          // a heap object which is in fact leaked.
          // I.e. in very rare and probably not repeatable/lasting cases
          // we might miss some real heap memory leaks.
          RAW_VLOG(14, "Found pointer to %p of %" PRIuS " bytes at %p "
                      "inside %p of size %" PRIuS "",
                      ptr, object_size, object, whole_object, whole_size);
          if (VLOG_IS_ON(15)) {
            // log call stacks to help debug how come something is not a leak
            HeapProfileTable::AllocInfo alloc;
            if (!heap_profile->FindAllocDetails(ptr, &alloc)) {
              RAW_LOG(FATAL, "FindAllocDetails failed on ptr %p", ptr);
            }
            RAW_LOG(INFO, "New live %p object's alloc stack:", ptr);
            for (int i = 0; i < alloc.stack_depth; ++i) {
              RAW_LOG(INFO, "  @ %p", alloc.call_stack[i]);
            }
          }
          live_object_count += 1;
          live_byte_count += object_size;
          live_objects->push_back(AllocObject(ptr, object_size,
                                              IGNORED_ON_HEAP));
        }
      }
      object += pointer_source_alignment;
    }
  }
  live_objects_total += live_object_count;
  live_bytes_total += live_byte_count;
  if (live_object_count) {
    RAW_VLOG(10, "Removed %" PRId64 " live heap objects of %" PRId64 " bytes: %s%s",
                live_object_count, live_byte_count, name, name2);
  }
}

//----------------------------------------------------------------------
// HeapLeakChecker leak check disabling components
//----------------------------------------------------------------------

// static
void HeapLeakChecker::DisableChecksIn(const char* pattern) {
  RAW_LOG(WARNING, "DisableChecksIn(%s) is ignored", pattern);
}

// static
void HeapLeakChecker::DoIgnoreObject(const void* ptr) {
  SpinLockHolder l(&heap_checker_lock);
  if (!heap_checker_on) return;
  size_t object_size;
  if (!HaveOnHeapLocked(&ptr, &object_size)) {
    RAW_LOG(ERROR, "No live heap object at %p to ignore", ptr);
  } else {
    RAW_VLOG(10, "Going to ignore live object at %p of %" PRIuS " bytes",
                ptr, object_size);
    if (ignored_objects == NULL)  {
      ignored_objects = new(Allocator::Allocate(sizeof(IgnoredObjectsMap)))
                          IgnoredObjectsMap;
    }
    if (!ignored_objects->insert(make_pair(AsInt(ptr), object_size)).second) {
      RAW_LOG(WARNING, "Object at %p is already being ignored", ptr);
    }
  }
}

// static
void HeapLeakChecker::UnIgnoreObject(const void* ptr) {
  SpinLockHolder l(&heap_checker_lock);
  if (!heap_checker_on) return;
  size_t object_size;
  if (!HaveOnHeapLocked(&ptr, &object_size)) {
    RAW_LOG(FATAL, "No live heap object at %p to un-ignore", ptr);
  } else {
    bool found = false;
    if (ignored_objects) {
      IgnoredObjectsMap::iterator object = ignored_objects->find(AsInt(ptr));
      if (object != ignored_objects->end()  &&  object_size == object->second) {
        ignored_objects->erase(object);
        found = true;
        RAW_VLOG(10, "Now not going to ignore live object "
                    "at %p of %" PRIuS " bytes", ptr, object_size);
      }
    }
    if (!found)  RAW_LOG(FATAL, "Object at %p has not been ignored", ptr);
  }
}

//----------------------------------------------------------------------
// HeapLeakChecker non-static functions
//----------------------------------------------------------------------

char* HeapLeakChecker::MakeProfileNameLocked() {
  RAW_DCHECK(lock_->IsHeld(), "");
  RAW_DCHECK(heap_checker_lock.IsHeld(), "");
  const int len = profile_name_prefix->size() + strlen(name_) + 5 +
                  strlen(HeapProfileTable::kFileExt) + 1;
  char* file_name = reinterpret_cast<char*>(Allocator::Allocate(len));
  snprintf(file_name, len, "%s.%s-end%s",
           profile_name_prefix->c_str(), name_,
           HeapProfileTable::kFileExt);
  return file_name;
}

void HeapLeakChecker::Create(const char *name, bool make_start_snapshot) {
  SpinLockHolder l(lock_);
  name_ = NULL;  // checker is inactive
  start_snapshot_ = NULL;
  has_checked_ = false;
  inuse_bytes_increase_ = 0;
  inuse_allocs_increase_ = 0;
  keep_profiles_ = false;
  char* n = new char[strlen(name) + 1];   // do this before we lock
  IgnoreObject(n);  // otherwise it might be treated as live due to our stack
  { // Heap activity in other threads is paused for this whole scope.
    SpinLockHolder al(&alignment_checker_lock);
    SpinLockHolder hl(&heap_checker_lock);
    MemoryRegionMap::LockHolder ml;
    if (heap_checker_on  &&  profile_name_prefix != NULL) {
      RAW_DCHECK(strchr(name, '/') == NULL, "must be a simple name");
      memcpy(n, name, strlen(name) + 1);
      name_ = n;  // checker is active
      if (make_start_snapshot) {
        start_snapshot_ = heap_profile->TakeSnapshot();
      }

      const HeapProfileTable::Stats& t = heap_profile->total();
      const size_t start_inuse_bytes = t.alloc_size - t.free_size;
      const size_t start_inuse_allocs = t.allocs - t.frees;
      RAW_VLOG(10, "Start check \"%s\" profile: %" PRIuS " bytes "
               "in %" PRIuS " objects",
               name_, start_inuse_bytes, start_inuse_allocs);
    } else {
      RAW_LOG(WARNING, "Heap checker is not active, "
                       "hence checker \"%s\" will do nothing!", name);
    RAW_LOG(WARNING, "To activate set the HEAPCHECK environment variable.\n");
    }
  }
  if (name_ == NULL) {
    UnIgnoreObject(n);
    delete[] n;  // must be done after we unlock
  }
}

HeapLeakChecker::HeapLeakChecker(const char *name) : lock_(new SpinLock) {
  RAW_DCHECK(strcmp(name, "_main_") != 0, "_main_ is reserved");
  Create(name, true/*create start_snapshot_*/);
}

HeapLeakChecker::HeapLeakChecker() : lock_(new SpinLock) {
  if (FLAGS_heap_check_before_constructors) {
    // We want to check for leaks of objects allocated during global
    // constructors (i.e., objects allocated already).  So we do not
    // create a baseline snapshot and hence check for leaks of objects
    // that may have already been created.
    Create("_main_", false);
  } else {
    // We want to ignore leaks of objects allocated during global
    // constructors (i.e., objects allocated already).  So we snapshot
    // the current heap contents and use them as a baseline that is
    // not reported by the leak checker.
    Create("_main_", true);
  }
}

ssize_t HeapLeakChecker::BytesLeaked() const {
  SpinLockHolder l(lock_);
  if (!has_checked_) {
    RAW_LOG(FATAL, "*NoLeaks|SameHeap must execute before this call");
  }
  return inuse_bytes_increase_;
}

ssize_t HeapLeakChecker::ObjectsLeaked() const {
  SpinLockHolder l(lock_);
  if (!has_checked_) {
    RAW_LOG(FATAL, "*NoLeaks|SameHeap must execute before this call");
  }
  return inuse_allocs_increase_;
}

// Save pid of main thread for using in naming dump files
static int32 main_thread_pid = getpid();
#ifdef HAVE_PROGRAM_INVOCATION_NAME
#ifdef __UCLIBC__
extern const char* program_invocation_name;
extern const char* program_invocation_short_name;
#else
extern char* program_invocation_name;
extern char* program_invocation_short_name;
#endif
static const char* invocation_name() { return program_invocation_short_name; }
static string invocation_path() { return program_invocation_name; }
#else
static const char* invocation_name() { return "<your binary>"; }
static string invocation_path() { return "<your binary>"; }
#endif

// Prints commands that users can run to get more information
// about the reported leaks.
static void SuggestPprofCommand(const char* pprof_file_arg) {
  // Extra help information to print for the user when the test is
  // being run in a way where the straightforward pprof command will
  // not suffice.
  string extra_help;

  // Common header info to print for remote runs
  const string remote_header =
      "This program is being executed remotely and therefore the pprof\n"
      "command printed above will not work.  Either run this program\n"
      "locally, or adjust the pprof command as follows to allow it to\n"
      "work on your local machine:\n";

  // Extra command for fetching remote data
  string fetch_cmd;

  RAW_LOG(WARNING,
          "\n\n"
          "If the preceding stack traces are not enough to find "
          "the leaks, try running THIS shell command:\n\n"
          "%s%s %s \"%s\" --inuse_objects --lines --heapcheck "
          " --edgefraction=1e-10 --nodefraction=1e-10 --gv\n"
          "\n"
          "%s"
          "If you are still puzzled about why the leaks are "
          "there, try rerunning this program with "
          "HEAP_CHECK_TEST_POINTER_ALIGNMENT=1 and/or with "
          "HEAP_CHECK_MAX_POINTER_OFFSET=-1\n"
          "If the leak report occurs in a small fraction of runs, "
          "try running with TCMALLOC_MAX_FREE_QUEUE_SIZE of few hundred MB "
          "or with TCMALLOC_RECLAIM_MEMORY=false, "  // only works for debugalloc
          "it might help find leaks more repeatably\n",
          fetch_cmd.c_str(),
          "pprof",           // works as long as pprof is on your path
          invocation_path().c_str(),
          pprof_file_arg,
          extra_help.c_str()
          );
}

bool HeapLeakChecker::DoNoLeaks(ShouldSymbolize should_symbolize) {
  SpinLockHolder l(lock_);
  // The locking also helps us keep the messages
  // for the two checks close together.
  SpinLockHolder al(&alignment_checker_lock);

  // thread-safe: protected by alignment_checker_lock
  static bool have_disabled_hooks_for_symbolize = false;
  // Once we've checked for leaks and symbolized the results once, it's
  // not safe to do it again.  This is because in order to symbolize
  // safely, we had to disable all the malloc hooks here, so we no
  // longer can be confident we've collected all the data we need.
  if (have_disabled_hooks_for_symbolize) {
    RAW_LOG(FATAL, "Must not call heap leak checker manually after "
            " program-exit's automatic check.");
  }

  HeapProfileTable::Snapshot* leaks = NULL;
  char* pprof_file = NULL;

  {
    // Heap activity in other threads is paused during this function
    // (i.e. until we got all profile difference info).
    SpinLockHolder hl(&heap_checker_lock);
    if (heap_checker_on == false) {
      if (name_ != NULL) {  // leak checking enabled when created the checker
        RAW_LOG(WARNING, "Heap leak checker got turned off after checker "
                "\"%s\" has been created, no leak check is being done for it!",
                name_);
      }
      return true;
    }

    // Update global_region_caller_ranges. They may need to change since
    // e.g. initialization because shared libraries might have been loaded or
    // unloaded.
    Allocator::DeleteAndNullIfNot(&global_region_caller_ranges);
    ProcMapsResult pm_result = UseProcMapsLocked(DISABLE_LIBRARY_ALLOCS);
    RAW_CHECK(pm_result == PROC_MAPS_USED, "");

    // Keep track of number of internally allocated objects so we
    // can detect leaks in the heap-leak-checket itself
    const int initial_allocs = Allocator::alloc_count();

    if (name_ == NULL) {
      RAW_LOG(FATAL, "Heap leak checker must not be turned on "
              "after construction of a HeapLeakChecker");
    }

    MemoryRegionMap::LockHolder ml;
    int a_local_var;  // Use our stack ptr to make stack data live:

    // Make the heap profile, other threads are locked out.
    HeapProfileTable::Snapshot* base =
        reinterpret_cast<HeapProfileTable::Snapshot*>(start_snapshot_);
    RAW_DCHECK(FLAGS_heap_check_pointer_source_alignment > 0, "");
    pointer_source_alignment = FLAGS_heap_check_pointer_source_alignment;
    IgnoreAllLiveObjectsLocked(&a_local_var);
    leaks = heap_profile->NonLiveSnapshot(base);

    inuse_bytes_increase_ = static_cast<ssize_t>(leaks->total().alloc_size);
    inuse_allocs_increase_ = static_cast<ssize_t>(leaks->total().allocs);
    if (leaks->Empty()) {
      heap_profile->ReleaseSnapshot(leaks);
      leaks = NULL;

      // We can only check for internal leaks along the no-user-leak
      // path since in the leak path we temporarily release
      // heap_checker_lock and another thread can come in and disturb
      // allocation counts.
      if (Allocator::alloc_count() != initial_allocs) {
        RAW_LOG(FATAL, "Internal HeapChecker leak of %d objects ; %d -> %d",
                Allocator::alloc_count() - initial_allocs,
                initial_allocs, Allocator::alloc_count());
      }
    } else if (FLAGS_heap_check_test_pointer_alignment) {
      if (pointer_source_alignment == 1) {
        RAW_LOG(WARNING, "--heap_check_test_pointer_alignment has no effect: "
                "--heap_check_pointer_source_alignment was already set to 1");
      } else {
        // Try with reduced pointer aligment
        pointer_source_alignment = 1;
        IgnoreAllLiveObjectsLocked(&a_local_var);
        HeapProfileTable::Snapshot* leaks_wo_align =
            heap_profile->NonLiveSnapshot(base);
        pointer_source_alignment = FLAGS_heap_check_pointer_source_alignment;
        if (leaks_wo_align->Empty()) {
          RAW_LOG(WARNING, "Found no leaks without pointer alignment: "
                  "something might be placing pointers at "
                  "unaligned addresses! This needs to be fixed.");
        } else {
          RAW_LOG(INFO, "Found leaks without pointer alignment as well: "
                  "unaligned pointers must not be the cause of leaks.");
          RAW_LOG(INFO, "--heap_check_test_pointer_alignment did not help "
                  "to diagnose the leaks.");
        }
        heap_profile->ReleaseSnapshot(leaks_wo_align);
      }
    }

    if (leaks != NULL) {
      pprof_file = MakeProfileNameLocked();
    }
  }

  has_checked_ = true;
  if (leaks == NULL) {
    if (FLAGS_heap_check_max_pointer_offset == -1) {
      RAW_LOG(WARNING,
              "Found no leaks without max_pointer_offset restriction: "
              "it's possible that the default value of "
              "heap_check_max_pointer_offset flag is too low. "
              "Do you use pointers with larger than that offsets "
              "pointing in the middle of heap-allocated objects?");
    }
    const HeapProfileTable::Stats& stats = heap_profile->total();
    RAW_VLOG(heap_checker_info_level,
             "No leaks found for check \"%s\" "
             "(but no 100%% guarantee that there aren't any): "
             "found %" PRId64 " reachable heap objects of %" PRId64 " bytes",
             name_,
             int64(stats.allocs - stats.frees),
             int64(stats.alloc_size - stats.free_size));
  } else {
    if (should_symbolize == SYMBOLIZE) {
      // To turn addresses into symbols, we need to fork, which is a
      // problem if both parent and child end up trying to call the
      // same malloc-hooks we've set up, at the same time.  To avoid
      // trouble, we turn off the hooks before symbolizing.  Note that
      // this makes it unsafe to ever leak-report again!  Luckily, we
      // typically only want to report once in a program's run, at the
      // very end.
      if (MallocHook::GetNewHook() == NewHook)
        MallocHook::SetNewHook(NULL);
      if (MallocHook::GetDeleteHook() == DeleteHook)
        MallocHook::SetDeleteHook(NULL);
      MemoryRegionMap::Shutdown();
      // Make sure all the hooks really got unset:
      RAW_CHECK(MallocHook::GetNewHook() == NULL, "");
      RAW_CHECK(MallocHook::GetDeleteHook() == NULL, "");
      RAW_CHECK(MallocHook::GetMmapHook() == NULL, "");
      RAW_CHECK(MallocHook::GetSbrkHook() == NULL, "");
      have_disabled_hooks_for_symbolize = true;
      leaks->ReportLeaks(name_, pprof_file, true);  // true = should_symbolize
    } else {
      leaks->ReportLeaks(name_, pprof_file, false);
    }
    if (FLAGS_heap_check_identify_leaks) {
      leaks->ReportIndividualObjects();
    }

    SuggestPprofCommand(pprof_file);

    {
      SpinLockHolder hl(&heap_checker_lock);
      heap_profile->ReleaseSnapshot(leaks);
      Allocator::Free(pprof_file);
    }
  }

  return (leaks == NULL);
}

HeapLeakChecker::~HeapLeakChecker() {
  if (name_ != NULL) {  // had leak checking enabled when created the checker
    if (!has_checked_) {
      RAW_LOG(FATAL, "Some *NoLeaks|SameHeap method"
                     " must be called on any created HeapLeakChecker");
    }

    // Deallocate any snapshot taken at start
    if (start_snapshot_ != NULL) {
      SpinLockHolder l(&heap_checker_lock);
      heap_profile->ReleaseSnapshot(
          reinterpret_cast<HeapProfileTable::Snapshot*>(start_snapshot_));
    }

    UnIgnoreObject(name_);
    delete[] name_;
    name_ = NULL;
  }
  delete lock_;
}

//----------------------------------------------------------------------
// HeapLeakChecker overall heap check components
//----------------------------------------------------------------------

// static
bool HeapLeakChecker::IsActive() {
  SpinLockHolder l(&heap_checker_lock);
  return heap_checker_on;
}

vector<HeapCleaner::void_function>* HeapCleaner::heap_cleanups_ = NULL;

// When a HeapCleaner object is intialized, add its function to the static list
// of cleaners to be run before leaks checking.
HeapCleaner::HeapCleaner(void_function f) {
  if (heap_cleanups_ == NULL)
    heap_cleanups_ = new vector<HeapCleaner::void_function>;
  heap_cleanups_->push_back(f);
}

// Run all of the cleanup functions and delete the vector.
void HeapCleaner::RunHeapCleanups() {
  if (!heap_cleanups_)
    return;
  for (int i = 0; i < heap_cleanups_->size(); i++) {
    void (*f)(void) = (*heap_cleanups_)[i];
    f();
  }
  delete heap_cleanups_;
  heap_cleanups_ = NULL;
}

// Program exit heap cleanup registered as a module object destructor.
// Will not get executed when we crash on a signal.
//
void HeapLeakChecker_RunHeapCleanups() {
  if (FLAGS_heap_check == "local")   // don't check heap in this mode
    return;
  { SpinLockHolder l(&heap_checker_lock);
    // can get here (via forks?) with other pids
    if (heap_checker_pid != getpid()) return;
  }
  HeapCleaner::RunHeapCleanups();
  if (!FLAGS_heap_check_after_destructors) HeapLeakChecker::DoMainHeapCheck();
}

static bool internal_init_start_has_run = false;

// Called exactly once, before main() (but hopefully just before).
// This picks a good unique name for the dumped leak checking heap profiles.
//
// Because we crash when InternalInitStart is called more than once,
// it's fine that we hold heap_checker_lock only around pieces of
// this function: this is still enough for thread-safety w.r.t. other functions
// of this module.
// We can't hold heap_checker_lock throughout because it would deadlock
// on a memory allocation since our new/delete hooks can be on.
//
void HeapLeakChecker_InternalInitStart() {
  { SpinLockHolder l(&heap_checker_lock);
    RAW_CHECK(!internal_init_start_has_run,
              "Heap-check constructor called twice.  Perhaps you both linked"
              " in the heap checker, and also used LD_PRELOAD to load it?");
    internal_init_start_has_run = true;

#ifdef ADDRESS_SANITIZER
    // AddressSanitizer's custom malloc conflicts with HeapChecker.
    FLAGS_heap_check = "";
#endif

    if (FLAGS_heap_check.empty()) {
      // turns out we do not need checking in the end; can stop profiling
      HeapLeakChecker::TurnItselfOffLocked();
      return;
    } else if (RunningOnValgrind()) {
      // There is no point in trying -- we'll just fail.
      RAW_LOG(WARNING, "Can't run under Valgrind; will turn itself off");
      HeapLeakChecker::TurnItselfOffLocked();
      return;
    }
  }

  // Changing this to false can be useful when debugging heap-checker itself:
  if (!FLAGS_heap_check_run_under_gdb && IsDebuggerAttached()) {
    RAW_LOG(WARNING, "Someone is ptrace()ing us; will turn itself off");
    SpinLockHolder l(&heap_checker_lock);
    HeapLeakChecker::TurnItselfOffLocked();
    return;
  }

  { SpinLockHolder l(&heap_checker_lock);
    if (!constructor_heap_profiling) {
      RAW_LOG(FATAL, "Can not start so late. You have to enable heap checking "
	             "with HEAPCHECK=<mode>.");
    }
  }

  // Set all flags
  RAW_DCHECK(FLAGS_heap_check_pointer_source_alignment > 0, "");
  if (FLAGS_heap_check == "minimal") {
    // The least we can check.
    FLAGS_heap_check_before_constructors = false;  // from after main
                                                   // (ignore more)
    FLAGS_heap_check_after_destructors = false;  // to after cleanup
                                                 // (most data is live)
    FLAGS_heap_check_ignore_thread_live = true;  // ignore all live
    FLAGS_heap_check_ignore_global_live = true;  // ignore all live
  } else if (FLAGS_heap_check == "normal") {
    // Faster than 'minimal' and not much stricter.
    FLAGS_heap_check_before_constructors = true;  // from no profile (fast)
    FLAGS_heap_check_after_destructors = false;  // to after cleanup
                                                 // (most data is live)
    FLAGS_heap_check_ignore_thread_live = true;  // ignore all live
    FLAGS_heap_check_ignore_global_live = true;  // ignore all live
  } else if (FLAGS_heap_check == "strict") {
    // A bit stricter than 'normal': global destructors must fully clean up
    // after themselves if they are present.
    FLAGS_heap_check_before_constructors = true;  // from no profile (fast)
    FLAGS_heap_check_after_destructors = true;  // to after destructors
                                                // (less data live)
    FLAGS_heap_check_ignore_thread_live = true;  // ignore all live
    FLAGS_heap_check_ignore_global_live = true;  // ignore all live
  } else if (FLAGS_heap_check == "draconian") {
    // Drop not very portable and not very exact live heap flooding.
    FLAGS_heap_check_before_constructors = true;  // from no profile (fast)
    FLAGS_heap_check_after_destructors = true;  // to after destructors
                                                // (need them)
    FLAGS_heap_check_ignore_thread_live = false;  // no live flood (stricter)
    FLAGS_heap_check_ignore_global_live = false;  // no live flood (stricter)
  } else if (FLAGS_heap_check == "as-is") {
    // do nothing: use other flags as is
  } else if (FLAGS_heap_check == "local") {
    // do nothing
  } else {
    RAW_LOG(FATAL, "Unsupported heap_check flag: %s",
                   FLAGS_heap_check.c_str());
  }
  // FreeBSD doesn't seem to honor atexit execution order:
  //    http://code.google.com/p/gperftools/issues/detail?id=375
  // Since heap-checking before destructors depends on atexit running
  // at the right time, on FreeBSD we always check after, even in the
  // less strict modes.  This just means FreeBSD is always a bit
  // stricter in its checking than other OSes.
  // This now appears to be the case in other OSes as well;
  // so always check afterwards.
  FLAGS_heap_check_after_destructors = true;

  { SpinLockHolder l(&heap_checker_lock);
    RAW_DCHECK(heap_checker_pid == getpid(), "");
    heap_checker_on = true;
    RAW_DCHECK(heap_profile, "");
    HeapLeakChecker::ProcMapsResult pm_result = HeapLeakChecker::UseProcMapsLocked(HeapLeakChecker::DISABLE_LIBRARY_ALLOCS);
      // might neeed to do this more than once
      // if one later dynamically loads libraries that we want disabled
    if (pm_result != HeapLeakChecker::PROC_MAPS_USED) {  // can't function
      HeapLeakChecker::TurnItselfOffLocked();
      return;
    }
  }

  // make a good place and name for heap profile leak dumps
  string* profile_prefix =
    new string(FLAGS_heap_check_dump_directory + "/" + invocation_name());

  // Finalize prefix for dumping leak checking profiles.
  const int32 our_pid = getpid();   // safest to call getpid() outside lock
  { SpinLockHolder l(&heap_checker_lock);
    // main_thread_pid might still be 0 if this function is being called before
    // global constructors.  In that case, our pid *is* the main pid.
    if (main_thread_pid == 0)
      main_thread_pid = our_pid;
  }
  char pid_buf[15];
  snprintf(pid_buf, sizeof(pid_buf), ".%d", main_thread_pid);
  *profile_prefix += pid_buf;
  { SpinLockHolder l(&heap_checker_lock);
    RAW_DCHECK(profile_name_prefix == NULL, "");
    profile_name_prefix = profile_prefix;
  }

  // Make sure new/delete hooks are installed properly
  // and heap profiler is indeed able to keep track
  // of the objects being allocated.
  // We test this to make sure we are indeed checking for leaks.
  char* test_str = new char[5];
  size_t size;
  { SpinLockHolder l(&heap_checker_lock);
    RAW_CHECK(heap_profile->FindAlloc(test_str, &size),
              "our own new/delete not linked?");
  }
  delete[] test_str;
  { SpinLockHolder l(&heap_checker_lock);
    // This check can fail when it should not if another thread allocates
    // into this same spot right this moment,
    // which is unlikely since this code runs in InitGoogle.
    RAW_CHECK(!heap_profile->FindAlloc(test_str, &size),
              "our own new/delete not linked?");
  }
  // If we crash in the above code, it probably means that
  // "nm <this_binary> | grep new" will show that tcmalloc's new/delete
  // implementation did not get linked-in into this binary
  // (i.e. nm will list __builtin_new and __builtin_vec_new as undefined).
  // If this happens, it is a BUILD bug to be fixed.

  RAW_VLOG(heap_checker_info_level,
           "WARNING: Perftools heap leak checker is active "
           "-- Performance may suffer");

  if (FLAGS_heap_check != "local") {
    HeapLeakChecker* main_hc = new HeapLeakChecker();
    SpinLockHolder l(&heap_checker_lock);
    RAW_DCHECK(main_heap_checker == NULL,
               "Repeated creation of main_heap_checker");
    main_heap_checker = main_hc;
    do_main_heap_check = true;
  }

  { SpinLockHolder l(&heap_checker_lock);
    RAW_CHECK(heap_checker_on  &&  constructor_heap_profiling,
              "Leak checking is expected to be fully turned on now");
  }

  // For binaries built in debug mode, this will set release queue of
  // debugallocation.cc to 100M to make it less likely for real leaks to
  // be hidden due to reuse of heap memory object addresses.
  // Running a test with --malloc_reclaim_memory=0 would help find leaks even
  // better, but the test might run out of memory as a result.
  // The scenario is that a heap object at address X is allocated and freed,
  // but some other data-structure still retains a pointer to X.
  // Then the same heap memory is used for another object, which is leaked,
  // but the leak is not noticed due to the pointer to the original object at X.
  // TODO(csilvers): support this in some manner.
#if 0
  SetCommandLineOptionWithMode("max_free_queue_size", "104857600",  // 100M
                               SET_FLAG_IF_DEFAULT);
#endif
}

// We want this to run early as well, but not so early as
// ::BeforeConstructors (we want flag assignments to have already
// happened, for instance).  Initializer-registration does the trick.
REGISTER_MODULE_INITIALIZER(init_start, HeapLeakChecker_InternalInitStart());
REGISTER_MODULE_DESTRUCTOR(init_start, HeapLeakChecker_RunHeapCleanups());

// static
bool HeapLeakChecker::NoGlobalLeaksMaybeSymbolize(
    ShouldSymbolize should_symbolize) {
  // we never delete or change main_heap_checker once it's set:
  HeapLeakChecker* main_hc = GlobalChecker();
  if (main_hc) {
    RAW_VLOG(10, "Checking for whole-program memory leaks");
    return main_hc->DoNoLeaks(should_symbolize);
  }
  return true;
}

// static
bool HeapLeakChecker::DoMainHeapCheck() {
  if (FLAGS_heap_check_delay_seconds > 0) {
    sleep(FLAGS_heap_check_delay_seconds);
  }
  { SpinLockHolder l(&heap_checker_lock);
    if (!do_main_heap_check) return false;
    RAW_DCHECK(heap_checker_pid == getpid(), "");
    do_main_heap_check = false;  // will do it now; no need to do it more
  }

  // The program is over, so it's safe to symbolize addresses (which
  // requires a fork) because no serious work is expected to be done
  // after this.  Symbolizing is really useful -- knowing what
  // function has a leak is better than knowing just an address --
  // and while we can only safely symbolize once in a program run,
  // now is the time (after all, there's no "later" that would be better).
  if (!NoGlobalLeaksMaybeSymbolize(SYMBOLIZE)) {
    if (FLAGS_heap_check_identify_leaks) {
      RAW_LOG(FATAL, "Whole-program memory leaks found.");
    }
    RAW_LOG(ERROR, "Exiting with error code (instead of crashing) "
                   "because of whole-program memory leaks");
    _exit(1);    // we don't want to call atexit() routines!
  }
  return true;
}

// static
HeapLeakChecker* HeapLeakChecker::GlobalChecker() {
  SpinLockHolder l(&heap_checker_lock);
  return main_heap_checker;
}

// static
bool HeapLeakChecker::NoGlobalLeaks() {
  // symbolizing requires a fork, which isn't safe to do in general.
  return NoGlobalLeaksMaybeSymbolize(DO_NOT_SYMBOLIZE);
}

// static
void HeapLeakChecker::CancelGlobalCheck() {
  SpinLockHolder l(&heap_checker_lock);
  if (do_main_heap_check) {
    RAW_VLOG(heap_checker_info_level,
             "Canceling the automatic at-exit whole-program memory leak check");
    do_main_heap_check = false;
  }
}

// static
void HeapLeakChecker::BeforeConstructorsLocked() {
  RAW_DCHECK(heap_checker_lock.IsHeld(), "");
  RAW_CHECK(!constructor_heap_profiling,
            "BeforeConstructorsLocked called multiple times");
#ifdef ADDRESS_SANITIZER
  // AddressSanitizer's custom malloc conflicts with HeapChecker.
  return;
#endif
  // Set hooks early to crash if 'new' gets called before we make heap_profile,
  // and make sure no other hooks existed:
  RAW_CHECK(MallocHook::AddNewHook(&NewHook), "");
  RAW_CHECK(MallocHook::AddDeleteHook(&DeleteHook), "");
  constructor_heap_profiling = true;
  MemoryRegionMap::Init(1, /* use_buckets */ false);
    // Set up MemoryRegionMap with (at least) one caller stack frame to record
    // (important that it's done before HeapProfileTable creation below).
  Allocator::Init();
  RAW_CHECK(heap_profile == NULL, "");
  heap_profile = new(Allocator::Allocate(sizeof(HeapProfileTable)))
      HeapProfileTable(&Allocator::Allocate, &Allocator::Free,
                       /* profile_mmap */ false);
  RAW_VLOG(10, "Starting tracking the heap");
  heap_checker_on = true;
}

// static
void HeapLeakChecker::TurnItselfOffLocked() {
  RAW_DCHECK(heap_checker_lock.IsHeld(), "");
  // Set FLAGS_heap_check to "", for users who test for it
  if (!FLAGS_heap_check.empty())  // be a noop in the common case
    FLAGS_heap_check.clear();     // because clear() could allocate memory
  if (constructor_heap_profiling) {
    RAW_CHECK(heap_checker_on, "");
    RAW_VLOG(heap_checker_info_level, "Turning perftools heap leak checking off");
    heap_checker_on = false;
    // Unset our hooks checking they were set:
    RAW_CHECK(MallocHook::RemoveNewHook(&NewHook), "");
    RAW_CHECK(MallocHook::RemoveDeleteHook(&DeleteHook), "");
    Allocator::DeleteAndNull(&heap_profile);
    // free our optional global data:
    Allocator::DeleteAndNullIfNot(&ignored_objects);
    Allocator::DeleteAndNullIfNot(&disabled_ranges);
    Allocator::DeleteAndNullIfNot(&global_region_caller_ranges);
    Allocator::Shutdown();
    MemoryRegionMap::Shutdown();
  }
  RAW_CHECK(!heap_checker_on, "");
}

extern bool heap_leak_checker_bcad_variable;  // in heap-checker-bcad.cc

static bool has_called_before_constructors = false;

// TODO(maxim): inline this function with
// MallocHook_InitAtFirstAllocation_HeapLeakChecker, and also rename
// HeapLeakChecker::BeforeConstructorsLocked.
void HeapLeakChecker_BeforeConstructors() {
  SpinLockHolder l(&heap_checker_lock);
  // We can be called from several places: the first mmap/sbrk/alloc call
  // or the first global c-tor from heap-checker-bcad.cc:
  // Do not re-execute initialization:
  if (has_called_before_constructors) return;
  has_called_before_constructors = true;

  heap_checker_pid = getpid();  // set it always
  heap_leak_checker_bcad_variable = true;
  // just to reference it, so that heap-checker-bcad.o is linked in

  // This function can be called *very* early, before the normal
  // global-constructor that sets FLAGS_verbose.  Set it manually now,
  // so the RAW_LOG messages here are controllable.
  const char* verbose_str = GetenvBeforeMain("PERFTOOLS_VERBOSE");
  if (verbose_str && atoi(verbose_str)) {  // different than the default of 0?
    FLAGS_verbose = atoi(verbose_str);
  }

  bool need_heap_check = true;
  // The user indicates a desire for heap-checking via the HEAPCHECK
  // environment variable.  If it's not set, there's no way to do
  // heap-checking.
  if (!GetenvBeforeMain("HEAPCHECK")) {
    need_heap_check = false;
  }
#ifdef HAVE_GETEUID
  if (need_heap_check && getuid() != geteuid()) {
    // heap-checker writes out files.  Thus, for security reasons, we don't
    // recognize the env. var. to turn on heap-checking if we're setuid.
    RAW_LOG(WARNING, ("HeapChecker: ignoring HEAPCHECK because "
                      "program seems to be setuid\n"));
    need_heap_check = false;
  }
#endif
  if (need_heap_check) {
    HeapLeakChecker::BeforeConstructorsLocked();
  }
}

// This function overrides the weak function defined in malloc_hook.cc and
// called by one of the initial malloc hooks (malloc_hook.cc) when the very
// first memory allocation or an mmap/sbrk happens.  This ensures that
// HeapLeakChecker is initialized and installs all its hooks early enough to
// track absolutely all memory allocations and all memory region acquisitions
// via mmap and sbrk.
extern "C" void MallocHook_InitAtFirstAllocation_HeapLeakChecker() {
  HeapLeakChecker_BeforeConstructors();
}

// This function is executed after all global object destructors run.
void HeapLeakChecker_AfterDestructors() {
  { SpinLockHolder l(&heap_checker_lock);
    // can get here (via forks?) with other pids
    if (heap_checker_pid != getpid()) return;
  }
  if (FLAGS_heap_check_after_destructors) {
    if (HeapLeakChecker::DoMainHeapCheck()) {
      const struct timespec sleep_time = { 0, 500000000 };  // 500 ms
      nanosleep(&sleep_time, NULL);
        // Need this hack to wait for other pthreads to exit.
        // Otherwise tcmalloc find errors
        // on a free() call from pthreads.
    }
  }
  SpinLockHolder l(&heap_checker_lock);
  RAW_CHECK(!do_main_heap_check, "should have done it");
}

//----------------------------------------------------------------------
// HeapLeakChecker disabling helpers
//----------------------------------------------------------------------

// These functions are at the end of the file to prevent their inlining:

// static
void HeapLeakChecker::DisableChecksFromToLocked(const void* start_address,
                                                const void* end_address,
                                                int max_depth) {
  RAW_DCHECK(heap_checker_lock.IsHeld(), "");
  RAW_DCHECK(start_address < end_address, "");
  if (disabled_ranges == NULL) {
    disabled_ranges = new(Allocator::Allocate(sizeof(DisabledRangeMap)))
                        DisabledRangeMap;
  }
  RangeValue value;
  value.start_address = AsInt(start_address);
  value.max_depth = max_depth;
  if (disabled_ranges->insert(make_pair(AsInt(end_address), value)).second) {
    RAW_VLOG(10, "Disabling leak checking in stack traces "
                "under frame addresses between %p..%p",
                start_address, end_address);
  } else {  // check that this is just a verbatim repetition
    RangeValue const& val = disabled_ranges->find(AsInt(end_address))->second;
    if (val.max_depth != value.max_depth  ||
        val.start_address != value.start_address) {
      RAW_LOG(FATAL, "Two DisableChecksToHereFrom calls conflict: "
                     "(%p, %p, %d) vs. (%p, %p, %d)",
                     AsPtr(val.start_address), end_address, val.max_depth,
                     start_address, end_address, max_depth);
    }
  }
}

// static
inline bool HeapLeakChecker::HaveOnHeapLocked(const void** ptr,
                                              size_t* object_size) {
  // Commented-out because HaveOnHeapLocked is very performance-critical:
  // RAW_DCHECK(heap_checker_lock.IsHeld(), "");
  const uintptr_t addr = AsInt(*ptr);
  if (heap_profile->FindInsideAlloc(
        *ptr, max_heap_object_size, ptr, object_size)) {
    RAW_VLOG(16, "Got pointer into %p at +%" PRIuPTR " offset",
             *ptr, addr - AsInt(*ptr));
    return true;
  }
  return false;
}

// static
const void* HeapLeakChecker::GetAllocCaller(void* ptr) {
  // this is used only in the unittest, so the heavy checks are fine
  HeapProfileTable::AllocInfo info;
  { SpinLockHolder l(&heap_checker_lock);
    RAW_CHECK(heap_profile->FindAllocDetails(ptr, &info), "");
  }
  RAW_CHECK(info.stack_depth >= 1, "");
  return info.call_stack[0];
}