1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
|
<!DOCTYPE html>
<html lang="en" data-content_root="../" >
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Hints for Performance Tuning — PETSc 3.23.1 documentation</title>
<script data-cfasync="false">
document.documentElement.dataset.mode = localStorage.getItem("mode") || "";
document.documentElement.dataset.theme = localStorage.getItem("theme") || "light";
</script>
<!-- Loaded before other Sphinx assets -->
<link href="../_static/styles/theme.css?digest=bd9e20870c6007c4c509" rel="stylesheet" />
<link href="../_static/styles/bootstrap.css?digest=bd9e20870c6007c4c509" rel="stylesheet" />
<link href="../_static/styles/pydata-sphinx-theme.css?digest=bd9e20870c6007c4c509" rel="stylesheet" />
<link href="../_static/vendor/fontawesome/6.5.1/css/all.min.css?digest=bd9e20870c6007c4c509" rel="stylesheet" />
<link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.5.1/webfonts/fa-solid-900.woff2" />
<link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.5.1/webfonts/fa-brands-400.woff2" />
<link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.5.1/webfonts/fa-regular-400.woff2" />
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=8f2a1f02" />
<link rel="stylesheet" type="text/css" href="../_static/copybutton.css?v=76b2166b" />
<link rel="stylesheet" type="text/css" href="../_static/sphinx-design.min.css?v=87e54e7c" />
<link rel="stylesheet" type="text/css" href="https://cdn.jsdelivr.net/npm/katex@0.16.10/dist/katex.min.css" />
<link rel="stylesheet" type="text/css" href="../_static/katex-math.css?v=91adb8b6" />
<link rel="stylesheet" type="text/css" href="../_static/css/custom.css?v=dbe1606d" />
<!-- Pre-loaded scripts that we'll load fully later -->
<link rel="preload" as="script" href="../_static/scripts/bootstrap.js?digest=bd9e20870c6007c4c509" />
<link rel="preload" as="script" href="../_static/scripts/pydata-sphinx-theme.js?digest=bd9e20870c6007c4c509" />
<script src="../_static/vendor/fontawesome/6.5.1/js/all.min.js?digest=bd9e20870c6007c4c509"></script>
<script src="../_static/documentation_options.js?v=34da53a5"></script>
<script src="../_static/doctools.js?v=9a2dae69"></script>
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
<script src="../_static/clipboard.min.js?v=a7894cd8"></script>
<script src="../_static/copybutton.js?v=a56c686a"></script>
<script src="../_static/design-tabs.js?v=f930bc37"></script>
<script src="../_static/katex.min.js?v=be8ff15f"></script>
<script src="../_static/auto-render.min.js?v=ad136472"></script>
<script src="../_static/katex_autorenderer.js?v=bebc588a"></script>
<script>DOCUMENTATION_OPTIONS.pagename = 'manual/performance';</script>
<link rel="icon" href="../_static/petsc_favicon.png"/>
<link rel="index" title="Index" href="../genindex.html" />
<link rel="search" title="Search" href="../search.html" />
<link rel="next" title="STREAMS: Example Study" href="streams.html" />
<link rel="prev" title="Profiling" href="profiling.html" />
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<meta name="docsearch:language" content="en"/>
<meta name="docbuild:last-update" content="2025-04-30T13:10:40-0500 (v3.23.1)"/>
</head>
<body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="">
<a id="pst-skip-link" class="skip-link" href="#main-content">Skip to main content</a>
<div id="pst-scroll-pixel-helper"></div>
<button type="button" class="btn rounded-pill" id="pst-back-to-top">
<i class="fa-solid fa-arrow-up"></i>
Back to top
</button>
<input type="checkbox"
class="sidebar-toggle"
name="__primary"
id="__primary"/>
<label class="overlay overlay-primary" for="__primary"></label>
<input type="checkbox"
class="sidebar-toggle"
name="__secondary"
id="__secondary"/>
<label class="overlay overlay-secondary" for="__secondary"></label>
<div class="search-button__wrapper">
<div class="search-button__overlay"></div>
<div class="search-button__search-container">
<form class="bd-search d-flex align-items-center"
action="../search.html"
method="get">
<i class="fa-solid fa-magnifying-glass"></i>
<input type="search"
class="form-control"
name="q"
id="search-input"
placeholder="Search the docs ..."
aria-label="Search the docs ..."
autocomplete="off"
autocorrect="off"
autocapitalize="off"
spellcheck="false"/>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
</form></div>
</div>
<header>
<div class="bd-header navbar navbar-expand-lg bd-navbar">
<div class="bd-header__inner bd-page-width">
<label class="sidebar-toggle primary-toggle" for="__primary">
<span class="fa-solid fa-bars"></span>
</label>
<div class="col-lg-3 navbar-header-items__start">
<div class="navbar-item">
<a class="navbar-brand logo" href="../index.html">
<img src="../_static/PETSc-TAO_RGB.svg" class="logo__image only-light" alt="PETSc 3.23.1 documentation - Home"/>
<script>document.write(`<img src="../_static/PETSc-TAO_RGB_white.svg" class="logo__image only-dark" alt="PETSc 3.23.1 documentation - Home"/>`);</script>
</a></div>
</div>
<div class="col-lg-9 navbar-header-items">
<div class="me-auto navbar-header-items__center">
<div class="navbar-item">
<nav class="navbar-nav">
<ul class="bd-navbar-elements navbar-nav">
<li class="nav-item current active">
<a class="nav-link nav-internal" href="../overview/index.html">
Overview
</a>
</li>
<li class="nav-item">
<a class="nav-link nav-internal" href="../install/index.html">
Install
</a>
</li>
<li class="nav-item">
<a class="nav-link nav-internal" href="../tutorials/index.html">
Tutorials
</a>
</li>
<li class="nav-item">
<a class="nav-link nav-internal" href="index.html">
User-Guide
</a>
</li>
<li class="nav-item">
<a class="nav-link nav-internal" href="../manualpages/index.html">
C/Fortran API
</a>
</li>
<li class="nav-item">
<a class="nav-link nav-internal" href="../petsc4py/index.html">
petsc4py API
</a>
</li>
<li class="nav-item">
<a class="nav-link nav-internal" href="../faq/index.html">
FAQ
</a>
</li>
<li class="nav-item">
<a class="nav-link nav-internal" href="../community/index.html">
Community
</a>
</li>
<li class="nav-item">
<a class="nav-link nav-internal" href="../developers/index.html">
Developers
</a>
</li>
<li class="nav-item">
<a class="nav-link nav-internal" href="../miscellaneous/index.html">
Misc.
</a>
</li>
</ul>
</nav></div>
</div>
<div class="navbar-header-items__end">
<div class="navbar-item navbar-persistent--container">
<script>
document.write(`
<button class="btn navbar-btn search-button-field search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="fa-solid fa-magnifying-glass"></i>
<span class="search-button__default-text">Search</span>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
</button>
`);
</script>
</div>
<div class="navbar-item">
<script>
document.write(`
<button class="btn btn-sm navbar-btn theme-switch-button" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip">
<span class="theme-switch nav-link" data-mode="light"><i class="fa-solid fa-sun fa-lg"></i></span>
<span class="theme-switch nav-link" data-mode="dark"><i class="fa-solid fa-moon fa-lg"></i></span>
<span class="theme-switch nav-link" data-mode="auto"><i class="fa-solid fa-circle-half-stroke fa-lg"></i></span>
</button>
`);
</script></div>
<div class="navbar-item"><ul class="navbar-icon-links navbar-nav"
aria-label="Icon Links">
<li class="nav-item">
<a href="https://gitlab.com/petsc/petsc" title="GitLab" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fab fa-gitlab fa-lg" aria-hidden="true"></i></span>
<span class="sr-only">GitLab</span></a>
</li>
</ul></div>
</div>
</div>
<div class="navbar-persistent--mobile">
<script>
document.write(`
<button class="btn navbar-btn search-button-field search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="fa-solid fa-magnifying-glass"></i>
<span class="search-button__default-text">Search</span>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
</button>
`);
</script>
</div>
<label class="sidebar-toggle secondary-toggle" for="__secondary" tabindex="0">
<span class="fa-solid fa-outdent"></span>
</label>
</div>
</div>
</header>
<div class="bd-container">
<div class="bd-container__inner bd-page-width">
<div class="bd-sidebar-primary bd-sidebar">
<div class="sidebar-header-items sidebar-primary__section">
<div class="sidebar-header-items__center">
<div class="navbar-item">
<nav class="navbar-nav">
<ul class="bd-navbar-elements navbar-nav">
<li class="nav-item current active">
<a class="nav-link nav-internal" href="../overview/index.html">
Overview
</a>
</li>
<li class="nav-item">
<a class="nav-link nav-internal" href="../install/index.html">
Install
</a>
</li>
<li class="nav-item">
<a class="nav-link nav-internal" href="../tutorials/index.html">
Tutorials
</a>
</li>
<li class="nav-item">
<a class="nav-link nav-internal" href="index.html">
User-Guide
</a>
</li>
<li class="nav-item">
<a class="nav-link nav-internal" href="../manualpages/index.html">
C/Fortran API
</a>
</li>
<li class="nav-item">
<a class="nav-link nav-internal" href="../petsc4py/index.html">
petsc4py API
</a>
</li>
<li class="nav-item">
<a class="nav-link nav-internal" href="../faq/index.html">
FAQ
</a>
</li>
<li class="nav-item">
<a class="nav-link nav-internal" href="../community/index.html">
Community
</a>
</li>
<li class="nav-item">
<a class="nav-link nav-internal" href="../developers/index.html">
Developers
</a>
</li>
<li class="nav-item">
<a class="nav-link nav-internal" href="../miscellaneous/index.html">
Misc.
</a>
</li>
</ul>
</nav></div>
</div>
<div class="sidebar-header-items__end">
<div class="navbar-item">
<script>
document.write(`
<button class="btn btn-sm navbar-btn theme-switch-button" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip">
<span class="theme-switch nav-link" data-mode="light"><i class="fa-solid fa-sun fa-lg"></i></span>
<span class="theme-switch nav-link" data-mode="dark"><i class="fa-solid fa-moon fa-lg"></i></span>
<span class="theme-switch nav-link" data-mode="auto"><i class="fa-solid fa-circle-half-stroke fa-lg"></i></span>
</button>
`);
</script></div>
<div class="navbar-item"><ul class="navbar-icon-links navbar-nav"
aria-label="Icon Links">
<li class="nav-item">
<a href="https://gitlab.com/petsc/petsc" title="GitLab" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fab fa-gitlab fa-lg" aria-hidden="true"></i></span>
<span class="sr-only">GitLab</span></a>
</li>
</ul></div>
</div>
</div>
<div class="sidebar-primary-items__start sidebar-primary__section">
<div class="sidebar-primary-item">
<nav class="bd-docs-nav bd-links"
aria-label="Section Navigation">
<p class="bd-links__title" role="heading" aria-level="1">Section Navigation</p>
<div class="bd-toc-item navbar-nav"><ul class="current nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="../overview/nutshell.html">PETSc in a nutshell</a></li>
<li class="toctree-l1"><a class="reference internal" href="../overview/features.html">Supported Systems</a></li>
<li class="toctree-l1"><a class="reference internal" href="../overview/gpu_roadmap.html">GPU Support Roadmap</a></li>
<li class="toctree-l1"><a class="reference internal" href="../overview/vector_table.html">Summary of Vector Types Available In PETSc</a></li>
<li class="toctree-l1"><a class="reference internal" href="../overview/matrix_table.html">Summary of Matrix Types Available In PETSc</a></li>
<li class="toctree-l1"><a class="reference internal" href="../overview/linear_solve_table.html">Summary of Sparse Linear Solvers Available In PETSc</a></li>
<li class="toctree-l1"><a class="reference internal" href="../overview/nonlinear_solve_table.html">Summary of Nonlinear Solvers Available In PETSc</a></li>
<li class="toctree-l1"><a class="reference internal" href="../overview/integrator_table.html">Summary of Time Integrators Available In PETSc</a></li>
<li class="toctree-l1"><a class="reference internal" href="../overview/tao_solve_table.html">Summary of Tao Solvers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../overview/discrete_table.html">Summary of Discretization Management Systems</a></li>
<li class="toctree-l1"><a class="reference internal" href="../overview/plex_transform_table.html">Summary of Unstructured Mesh Transformations</a></li>
<li class="toctree-l1 current active has-children"><a class="reference internal" href="index.html">User-Guide</a><input checked="" class="toctree-checkbox" id="toctree-checkbox-1" name="toctree-checkbox-1" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-1"><i class="fa-solid fa-chevron-down"></i></label><ul class="current">
<li class="toctree-l2 has-children"><a class="reference internal" href="introduction.html">Introduction to PETSc</a><input class="toctree-checkbox" id="toctree-checkbox-2" name="toctree-checkbox-2" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-2"><i class="fa-solid fa-chevron-down"></i></label><ul>
<li class="toctree-l3"><a class="reference internal" href="about_this_manual.html">About This Manual</a></li>
<li class="toctree-l3"><a class="reference internal" href="getting_started.html">Getting Started</a></li>
</ul>
</li>
<li class="toctree-l2 has-children"><a class="reference internal" href="programming.html">The Solvers in PETSc/TAO</a><input class="toctree-checkbox" id="toctree-checkbox-3" name="toctree-checkbox-3" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-3"><i class="fa-solid fa-chevron-down"></i></label><ul>
<li class="toctree-l3"><a class="reference internal" href="vec.html">Vectors and Parallel Data</a></li>
<li class="toctree-l3"><a class="reference internal" href="mat.html">Matrices</a></li>
<li class="toctree-l3"><a class="reference internal" href="ksp.html">KSP: Linear System Solvers</a></li>
<li class="toctree-l3"><a class="reference internal" href="snes.html">SNES: Nonlinear Solvers</a></li>
<li class="toctree-l3"><a class="reference internal" href="ts.html">TS: Scalable ODE and DAE Solvers</a></li>
<li class="toctree-l3"><a class="reference internal" href="tao.html">TAO: Optimization Solvers</a></li>
</ul>
</li>
<li class="toctree-l2 has-children"><a class="reference internal" href="dm.html">DM: Interfacing Between Solvers and Models/Discretizations</a><input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-4"><i class="fa-solid fa-chevron-down"></i></label><ul>
<li class="toctree-l3"><a class="reference internal" href="dmbase.html">DM Basics</a></li>
<li class="toctree-l3"><a class="reference internal" href="section.html">PetscSection: Connecting Grids to Data</a></li>
<li class="toctree-l3"><a class="reference internal" href="dmplex.html">DMPlex: Unstructured Grids</a></li>
<li class="toctree-l3"><a class="reference internal" href="dmstag.html">DMSTAG: Staggered, Structured Grid</a></li>
<li class="toctree-l3"><a class="reference internal" href="dmnetwork.html">Networks</a></li>
<li class="toctree-l3"><a class="reference internal" href="dt.html">PetscDT: Discretization Technology in PETSc</a></li>
<li class="toctree-l3"><a class="reference internal" href="fe.html">PetscFE: Finite Element Infrastructure in PETSc</a></li>
</ul>
</li>
<li class="toctree-l2 current active has-children"><a class="reference internal" href="additional.html">Additional Information</a><input checked="" class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-5"><i class="fa-solid fa-chevron-down"></i></label><ul class="current">
<li class="toctree-l3"><a class="reference internal" href="fortran.html">PETSc for Fortran Users</a></li>
<li class="toctree-l3"><a class="reference internal" href="versionchecking.html">Checking the PETSc version</a></li>
<li class="toctree-l3"><a class="reference internal" href="matlab.html">Using MATLAB with PETSc</a></li>
<li class="toctree-l3"><a class="reference internal" href="profiling.html">Profiling</a></li>
<li class="toctree-l3 current active"><a class="current reference internal" href="#">Hints for Performance Tuning</a></li>
<li class="toctree-l3"><a class="reference internal" href="streams.html">STREAMS: Example Study</a></li>
<li class="toctree-l3"><a class="reference internal" href="blas-lapack.html">The Use of BLAS and LAPACK in PETSc and external libraries</a></li>
<li class="toctree-l3"><a class="reference internal" href="other.html">Other PETSc Features</a></li>
<li class="toctree-l3"><a class="reference internal" href="advanced.html">Advanced Features of Matrices and Solvers</a></li>
<li class="toctree-l3"><a class="reference internal" href="tests.html">Running PETSc Tests</a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l1 has-children"><a class="reference internal" href="../manualpages/index.html">C/Fortran API</a><input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-6"><i class="fa-solid fa-chevron-down"></i></label><ul>
<li class="toctree-l2 has-children"><a class="reference internal" href="../manualpages/Vector.html">Vectors and Index Sets</a><input class="toctree-checkbox" id="toctree-checkbox-7" name="toctree-checkbox-7" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-7"><i class="fa-solid fa-chevron-down"></i></label><ul>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/Vec/index.html">Vector Operations (Vec)</a></li>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/IS/index.html">Index sets (IS)</a></li>
</ul>
</li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../manualpages/Matrix.html">Matrices and Matrix Operations</a><input class="toctree-checkbox" id="toctree-checkbox-8" name="toctree-checkbox-8" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-8"><i class="fa-solid fa-chevron-down"></i></label><ul>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/Mat/index.html">Matrix Operations (Mat)</a></li>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/MatGraphOperations/index.html">Matrix colorings (MatColoring), orderings (MatOrdering), partitionings (MatPartitioning), and coarsening (MatCoarsen)</a></li>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/MatFD/index.html">Finite difference computation of Jacobians (MatFD)</a></li>
</ul>
</li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../manualpages/DataLayout.html">Data Layout and Communication</a><input class="toctree-checkbox" id="toctree-checkbox-9" name="toctree-checkbox-9" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-9"><i class="fa-solid fa-chevron-down"></i></label><ul>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/PetscSF/index.html">Star Forest Communication (PetscSF)</a></li>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/PetscSection/index.html">Section Data Layout (PetscSection)</a></li>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/AO/index.html">Application Orderings (AO)</a></li>
</ul>
</li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../manualpages/DataManagement.html">Data Management between Vec and Mat, and Distributed Mesh Data Structures</a><input class="toctree-checkbox" id="toctree-checkbox-10" name="toctree-checkbox-10" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-10"><i class="fa-solid fa-chevron-down"></i></label><ul>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/DM/index.html">Data Management (DM)</a></li>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/DMDA/index.html">Structured Grids (DMDA)</a></li>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/DMStag/index.html">Staggered, Structured Grids (DMSTAG)</a></li>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/DMPlex/index.html">Unstructured Grids and Cell Complexes (DMPLEX)</a></li>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/DMNetwork/index.html">Graphs and Networks (DMNETWORK)</a></li>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/DMForest/index.html">A Forest of Trees and Structured Adaptive Refinement (DMFOREST)</a></li>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/DMPatch/index.html">Sequences of parallel mesh patches (DMPATCH)</a></li>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/DMSwarm/index.html">Particle Discretizations (DMSWARM)</a></li>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/DMMOAB/index.html">MOAB Mesh Representation (DMMOAB)</a></li>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/DMLabel/index.html">Selecting Parts of Meshes (DMLabel)</a></li>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/DMPRODUCT/index.html">Tensor products of meshes (DMRODUCT)</a></li>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/DMComposite/index.html">DMComposite</a></li>
</ul>
</li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../manualpages/Discretization.html">Discretization and Function Spaces</a><input class="toctree-checkbox" id="toctree-checkbox-11" name="toctree-checkbox-11" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-11"><i class="fa-solid fa-chevron-down"></i></label><ul>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/DT/index.html">Discretization Technology and Quadrature (DT)</a></li>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/SPACE/index.html">Function Spaces (PetscSpace)</a></li>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/DUALSPACE/index.html">Dual Spaces (PetscDualSpace)</a></li>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/FE/index.html">Finite Elements (PetscFE)</a></li>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/FV/index.html">Finite Volumes (PetscFV)</a></li>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/PF/index.html">Defining your own mathematical functions (PF)</a></li>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/LANDAU/index.html">Landau Collision Operator</a></li>
</ul>
</li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../manualpages/LinearSolvers.html">Linear Solvers and Preconditioners</a><input class="toctree-checkbox" id="toctree-checkbox-12" name="toctree-checkbox-12" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-12"><i class="fa-solid fa-chevron-down"></i></label><ul>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/KSP/index.html">Linear Solvers and Krylov Methods (KSP)</a></li>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/PC/index.html">Preconditioners (PC)</a></li>
</ul>
</li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../manualpages/NonlinearSolvers.html">Nonlinear Solvers</a><input class="toctree-checkbox" id="toctree-checkbox-13" name="toctree-checkbox-13" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-13"><i class="fa-solid fa-chevron-down"></i></label><ul>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/SNES/index.html">Nonlinear Solvers (SNES)</a></li>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/SNESFAS/index.html">Full Approximation Scheme (FAS) nonlinear multigrid</a></li>
</ul>
</li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../manualpages/Timestepping.html">Forward and Adjoint Timestepping</a><input class="toctree-checkbox" id="toctree-checkbox-14" name="toctree-checkbox-14" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-14"><i class="fa-solid fa-chevron-down"></i></label><ul>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/TS/index.html">Time Stepping ODE and DAE Solvers (TS)</a></li>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/Sensitivity/index.html">Sensitivity Analysis for ODE and DAE</a></li>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/Characteristic/index.html">Semi-Lagrangian Solves using the Method of Characteristics</a></li>
</ul>
</li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../manualpages/Optimization.html">Optimization</a><input class="toctree-checkbox" id="toctree-checkbox-15" name="toctree-checkbox-15" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-15"><i class="fa-solid fa-chevron-down"></i></label><ul>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/Tao/index.html">Optimization Solvers (Tao)</a></li>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/TaoLineSearch/index.html">Optimization Line Search (TaoLineSearch)</a></li>
</ul>
</li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../manualpages/Visualization.html">Graphics and Visualization</a><input class="toctree-checkbox" id="toctree-checkbox-16" name="toctree-checkbox-16" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-16"><i class="fa-solid fa-chevron-down"></i></label><ul>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/Draw/index.html">Graphics (Draw)</a></li>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/Viewer/index.html">Viewing Objects (Viewer)</a></li>
</ul>
</li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../manualpages/System.html">System Routines, Profiling, Data Structures</a><input class="toctree-checkbox" id="toctree-checkbox-17" name="toctree-checkbox-17" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-17"><i class="fa-solid fa-chevron-down"></i></label><ul>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/Sys/index.html">PETSc Options, IO, and System Utilities</a></li>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/PetscH/index.html">Hash Tables</a></li>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/Log/index.html">Profiling and Logging</a></li>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/Device/index.html">Device</a></li>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/Matlab/index.html">Matlab</a></li>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/Bag/index.html">Bag</a></li>
<li class="toctree-l3"><a class="reference internal" href="../manualpages/BM/index.html">Benchmarks (BM)</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../changes/index.html">Changes for each release</a></li>
<li class="toctree-l2"><a class="reference internal" href="../manualpages/singleindex.html">Single Index of all PETSc Manual Pages</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../changes/index.html">Changes for each release</a></li>
<li class="toctree-l1"><a class="reference internal" href="../manualpages/singleindex.html">Single Index of all PETSc Manual Pages</a></li>
<li class="toctree-l1 has-children"><a class="reference internal" href="../overview/previous_release_docs.html">Documentation for previous PETSc releases</a><input class="toctree-checkbox" id="toctree-checkbox-18" name="toctree-checkbox-18" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-18"><i class="fa-solid fa-chevron-down"></i></label><ul>
<li class="toctree-l2"><a class="reference external" href="https://web.cels.anl.gov/projects/petsc/vault/petsc-3.22/docs"> 3.22</a></li>
<li class="toctree-l2"><a class="reference external" href="https://web.cels.anl.gov/projects/petsc/vault/petsc-3.21/docs"> 3.21</a></li>
<li class="toctree-l2"><a class="reference external" href="https://web.cels.anl.gov/projects/petsc/vault/petsc-3.20/docs"> 3.20</a></li>
<li class="toctree-l2"><a class="reference external" href="https://web.cels.anl.gov/projects/petsc/vault/petsc-3.19/docs"> 3.19</a></li>
<li class="toctree-l2"><a class="reference external" href="https://web.cels.anl.gov/projects/petsc/vault/petsc-3.18/docs"> 3.18</a></li>
<li class="toctree-l2"><a class="reference external" href="https://web.cels.anl.gov/projects/petsc/vault/petsc-3.17/docs"> 3.17</a></li>
<li class="toctree-l2"><a class="reference external" href="https://web.cels.anl.gov/projects/petsc/vault/petsc-3.16/docs"> 3.16</a></li>
<li class="toctree-l2"><a class="reference external" href="https://web.cels.anl.gov/projects/petsc/vault/petsc-3.15/docs"> 3.15</a></li>
<li class="toctree-l2"><a class="reference external" href="https://web.cels.anl.gov/projects/petsc/vault/petsc-3.14/docs"> 3.14</a></li>
<li class="toctree-l2"><a class="reference external" href="https://web.cels.anl.gov/projects/petsc/vault/petsc-3.13/docs"> 3.13</a></li>
<li class="toctree-l2"><a class="reference external" href="https://web.cels.anl.gov/projects/petsc/vault/petsc-3.12/docs"> 3.12</a></li>
<li class="toctree-l2"><a class="reference external" href="https://web.cels.anl.gov/projects/petsc/vault/petsc-3.11/docs"> 3.11</a></li>
<li class="toctree-l2"><a class="reference external" href="https://web.cels.anl.gov/projects/petsc/vault/petsc-3.10/docs"> 3.10</a></li>
<li class="toctree-l2"><a class="reference external" href="https://web.cels.anl.gov/projects/petsc/vault/petsc-3.9/docs"> 3.9</a></li>
<li class="toctree-l2"><a class="reference external" href="https://web.cels.anl.gov/projects/petsc/vault/petsc-3.8/docs"> 3.8</a></li>
<li class="toctree-l2"><a class="reference external" href="https://web.cels.anl.gov/projects/petsc/vault/petsc-3.7/docs"> 3.7</a></li>
<li class="toctree-l2"><a class="reference external" href="https://web.cels.anl.gov/projects/petsc/vault/petsc-3.6/docs"> 3.6</a></li>
<li class="toctree-l2"><a class="reference external" href="https://web.cels.anl.gov/projects/petsc/vault/petsc-3.5/docs"> 3.5</a></li>
<li class="toctree-l2"><a class="reference external" href="https://web.cels.anl.gov/projects/petsc/vault/petsc-3.4/docs"> 3.4</a></li>
<li class="toctree-l2"><a class="reference external" href="https://web.cels.anl.gov/projects/petsc/vault/petsc-3.3/docs"> 3.3</a></li>
</ul>
</li>
</ul>
</div>
</nav></div>
</div>
<div class="sidebar-primary-items__end sidebar-primary__section">
</div>
<div id="rtd-footer-container"></div>
</div>
<main id="main-content" class="bd-main">
<div class="bd-content">
<div class="bd-article-container">
<div class="bd-header-article">
<div class="header-article-items header-article__inner">
<div class="header-article-items__start">
<div class="header-article-item">
<nav aria-label="Breadcrumb">
<ul class="bd-breadcrumbs">
<li class="breadcrumb-item breadcrumb-home">
<a href="../index.html" class="nav-link" aria-label="Home">
<i class="fa-solid fa-home"></i>
</a>
</li>
<li class="breadcrumb-item"><a href="../overview/index.html" class="nav-link">Overview</a></li>
<li class="breadcrumb-item"><i class="fa-solid fa-ellipsis"></i></li>
<li class="breadcrumb-item"><a href="additional.html" class="nav-link">Additional Information</a></li>
<li class="breadcrumb-item active" aria-current="page">Hints for...</li>
</ul>
</nav>
</div>
</div>
</div>
</div>
<div id="searchbox"></div>
<article class="bd-article">
<section class="tex2jax_ignore mathjax_ignore" id="hints-for-performance-tuning">
<span id="ch-performance"></span><h1>Hints for Performance Tuning<a class="headerlink" href="#hints-for-performance-tuning" title="Link to this heading">#</a></h1>
<p>This chapter provides hints on how to get to achieve best performance
with PETSc, particularly on distributed-memory machines with multiple
CPU sockets per node. We focus on machine-related performance
optimization here; algorithmic aspects like preconditioner selection are
not the focus of this section.</p>
<section id="maximizing-memory-bandwidth">
<h2>Maximizing Memory Bandwidth<a class="headerlink" href="#maximizing-memory-bandwidth" title="Link to this heading">#</a></h2>
<p>Most operations in PETSc deal with large datasets (typically vectors and
sparse matrices) and perform relatively few arithmetic operations for
each byte loaded or stored from global memory. Therefore, the
<em>arithmetic intensity</em> expressed as the ratio of floating point
operations to the number of bytes loaded and stored is usually well
below unity for typical PETSc operations. On the other hand, modern CPUs
are able to execute on the order of 10 floating point operations for
each byte loaded or stored. As a consequence, almost all PETSc
operations are limited by the rate at which data can be loaded or stored
(<em>memory bandwidth limited</em>) rather than by the rate of floating point
operations.</p>
<p>This section discusses ways to maximize the memory bandwidth achieved by
applications based on PETSc. Where appropriate, we include benchmark
results in order to provide quantitative results on typical performance
gains one can achieve through parallelization, both on a single compute
node and across nodes. In particular, we start with the answer to the
common question of why performance generally does not increase 20-fold
with a 20-core CPU.</p>
<section id="memory-bandwidth-vs-processes">
<span id="subsec-bandwidth-vs-processes"></span><h3>Memory Bandwidth vs. Processes<a class="headerlink" href="#memory-bandwidth-vs-processes" title="Link to this heading">#</a></h3>
<p>Consider the addition of two large vectors, with the result written to a
third vector. Because there are no dependencies across the different
entries of each vector, the operation is embarrassingly parallel.</p>
<figure class="align-default" id="fig-stream-intel">
<a class="reference internal image-reference" href="../_images/stream-results-intel.png"><img alt="Memory bandwidth obtained on Intel hardware (dual socket except KNL) over the number of processes used. One can get close to peak memory bandwidth with only a few processes." src="../_images/stream-results-intel.png" style="width: 80.0%;" />
</a>
<figcaption>
<p><span class="caption-number">Fig. 14 </span><span class="caption-text">Memory bandwidth obtained on Intel hardware (dual socket except KNL)
over the number of processes used. One can get close to peak memory
bandwidth with only a few processes.</span><a class="headerlink" href="#fig-stream-intel" title="Link to this image">#</a></p>
</figcaption>
</figure>
<p>As <a class="reference internal" href="#fig-stream-intel"><span class="std std-numref">Fig. 14</span></a> shows, the performance gains due to
parallelization on different multi- and many-core CPUs quickly
saturates. The reason is that only a fraction of the total number of CPU
cores is required to saturate the memory channels. For example, a
dual-socket system equipped with Haswell 12-core Xeon CPUs achieves more
than 80 percent of achievable peak memory bandwidth with only four
processes per socket (8 total), cf. <a class="reference internal" href="#fig-stream-intel"><span class="std std-numref">Fig. 14</span></a>.
Consequently, running with more than 8 MPI ranks on such a system will
not increase performance substantially. For the same reason, PETSc-based
applications usually do not benefit from hyper-threading.</p>
<p>PETSc provides a simple way to measure memory bandwidth for different
numbers of processes via the target <code class="docutils notranslate"><span class="pre">make</span> <span class="pre">streams</span></code> executed from
<code class="docutils notranslate"><span class="pre">$PETSC_DIR</span></code>. The output provides an overview of the possible speedup
one can obtain on the given machine (not necessarily a shared memory
system). For example, the following is the most relevant output obtained
on a dual-socket system equipped with two six-core-CPUs with
hyperthreading:</p>
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>np speedup
1 1.0
2 1.58
3 2.19
4 2.42
5 2.63
6 2.69
...
21 3.82
22 3.49
23 3.79
24 3.71
Estimation of possible speedup of MPI programs based on Streams benchmark.
It appears you have 1 node(s)
</pre></div>
</div>
<p>On this machine, one should expect a speed-up of typical memory
bandwidth-bound PETSc applications of at most 4x when running multiple
MPI ranks on the node. Most of the gains are already obtained when
running with only 4-6 ranks. Because a smaller number of MPI ranks
usually implies better preconditioners and better performance for
smaller problems, the best performance for PETSc applications may be
obtained with fewer ranks than there are physical CPU cores available.</p>
<p>Following the results from the above run of <code class="docutils notranslate"><span class="pre">make</span> <span class="pre">streams</span></code>, we
recommend to use additional nodes instead of placing additional MPI
ranks on the nodes. In particular, weak scaling (i.e. constant load per
process, increasing the number of processes) and strong scaling
(i.e. constant total work, increasing the number of processes) studies
should keep the number of processes per node constant.</p>
</section>
<section id="non-uniform-memory-access-numa-and-process-placement">
<h3>Non-Uniform Memory Access (NUMA) and Process Placement<a class="headerlink" href="#non-uniform-memory-access-numa-and-process-placement" title="Link to this heading">#</a></h3>
<p>CPUs in nodes with more than one CPU socket are internally connected via
a high-speed fabric, cf. <a class="reference internal" href="#fig-numa"><span class="std std-numref">Fig. 15</span></a>, to enable data
exchange as well as cache coherency. Because main memory on modern
systems is connected via the integrated memory controllers on each CPU,
memory is accessed in a non-uniform way: A process running on one socket
has direct access to the memory channels of the respective CPU, whereas
requests for memory attached to a different CPU socket need to go
through the high-speed fabric. Consequently, best aggregate memory
bandwidth on the node is obtained when the memory controllers on each
CPU are fully saturated. However, full saturation of memory channels is
only possible if the data is distributed across the different memory
channels.</p>
<figure class="align-default" id="fig-numa">
<a class="reference internal image-reference" href="../_images/numa.png"><img alt="Schematic of a two-socket NUMA system. Processes should be spread across both CPUs to obtain full bandwidth." src="../_images/numa.png" style="width: 90.0%;" />
</a>
<figcaption>
<p><span class="caption-number">Fig. 15 </span><span class="caption-text">Schematic of a two-socket NUMA system. Processes should be spread
across both CPUs to obtain full bandwidth.</span><a class="headerlink" href="#fig-numa" title="Link to this image">#</a></p>
</figcaption>
</figure>
<p>Data in memory on modern machines is allocated by the operating system
based on a first-touch policy. That is, memory is not allocated at the
point of issuing <code class="docutils notranslate"><span class="pre">malloc()</span></code>, but at the point when the respective
memory segment is actually touched (read or write). Upon first-touch,
memory is allocated on the memory channel associated with the respective
CPU the process is running on. Only if all memory on the respective CPU
is already in use (either allocated or as IO cache), memory available
through other sockets is considered.</p>
<p>Maximum memory bandwidth can be achieved by ensuring that processes are
spread over all sockets in the respective node. For example, the
recommended placement of a 8-way parallel run on a four-socket machine
is to assign two processes to each CPU socket. To do so, one needs to
know the enumeration of cores and pass the requested information to
<code class="docutils notranslate"><span class="pre">mpirun</span></code>. Consider the hardware topology information returned by
<code class="docutils notranslate"><span class="pre">lstopo</span></code> (part of the hwloc package) for the following two-socket
machine, in which each CPU consists of six cores and supports
hyperthreading:</p>
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>Machine (126GB total)
NUMANode L#0 (P#0 63GB)
Package L#0 + L3 L#0 (15MB)
L2 L#0 (256KB) + L1d L#0 (32KB) + L1i L#0 (32KB) + Core L#0
PU L#0 (P#0)
PU L#1 (P#12)
L2 L#1 (256KB) + L1d L#1 (32KB) + L1i L#1 (32KB) + Core L#1
PU L#2 (P#1)
PU L#3 (P#13)
L2 L#2 (256KB) + L1d L#2 (32KB) + L1i L#2 (32KB) + Core L#2
PU L#4 (P#2)
PU L#5 (P#14)
L2 L#3 (256KB) + L1d L#3 (32KB) + L1i L#3 (32KB) + Core L#3
PU L#6 (P#3)
PU L#7 (P#15)
L2 L#4 (256KB) + L1d L#4 (32KB) + L1i L#4 (32KB) + Core L#4
PU L#8 (P#4)
PU L#9 (P#16)
L2 L#5 (256KB) + L1d L#5 (32KB) + L1i L#5 (32KB) + Core L#5
PU L#10 (P#5)
PU L#11 (P#17)
NUMANode L#1 (P#1 63GB)
Package L#1 + L3 L#1 (15MB)
L2 L#6 (256KB) + L1d L#6 (32KB) + L1i L#6 (32KB) + Core L#6
PU L#12 (P#6)
PU L#13 (P#18)
L2 L#7 (256KB) + L1d L#7 (32KB) + L1i L#7 (32KB) + Core L#7
PU L#14 (P#7)
PU L#15 (P#19)
L2 L#8 (256KB) + L1d L#8 (32KB) + L1i L#8 (32KB) + Core L#8
PU L#16 (P#8)
PU L#17 (P#20)
L2 L#9 (256KB) + L1d L#9 (32KB) + L1i L#9 (32KB) + Core L#9
PU L#18 (P#9)
PU L#19 (P#21)
L2 L#10 (256KB) + L1d L#10 (32KB) + L1i L#10 (32KB) + Core L#10
PU L#20 (P#10)
PU L#21 (P#22)
L2 L#11 (256KB) + L1d L#11 (32KB) + L1i L#11 (32KB) + Core L#11
PU L#22 (P#11)
PU L#23 (P#23)
</pre></div>
</div>
<p>The relevant physical processor IDs are shown in parentheses prefixed by
<code class="docutils notranslate"><span class="pre">P#</span></code>. Here, IDs 0 and 12 share the same physical core and have a
common L2 cache. IDs 0, 12, 1, 13, 2, 14, 3, 15, 4, 16, 5, 17 share the
same socket and have a common L3 cache.</p>
<p>A good placement for a run with six processes is to locate three
processes on the first socket and three processes on the second socket.
Unfortunately, mechanisms for process placement vary across MPI
implementations, so make sure to consult the manual of your MPI
implementation. The following discussion is based on how processor
placement is done with MPICH and Open MPI, where one needs to pass
<code class="docutils notranslate"><span class="pre">--bind-to</span> <span class="pre">core</span> <span class="pre">--map-by</span> <span class="pre">socket</span></code> to <code class="docutils notranslate"><span class="pre">mpirun</span></code>:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>mpirun<span class="w"> </span>-n<span class="w"> </span><span class="m">6</span><span class="w"> </span>--bind-to<span class="w"> </span>core<span class="w"> </span>--map-by<span class="w"> </span>socket<span class="w"> </span>./stream
<span class="go">process 0 binding: 100000000000100000000000</span>
<span class="go">process 1 binding: 000000100000000000100000</span>
<span class="go">process 2 binding: 010000000000010000000000</span>
<span class="go">process 3 binding: 000000010000000000010000</span>
<span class="go">process 4 binding: 001000000000001000000000</span>
<span class="go">process 5 binding: 000000001000000000001000</span>
<span class="go">Triad: 45403.1949 Rate (MB/s)</span>
</pre></div>
</div>
<p>In this configuration, process 0 is bound to the first physical core on
the first socket (with IDs 0 and 12), process 1 is bound to the first
core on the second socket (IDs 6 and 18), and similarly for the
remaining processes. The achieved bandwidth of 45 GB/sec is close to the
practical peak of about 50 GB/sec available on the machine. If, however,
all MPI processes are located on the same socket, memory bandwidth drops
significantly:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>mpirun<span class="w"> </span>-n<span class="w"> </span><span class="m">6</span><span class="w"> </span>--bind-to<span class="w"> </span>core<span class="w"> </span>--map-by<span class="w"> </span>core<span class="w"> </span>./stream
<span class="go">process 0 binding: 100000000000100000000000</span>
<span class="go">process 1 binding: 010000000000010000000000</span>
<span class="go">process 2 binding: 001000000000001000000000</span>
<span class="go">process 3 binding: 000100000000000100000000</span>
<span class="go">process 4 binding: 000010000000000010000000</span>
<span class="go">process 5 binding: 000001000000000001000000</span>
<span class="go">Triad: 25510.7507 Rate (MB/s)</span>
</pre></div>
</div>
<p>All processes are now mapped to cores on the same socket. As a result,
only the first memory channel is fully saturated at 25.5 GB/sec.</p>
<p>One must not assume that <code class="docutils notranslate"><span class="pre">mpirun</span></code> uses good defaults. To
demonstrate, compare the full output of <code class="docutils notranslate"><span class="pre">make</span> <span class="pre">streams</span></code> from <a class="reference internal" href="#subsec-bandwidth-vs-processes"><span class="std std-ref">Memory Bandwidth vs. Processes</span></a> first, followed by
the results obtained by passing <code class="docutils notranslate"><span class="pre">--bind-to</span> <span class="pre">core</span> <span class="pre">--map-by</span> <span class="pre">socket</span></code>:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>make<span class="w"> </span>streams
<span class="go">np speedup</span>
<span class="go">1 1.0</span>
<span class="go">2 1.58</span>
<span class="go">3 2.19</span>
<span class="go">4 2.42</span>
<span class="go">5 2.63</span>
<span class="go">6 2.69</span>
<span class="go">7 2.31</span>
<span class="go">8 2.42</span>
<span class="go">9 2.37</span>
<span class="go">10 2.65</span>
<span class="go">11 2.3</span>
<span class="go">12 2.53</span>
<span class="go">13 2.43</span>
<span class="go">14 2.63</span>
<span class="go">15 2.74</span>
<span class="go">16 2.7</span>
<span class="go">17 3.28</span>
<span class="go">18 3.66</span>
<span class="go">19 3.95</span>
<span class="go">20 3.07</span>
<span class="go">21 3.82</span>
<span class="go">22 3.49</span>
<span class="go">23 3.79</span>
<span class="go">24 3.71</span>
</pre></div>
</div>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>make<span class="w"> </span>streams<span class="w"> </span><span class="nv">MPI_BINDING</span><span class="o">=</span><span class="s2">"--bind-to core --map-by socket"</span>
<span class="go">np speedup</span>
<span class="go">1 1.0</span>
<span class="go">2 1.59</span>
<span class="go">3 2.66</span>
<span class="go">4 3.5</span>
<span class="go">5 3.56</span>
<span class="go">6 4.23</span>
<span class="go">7 3.95</span>
<span class="go">8 4.39</span>
<span class="go">9 4.09</span>
<span class="go">10 4.46</span>
<span class="go">11 4.15</span>
<span class="go">12 4.42</span>
<span class="go">13 3.71</span>
<span class="go">14 3.83</span>
<span class="go">15 4.08</span>
<span class="go">16 4.22</span>
<span class="go">17 4.18</span>
<span class="go">18 4.31</span>
<span class="go">19 4.22</span>
<span class="go">20 4.28</span>
<span class="go">21 4.25</span>
<span class="go">22 4.23</span>
<span class="go">23 4.28</span>
<span class="go">24 4.22</span>
</pre></div>
</div>
<p>For the non-optimized version on the left, the speedup obtained when
using any number of processes between 3 and 13 is essentially constant
up to fluctuations, indicating that all processes were by default
executed on the same socket. Only with 14 or more processes, the
speedup number increases again. In contrast, the results of</p>
<p><code class="docutils notranslate"><span class="pre">make</span> <span class="pre">streams</span></code></p>
<p>with proper processor placement shown second
resulted in slightly higher overall parallel speedup (identical
baselines), in smaller performance fluctuations, and more than 90
percent of peak bandwidth with only six processes.</p>
<p>Machines with job submission systems such as SLURM usually provide
similar mechanisms for processor placements through options specified in
job submission scripts. Please consult the respective manuals.</p>
<section id="additional-process-placement-considerations-and-details">
<h4>Additional Process Placement Considerations and Details<a class="headerlink" href="#additional-process-placement-considerations-and-details" title="Link to this heading">#</a></h4>
<p>For a typical, memory bandwidth-limited PETSc application, the primary
consideration in placing MPI processes is ensuring that processes are
evenly distributed among sockets, and hence using all available memory
channels. Increasingly complex processor designs and cache hierarchies,
however, mean that performance may also be sensitive to how processes
are bound to the resources within each socket. Performance on the two
processor machine in the preceding example may be relatively insensitive
to such placement decisions, because one L3 cache is shared by all cores
within a NUMA domain, and each core has its own L2 and L1 caches.
However, processors that are less “flat”, with more complex hierarchies,
may be more sensitive. In many AMD Opterons or the second-generation
“Knights Landing” Intel Xeon Phi, for instance, L2 caches are shared
between two cores. On these processors, placing consecutive MPI ranks on
cores that share the same L2 cache may benefit performance if the two
ranks communicate frequently with each other, because the latency
between cores sharing an L2 cache may be roughly half that of two cores
not sharing one. There may be benefit, however, in placing consecutive
ranks on cores that do not share an L2 cache, because (if there are
fewer MPI ranks than cores) this increases the total L2 cache capacity
and bandwidth available to the application. There is a trade-off to be
considered between placing processes close together (in terms of shared
resources) to optimize for efficient communication and synchronization
vs. farther apart to maximize available resources (memory channels,
caches, I/O channels, etc.), and the best strategy will depend on the
application and the software and hardware stack.</p>
<p>Different process placement strategies can affect performance at least
as much as some commonly explored settings, such as compiler
optimization levels. Unfortunately, exploration of this space is
complicated by two factors: First, processor and core numberings may be
completely arbitrary, changing with BIOS version, etc., and second—as
already noted—there is no standard mechanism used by MPI implementations
(or job schedulers) to specify process affinity. To overcome the first
issue, we recommend using the <code class="docutils notranslate"><span class="pre">lstopo</span></code> utility of the Portable
Hardware Locality (<code class="docutils notranslate"><span class="pre">hwloc</span></code>) software package (which can be installed
by configuring PETSc with <code class="docutils notranslate"><span class="pre">–download-hwloc</span></code>) to understand the
processor topology of your machine. We cannot fully address the second
issue—consult the documentation for your MPI implementation and/or job
scheduler—but we offer some general observations on understanding
placement options:</p>
<ul class="simple">
<li><p>An MPI implementation may support a notion of <em>domains</em> in which a
process may be pinned. A domain may simply correspond to a single
core; however, the MPI implementation may allow a deal of flexibility
in specifying domains that encompass multiple cores, span sockets,
etc. Some implementations, such as Intel MPI, provide means to
specify whether domains should be “compact”—composed of cores sharing
resources such as caches—or “scatter”-ed, with little resource
sharing (possibly even spanning sockets).</p></li>
<li><p>Separate from the specification of domains, MPI implementations often
support different <em>orderings</em> in which MPI ranks should be bound to
these domains. Intel MPI, for instance, supports “compact” ordering
to place consecutive ranks close in terms of shared resources,
“scatter” to place them far apart, and “bunch” to map proportionally
to sockets while placing ranks as close together as possible within
the sockets.</p></li>
<li><p>An MPI implementation that supports process pinning should offer some
way to view the rank assignments. Use this output in conjunction with
the topology obtained via <code class="docutils notranslate"><span class="pre">lstopo</span></code> or a similar tool to determine
if the placements correspond to something you believe is reasonable
for your application. Do not assume that the MPI implementation is
doing something sensible by default!</p></li>
</ul>
</section>
</section>
</section>
<section id="performance-pitfalls-and-advice">
<h2>Performance Pitfalls and Advice<a class="headerlink" href="#performance-pitfalls-and-advice" title="Link to this heading">#</a></h2>
<p>This section looks into a potpourri of performance pitfalls encountered
by users in the past. Many of these pitfalls require a deeper
understanding of the system and experience to detect. The purpose of
this section is to summarize and share our experience so that these
pitfalls can be avoided in the future.</p>
<section id="debug-vs-optimized-builds">
<h3>Debug vs. Optimized Builds<a class="headerlink" href="#debug-vs-optimized-builds" title="Link to this heading">#</a></h3>
<p>PETSc’s <code class="docutils notranslate"><span class="pre">configure</span></code> defaults to building PETSc with debug mode
enabled. Any code development should be done in this mode, because it
provides handy debugging facilities such as accurate stack traces,
memory leak checks, and memory corruption checks. Note that PETSc has no
reliable way of knowing whether a particular run is a production or
debug run. In the case that a user requests profiling information via
<code class="docutils notranslate"><span class="pre">-log_view</span></code>, a debug build of PETSc issues the following warning:</p>
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>##########################################################
# #
# WARNING!!! #
# #
# This code was compiled with a debugging option, #
# To get timing results run configure #
# using --with-debugging=no, the performance will #
# be generally two or three times faster. #
# #
##########################################################
</pre></div>
</div>
<p>Conversely, one way of checking whether a particular build of PETSc has
debugging enabled is to inspect the output of <code class="docutils notranslate"><span class="pre">-log_view</span></code>.</p>
<p>Debug mode will generally be most useful for code development if
appropriate compiler options are set to facilitate debugging. The
compiler should be instructed to generate binaries with debug symbols
(command line option <code class="docutils notranslate"><span class="pre">-g</span></code> for most compilers), and the optimization
level chosen should either completely disable optimizations (<code class="docutils notranslate"><span class="pre">-O0</span></code> for
most compilers) or enable only optimizations that do not interfere with
debugging (GCC, for instance, supports a <code class="docutils notranslate"><span class="pre">-Og</span></code> optimization level that
does this).</p>
<p>Only once the new code is thoroughly tested and ready for production,
one should disable debugging facilities by passing
<code class="docutils notranslate"><span class="pre">--with-debugging=no</span></code> to</p>
<p><code class="docutils notranslate"><span class="pre">configure</span></code>. One should also ensure that an appropriate compiler
optimization level is set. Note that some compilers (e.g., Intel)
default to fairly comprehensive optimization levels, while others (e.g.,
GCC) default to no optimization at all. The best optimization flags will
depend on your code, the compiler, and the target architecture, but we
offer a few guidelines for finding those that will offer the best
performance:</p>
<ul class="simple">
<li><p>Most compilers have a number of optimization levels (with level n
usually specified via <code class="docutils notranslate"><span class="pre">-On</span></code>) that provide a quick way to enable
sets of several optimization flags. We suggest trying the higher
optimization levels (the highest level is not guaranteed to produce
the fastest executable, so some experimentation may be merited). With
most recent processors now supporting some form of SIMD or vector
instructions, it is important to choose a level that enables the
compiler’s auto-vectorizer; many compilers do not enable
auto-vectorization at lower optimization levels (e.g., GCC does not
enable it below <code class="docutils notranslate"><span class="pre">-O3</span></code> and the Intel compiler does not enable it
below <code class="docutils notranslate"><span class="pre">-O2</span></code>).</p></li>
<li><p>For processors supporting newer vector instruction sets, such as
Intel AVX2 and AVX-512, it is also important to direct the compiler
to generate code that targets these processors (e.g., <code class="docutils notranslate"><span class="pre">-march=native</span></code>);
otherwise, the executables built will not
utilize the newer instructions sets and will not take advantage of
the vector processing units.</p></li>
<li><p>Beyond choosing the optimization levels, some value-unsafe
optimizations (such as using reciprocals of values instead of
dividing by those values, or allowing re-association of operands in a
series of calculations) for floating point calculations may yield
significant performance gains. Compilers often provide flags (e.g.,
<code class="docutils notranslate"><span class="pre">-ffast-math</span></code> in GCC) to enable a set of these optimizations, and
they may be turned on when using options for very aggressive
optimization (<code class="docutils notranslate"><span class="pre">-fast</span></code> or <code class="docutils notranslate"><span class="pre">-Ofast</span></code> in many compilers). These are
worth exploring to maximize performance, but, if employed, it
important to verify that these do not cause erroneous results with
your code, since calculations may violate the IEEE standard for
floating-point arithmetic.</p></li>
</ul>
</section>
<section id="profiling">
<h3>Profiling<a class="headerlink" href="#profiling" title="Link to this heading">#</a></h3>
<p>Users should not spend time optimizing a code until after having
determined where it spends the bulk of its time on realistically sized
problems. As discussed in detail in <a class="reference internal" href="profiling.html#ch-profiling"><span class="std std-ref">Profiling</span></a>, the
PETSc routines automatically log performance data if certain runtime
options are specified.</p>
<p>To obtain a summary of where and how much time is spent in different
sections of the code, use one of the following options:</p>
<ul class="simple">
<li><p>Run the code with the option <code class="docutils notranslate"><span class="pre">-log_view</span></code> to print a performance
summary for various phases of the code.</p></li>
<li><p>Run the code with the option <code class="docutils notranslate"><span class="pre">-log_mpe</span></code> <code class="docutils notranslate"><span class="pre">[logfilename]</span></code>, which
creates a logfile of events suitable for viewing with Jumpshot (part
of MPICH).</p></li>
</ul>
<p>Then, focus on the sections where most of the time is spent. If you
provided your own callback routines, e.g. for residual evaluations,
search the profiling output for routines such as <code class="docutils notranslate"><span class="pre">SNESFunctionEval</span></code> or
<code class="docutils notranslate"><span class="pre">SNESJacobianEval</span></code>. If their relative time is significant (say, more
than 30 percent), consider optimizing these routines first. Generic
instructions on how to optimize your callback functions are difficult;
you may start by reading performance optimization guides for your
system’s hardware.</p>
</section>
<section id="aggregation">
<h3>Aggregation<a class="headerlink" href="#aggregation" title="Link to this heading">#</a></h3>
<p>Performing operations on chunks of data rather than a single element at
a time can significantly enhance performance because of cache reuse or
lower data motion. Typical examples are:</p>
<ul class="simple">
<li><p>Insert several (many) elements of a matrix or vector at once, rather
than looping and inserting a single value at a time. In order to
access elements in of vector repeatedly, employ <code class="docutils notranslate"><span class="pre"><a href="../manualpages/Vec/VecGetArray.html">VecGetArray</a>()</span></code> to
allow direct manipulation of the vector elements.</p></li>
<li><p>When possible, use <code class="docutils notranslate"><span class="pre"><a href="../manualpages/Vec/VecMDot.html">VecMDot</a>()</span></code> rather than a series of calls to
<code class="docutils notranslate"><span class="pre"><a href="../manualpages/Vec/VecDot.html">VecDot</a>()</span></code>.</p></li>
<li><p>If you require a sequence of matrix-vector products with the same
matrix, consider packing your vectors into a single matrix and use
matrix-matrix multiplications.</p></li>
<li><p>Users should employ a reasonable number of <code class="docutils notranslate"><span class="pre"><a href="../manualpages/Sys/PetscMalloc.html">PetscMalloc</a>()</span></code> calls in
their codes. Hundreds or thousands of memory allocations may be
appropriate; however, if tens of thousands are being used, then
reducing the number of <code class="docutils notranslate"><span class="pre"><a href="../manualpages/Sys/PetscMalloc.html">PetscMalloc</a>()</span></code> calls may be warranted. For
example, reusing space or allocating large chunks and dividing it
into pieces can produce a significant savings in allocation overhead.
<a class="reference internal" href="#sec-dsreuse"><span class="std std-ref">Data Structure Reuse</span></a> gives details.</p></li>
</ul>
<p>Aggressive aggregation of data may result in inflexible datastructures
and code that is hard to maintain. We advise users to keep these
competing goals in mind and not blindly optimize for performance only.</p>
</section>
<section id="memory-allocation-for-sparse-matrix-factorization">
<span id="sec-symbolfactor"></span><h3>Memory Allocation for Sparse Matrix Factorization<a class="headerlink" href="#memory-allocation-for-sparse-matrix-factorization" title="Link to this heading">#</a></h3>
<p>When symbolically factoring an AIJ matrix, PETSc has to guess how much
fill there will be. Careful use of the fill parameter in the
<code class="docutils notranslate"><span class="pre"><a href="../manualpages/Mat/MatFactorInfo.html">MatFactorInfo</a></span></code> structure when calling <code class="docutils notranslate"><span class="pre"><a href="../manualpages/Mat/MatLUFactorSymbolic.html">MatLUFactorSymbolic</a>()</span></code> or
<code class="docutils notranslate"><span class="pre"><a href="../manualpages/Mat/MatILUFactorSymbolic.html">MatILUFactorSymbolic</a>()</span></code> can reduce greatly the number of mallocs and
copies required, and thus greatly improve the performance of the
factorization. One way to determine a good value for the fill parameter
is to run a program with the option <code class="docutils notranslate"><span class="pre">-info</span></code>. The symbolic
factorization phase will then print information such as</p>
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>Info:MatILUFactorSymbolic_SeqAIJ:Reallocs 12 Fill ratio:given 1 needed 2.16423
</pre></div>
</div>
<p>This indicates that the user should have used a fill estimate factor of
about 2.17 (instead of 1) to prevent the 12 required mallocs and copies.
The command line option</p>
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>-pc_factor_fill 2.17
</pre></div>
</div>
<p>will cause PETSc to preallocate the correct amount of space for
the factorization.</p>
</section>
<section id="detecting-memory-allocation-problems-and-memory-usage">
<span id="detecting-memory-problems"></span><h3>Detecting Memory Allocation Problems and Memory Usage<a class="headerlink" href="#detecting-memory-allocation-problems-and-memory-usage" title="Link to this heading">#</a></h3>
<p>PETSc provides tools to aid in understanding PETSc memory usage and detecting problems with
memory allocation, including leaks and use of uninitialized space. Internally, PETSc uses
the routines <code class="docutils notranslate"><span class="pre"><a href="../manualpages/Sys/PetscMalloc.html">PetscMalloc</a>()</span></code> and <code class="docutils notranslate"><span class="pre"><a href="../manualpages/Sys/PetscFree.html">PetscFree</a>()</span></code> for memory allocation; instead of directly calling <code class="docutils notranslate"><span class="pre">malloc()</span></code> and <code class="docutils notranslate"><span class="pre">free()</span></code>.
This allows PETSc to track its memory usage and perform error checking. Users are urged to use these routines as well when
appropriate.</p>
<ul class="simple">
<li><p>The option <code class="docutils notranslate"><span class="pre">-malloc_debug</span></code> turns on PETSc’s extensive runtime error checking of memory for corruption.
This checking can be expensive, so should not be used for
production runs. The option <code class="docutils notranslate"><span class="pre">-malloc_test</span></code> is equivalent to <code class="docutils notranslate"><span class="pre">-malloc_debug</span></code>
but only works when PETSc is configured with <code class="docutils notranslate"><span class="pre">--with-debugging</span></code> (the default configuration).
We suggest setting the environmental variable <code class="docutils notranslate"><span class="pre">PETSC_OPTIONS=-malloc_test</span></code>
in your shell startup file to automatically enable runtime check memory for developing code but not
running optimized code. Using <code class="docutils notranslate"><span class="pre">-malloc_debug</span></code> or <code class="docutils notranslate"><span class="pre">-malloc_test</span></code> for large runs can slow them significantly, thus we
recommend turning them off if you code is painfully slow and you don’t need the testing. In addition, you can use
<code class="docutils notranslate"><span class="pre">-check_pointer_intensity</span> <span class="pre">0</span></code> for long run debug runs that do not need extensive memory corruption testing. This option
is occasionally added to the <code class="docutils notranslate"><span class="pre">PETSC_OPTIONS</span></code> environmental variable by some users.</p></li>
<li><p>The option
<code class="docutils notranslate"><span class="pre">-malloc_dump</span></code> will print a list of memory locations that have not been freed at the
conclusion of a program. If all memory has been freed no message
is printed. Note that
the option <code class="docutils notranslate"><span class="pre">-malloc_dump</span></code> activates a call to
<code class="docutils notranslate"><span class="pre"><a href="../manualpages/Sys/PetscMallocDump.html">PetscMallocDump</a>()</span></code> during <code class="docutils notranslate"><span class="pre"><a href="../manualpages/Sys/PetscFinalize.html">PetscFinalize</a>()</span></code>. The user can also
call <code class="docutils notranslate"><span class="pre"><a href="../manualpages/Sys/PetscMallocDump.html">PetscMallocDump</a>()</span></code> elsewhere in a program.</p></li>
<li><p>Another useful option
is <code class="docutils notranslate"><span class="pre">-malloc_view</span></code>, which reports memory usage in all routines at the conclusion of the program.
Note that this option
activates logging by calling <code class="docutils notranslate"><span class="pre"><a href="../manualpages/Sys/PetscMallocViewSet.html">PetscMallocViewSet</a>()</span></code> in
<code class="docutils notranslate"><span class="pre"><a href="../manualpages/Sys/PetscInitialize.html">PetscInitialize</a>()</span></code> and then prints the log by calling
<code class="docutils notranslate"><span class="pre"><a href="../manualpages/Sys/PetscMallocView.html">PetscMallocView</a>()</span></code> in <code class="docutils notranslate"><span class="pre"><a href="../manualpages/Sys/PetscFinalize.html">PetscFinalize</a>()</span></code>. The user can also call
these routines elsewhere in a program.</p></li>
<li><p>When finer granularity is
desired, the user can call <code class="docutils notranslate"><span class="pre"><a href="../manualpages/Sys/PetscMallocGetCurrentUsage.html">PetscMallocGetCurrentUsage</a>()</span></code> and
<code class="docutils notranslate"><span class="pre"><a href="../manualpages/Sys/PetscMallocGetMaximumUsage.html">PetscMallocGetMaximumUsage</a>()</span></code> for memory allocated by PETSc, or
<code class="docutils notranslate"><span class="pre"><a href="../manualpages/Sys/PetscMemoryGetCurrentUsage.html">PetscMemoryGetCurrentUsage</a>()</span></code> and <code class="docutils notranslate"><span class="pre"><a href="../manualpages/Sys/PetscMemoryGetMaximumUsage.html">PetscMemoryGetMaximumUsage</a>()</span></code>
for the total memory used by the program. Note that
<code class="docutils notranslate"><span class="pre"><a href="../manualpages/Sys/PetscMemorySetGetMaximumUsage.html">PetscMemorySetGetMaximumUsage</a>()</span></code> must be called before
<code class="docutils notranslate"><span class="pre"><a href="../manualpages/Sys/PetscMemoryGetMaximumUsage.html">PetscMemoryGetMaximumUsage</a>()</span></code> (typically at the beginning of the
program).</p></li>
<li><p>The option <code class="docutils notranslate"><span class="pre">-memory_view</span></code> provides a high-level view of all memory usage,
not just the memory used by <code class="docutils notranslate"><span class="pre"><a href="../manualpages/Sys/PetscMalloc.html">PetscMalloc</a>()</span></code>, at the conclusion of the program.</p></li>
<li><p>When running with <code class="docutils notranslate"><span class="pre">-log_view</span></code>, the additional option <code class="docutils notranslate"><span class="pre">-log_view_memory</span></code>
causes the display of additional columns of information about how much
memory was allocated and freed during each logged event. This is useful
to understand what phases of a computation require the most memory.</p></li>
</ul>
<p>One can also use <a class="reference external" href="http://valgrind.org">Valgrind</a> to track memory usage and find bugs, see <a class="reference internal" href="../faq/index.html#valgrind"><span class="std std-ref">FAQ: Valgrind usage</span></a>.</p>
</section>
<section id="data-structure-reuse">
<span id="sec-dsreuse"></span><h3>Data Structure Reuse<a class="headerlink" href="#data-structure-reuse" title="Link to this heading">#</a></h3>
<p>Data structures should be reused whenever possible. For example, if a
code often creates new matrices or vectors, there often may be a way to
reuse some of them. Very significant performance improvements can be
achieved by reusing matrix data structures with the same nonzero
pattern. If a code creates thousands of matrix or vector objects,
performance will be degraded. For example, when solving a nonlinear
problem or timestepping, reusing the matrices and their nonzero
structure for many steps when appropriate can make the code run
significantly faster.</p>
<p>A simple technique for saving work vectors, matrices, etc. is employing
a user-defined context. In C and C++ such a context is merely a
structure in which various objects can be stashed; in Fortran a user
context can be an integer array that contains both parameters and
pointers to PETSc objects. See
<a href="../src/snes/tutorials/ex5.c.html">SNES Tutorial ex5</a>
and
<a href="../src/snes/tutorials/ex5f90.F90.html">SNES Tutorial ex5f90</a>
for examples of user-defined application contexts in C and Fortran,
respectively.</p>
</section>
<section id="numerical-experiments">
<h3>Numerical Experiments<a class="headerlink" href="#numerical-experiments" title="Link to this heading">#</a></h3>
<p>PETSc users should run a variety of tests. For example, there are a
large number of options for the linear and nonlinear equation solvers in
PETSc, and different choices can make a <em>very</em> big difference in
convergence rates and execution times. PETSc employs defaults that are
generally reasonable for a wide range of problems, but clearly these
defaults cannot be best for all cases. Users should experiment with many
combinations to determine what is best for a given problem and customize
the solvers accordingly.</p>
<ul class="simple">
<li><p>Use the options <code class="docutils notranslate"><span class="pre">-snes_view</span></code>, <code class="docutils notranslate"><span class="pre">-ksp_view</span></code>, etc. (or the routines
<code class="docutils notranslate"><span class="pre"><a href="../manualpages/KSP/KSPView.html">KSPView</a>()</span></code>, <code class="docutils notranslate"><span class="pre"><a href="../manualpages/SNES/SNESView.html">SNESView</a>()</span></code>, etc.) to view the options that have
been used for a particular solver.</p></li>
<li><p>Run the code with the option <code class="docutils notranslate"><span class="pre">-help</span></code> for a list of the available
runtime commands.</p></li>
<li><p>Use the option <code class="docutils notranslate"><span class="pre">-info</span></code> to print details about the solvers’
operation.</p></li>
<li><p>Use the PETSc monitoring discussed in <a class="reference internal" href="profiling.html#ch-profiling"><span class="std std-ref">Profiling</span></a>
to evaluate the performance of various numerical methods.</p></li>
</ul>
</section>
<section id="tips-for-efficient-use-of-linear-solvers">
<span id="sec-slestips"></span><h3>Tips for Efficient Use of Linear Solvers<a class="headerlink" href="#tips-for-efficient-use-of-linear-solvers" title="Link to this heading">#</a></h3>
<p>As discussed in <a class="reference internal" href="ksp.html#ch-ksp"><span class="std std-ref">KSP: Linear System Solvers</span></a>, the default linear
solvers are</p>
<ul class="simple">
<li><p>uniprocess: GMRES(30) with ILU(0) preconditioning</p></li>
<li><p>multiprocess: GMRES(30) with block Jacobi preconditioning, where
there is 1 block per process, and each block is solved with ILU(0)</p></li>
</ul>
<p>One should experiment to determine alternatives that may be better for
various applications. Recall that one can specify the <code class="docutils notranslate"><span class="pre"><a href="../manualpages/KSP/KSP.html">KSP</a></span></code> methods
and preconditioners at runtime via the options:</p>
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>-ksp_type <ksp_name> -pc_type <pc_name>
</pre></div>
</div>
<p>One can also specify a variety of runtime customizations for the
solvers, as discussed throughout the manual.</p>
<p>In particular, note that the default restart parameter for GMRES is 30,
which may be too small for some large-scale problems. One can alter this
parameter with the option <code class="docutils notranslate"><span class="pre">-ksp_gmres_restart</span> <span class="pre"><restart></span></code> or by calling
<code class="docutils notranslate"><span class="pre"><a href="../manualpages/KSP/KSPGMRESSetRestart.html">KSPGMRESSetRestart</a>()</span></code>. <a class="reference internal" href="ksp.html#sec-ksp"><span class="std std-ref">Krylov Methods</span></a> gives
information on setting alternative GMRES orthogonalization routines,
which may provide much better parallel performance.</p>
<p>For elliptic problems one often obtains good performance and scalability
with multigrid solvers. Consult <a class="reference internal" href="ksp.html#sec-amg"><span class="std std-ref">Algebraic Multigrid (AMG) Preconditioners</span></a> for
available options. Our experience is that GAMG works particularly well
for elasticity problems, whereas hypre does well for scalar problems.</p>
</section>
<section id="system-related-problems">
<h3>System-Related Problems<a class="headerlink" href="#system-related-problems" title="Link to this heading">#</a></h3>
<p>The performance of a code can be affected by a variety of factors,
including the cache behavior, other users on the machine, etc. Below we
briefly describe some common problems and possibilities for overcoming
them.</p>
<ul class="simple">
<li><p><strong>Problem too large for physical memory size</strong>: When timing a
program, one should always leave at least a ten percent margin
between the total memory a process is using and the physical size of
the machine’s memory. One way to estimate the amount of memory used
by given process is with the Unix <code class="docutils notranslate"><span class="pre">getrusage</span></code> system routine.
The PETSc option <code class="docutils notranslate"><span class="pre">-malloc_view</span></code> reports all
memory usage, including any Fortran arrays in an application code.</p></li>
<li><p><strong>Effects of other users</strong>: If other users are running jobs on the
same physical processor nodes on which a program is being profiled,
the timing results are essentially meaningless.</p></li>
<li><p><strong>Overhead of timing routines on certain machines</strong>: On certain
machines, even calling the system clock in order to time routines is
slow; this skews all of the flop rates and timing results. The file
<code class="docutils notranslate"><span class="pre">$PETSC_DIR/src/benchmarks/<a href="../manualpages/Sys/PetscTime.html">PetscTime</a>.c</span></code> (<a href="../src/benchmarks/PetscTime.c.html">source</a>)
contains a simple test problem that will approximate the amount of
time required to get the current time in a running program. On good
systems it will on the order of <span class="math">\(10^{-6}\)</span> seconds or less.</p></li>
<li><p><strong>Problem too large for good cache performance</strong>: Certain machines
with lower memory bandwidths (slow memory access) attempt to
compensate by having a very large cache. Thus, if a significant
portion of an application fits within the cache, the program will
achieve very good performance; if the code is too large, the
performance can degrade markedly. To analyze whether this situation
affects a particular code, one can try plotting the total flop rate
as a function of problem size. If the flop rate decreases rapidly at
some point, then the problem may likely be too large for the cache
size.</p></li>
<li><p><strong>Inconsistent timings</strong>: Inconsistent timings are likely due to
other users on the machine, thrashing (using more virtual memory than
available physical memory), or paging in of the initial executable.
<a class="reference internal" href="profiling.html#sec-profaccuracy"><span class="std std-ref">Accurate Profiling and Paging Overheads</span></a> provides information on
overcoming paging overhead when profiling a code. We have found on
all systems that if you follow all the advise above your timings will
be consistent within a variation of less than five percent.</p></li>
</ul>
</section>
</section>
</section>
</article>
<footer class="prev-next-footer">
<div class="prev-next-area">
<a class="left-prev"
href="profiling.html"
title="previous page">
<i class="fa-solid fa-angle-left"></i>
<div class="prev-next-info">
<p class="prev-next-subtitle">previous</p>
<p class="prev-next-title">Profiling</p>
</div>
</a>
<a class="right-next"
href="streams.html"
title="next page">
<div class="prev-next-info">
<p class="prev-next-subtitle">next</p>
<p class="prev-next-title">STREAMS: Example Study</p>
</div>
<i class="fa-solid fa-angle-right"></i>
</a>
</div>
</footer>
</div>
<div class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner">
<div class="sidebar-secondary-item">
<div
id="pst-page-navigation-heading-2"
class="page-toc tocsection onthispage">
<i class="fa-solid fa-list"></i> On this page
</div>
<nav class="bd-toc-nav page-toc" aria-labelledby="pst-page-navigation-heading-2">
<ul class="visible nav section-nav flex-column">
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#maximizing-memory-bandwidth">Maximizing Memory Bandwidth</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#memory-bandwidth-vs-processes">Memory Bandwidth vs. Processes</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#non-uniform-memory-access-numa-and-process-placement">Non-Uniform Memory Access (NUMA) and Process Placement</a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#additional-process-placement-considerations-and-details">Additional Process Placement Considerations and Details</a></li>
</ul>
</li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#performance-pitfalls-and-advice">Performance Pitfalls and Advice</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#debug-vs-optimized-builds">Debug vs. Optimized Builds</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#profiling">Profiling</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#aggregation">Aggregation</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#memory-allocation-for-sparse-matrix-factorization">Memory Allocation for Sparse Matrix Factorization</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#detecting-memory-allocation-problems-and-memory-usage">Detecting Memory Allocation Problems and Memory Usage</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#data-structure-reuse">Data Structure Reuse</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#numerical-experiments">Numerical Experiments</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#tips-for-efficient-use-of-linear-solvers">Tips for Efficient Use of Linear Solvers</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#system-related-problems">System-Related Problems</a></li>
</ul>
</li>
</ul>
</nav></div>
<div class="sidebar-secondary-item">
<div class="tocsection editthispage">
<a href="https://gitlab.com/petsc/petsc/-/edit/release/doc/manual/performance.md">
<i class="fa-solid fa-pencil"></i>
Edit on GitLab
</a>
</div>
</div>
<div class="sidebar-secondary-item">
<div class="tocsection sourcelink">
<a href="../_sources/manual/performance.md.txt">
<i class="fa-solid fa-file-lines"></i> Show Source
</a>
</div>
</div>
</div></div>
</div>
<footer class="bd-footer-content">
</footer>
</main>
</div>
</div>
<!-- Scripts loaded after <body> so the DOM is not blocked -->
<script src="../_static/scripts/bootstrap.js?digest=bd9e20870c6007c4c509"></script>
<script src="../_static/scripts/pydata-sphinx-theme.js?digest=bd9e20870c6007c4c509"></script>
<footer class="bd-footer">
<div class="bd-footer__inner bd-page-width">
<div class="footer-items__start">
<div class="footer-item">
<p class="copyright">
© Copyright 1991-2025, UChicago Argonne, LLC and the PETSc Development Team.
<br/>
</p>
</div>
<div class="footer-item">
<p class="sphinx-version">
Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 7.3.7.
<br/>
</p>
</div>
</div>
<div class="footer-items__end">
<div class="footer-item">
<p class="theme-version">
Built with the <a href="https://pydata-sphinx-theme.readthedocs.io/en/stable/index.html">PyData Sphinx Theme</a> 0.15.1.
</p></div>
<div class="footer-item"><p class="last-updated">
Last updated on 2025-04-30T13:10:40-0500 (v3.23.1).
<br/>
</p></div>
</div>
</div>
</footer>
</body>
</html>
|
