File: perlunicode.html

package info (click to toggle)
perl-doc-html 5.26.0-4
  • links: PTS, VCS
  • area: main
  • in suites: sid
  • size: 39,400 kB
  • sloc: xml: 36; makefile: 7
file content (2295 lines) | stat: -rw-r--r-- 179,206 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
"http://www.w3.org/TR/html4/loose.dtd">
<html>
<head>
  <title>perlunicode - perldoc.perl.org</title>
  <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
  <meta http-equiv="Content-Language" content="en-gb">
  <link rel="search" type="application/opensearchdescription+xml" title="Search perldoc.perl.org" href="/static/search.xml"/>
  <link href="static/css-20100830.css" rel="stylesheet" rev="stylesheet" type="text/css" media="screen">
  <link href="static/exploreperl.css" rel="stylesheet" rev="stylesheet" type="text/css">
</head>

<body onLoad="perldoc.startup();" onPageShow="if (event.persisted) perldoc.startup();">
    <div id="page">
      
      <div id="header">
	<div id="homepage_link">
	  <a href="index.html"></a>
	</div>
	<div id="strapline">
	  Perl Programming Documentation
	</div>
	<div id="download_link" class="download">
	  <a href="http://www.perl.org/get.html">Download Perl</a>
	</div>
	<div id="explore_link" class="download">
	  <a id="explore_anchor" href="#">Explore</a>
	</div>
      </div>
      
      <div id="body">
        <div id="left_column">
          <div class="side_group">
            
	    <div class="side_panel doc_panel">
              <p>Manual</p>
              <ul>
                <li><a href="index-overview.html">Overview</a>
                <li><a href="index-tutorials.html">Tutorials</a>
                <li><a href="index-faq.html">FAQs</a>
                <li><a href="index-history.html">History / Changes</a>
                <li><a href="index-licence.html">License</a>
              </ul>
            </div>
            <div class="side_panel doc_panel">
              <p>Reference</p>
              <ul>
                <li><a href="index-language.html">Language</a>
                <li><a href="index-functions.html">Functions</a>
                <li><a href="perlop.html">Operators</a>
                <li><a href="perlvar.html">Special Variables</a>
                <li><a href="index-pragmas.html">Pragmas</a>
                <li><a href="index-utilities.html">Utilities</a>
                <li><a href="index-internals.html">Internals</a>
                <li><a href="index-platforms.html">Platform Specific</a>
              </ul>
            </div>
            <div class="side_panel doc_panel">
              <p>Modules</p>
              <ul>
		<li>
		
                
                  
		    
		  
		
                  
		    
		  
		
                  
		    
		  
		
                  
		    
		  
		
                  
		    
		  
		
                  
		    
		  
		
                  
		    
		  
		
                  
		    
		  
		
                  
		    
		  
		
                  
		
                  
		
                  
		    
		  
		
                  
		    
		  
		
                  
		    
		  
		
                  
		    
		  
		
                  
		    
		  
		
                  
		
                  
		
                  
		    
		  
		
                  
		    
		  
		
                  
		    
		  
		
                  
		
                  
		
                  
		    
		  
		
                  
		
                  
		
		
                    <a href="index-modules-A.html">A</a>
                    
                      
                        &bull;
                      
                    
                
                    <a href="index-modules-B.html">B</a>
                    
                      
                        &bull;
                      
                    
                
                    <a href="index-modules-C.html">C</a>
                    
                      
                        &bull;
                      
                    
                
                    <a href="index-modules-D.html">D</a>
                    
                      
                        &bull;
                      
                    
                
                    <a href="index-modules-E.html">E</a>
                    
                      
                        <li>
                      
                    
                
                    <a href="index-modules-F.html">F</a>
                    
                      
                        &bull;
                      
                    
                
                    <a href="index-modules-G.html">G</a>
                    
                      
                        &bull;
                      
                    
                
                    <a href="index-modules-H.html">H</a>
                    
                      
                        &bull;
                      
                    
                
                    <a href="index-modules-I.html">I</a>
                    
                      
                        &bull;
                      
                    
                
                    <a href="index-modules-L.html">L</a>
                    
                      
                        <li>
                      
                    
                
                    <a href="index-modules-M.html">M</a>
                    
                      
                        &bull;
                      
                    
                
                    <a href="index-modules-N.html">N</a>
                    
                      
                        &bull;
                      
                    
                
                    <a href="index-modules-O.html">O</a>
                    
                      
                        &bull;
                      
                    
                
                    <a href="index-modules-P.html">P</a>
                    
                      
                        &bull;
                      
                    
                
                    <a href="index-modules-S.html">S</a>
                    
                      
                        <li>
                      
                    
                
                    <a href="index-modules-T.html">T</a>
                    
                      
                        &bull;
                      
                    
                
                    <a href="index-modules-U.html">U</a>
                    
                      
                        &bull;
                      
                    
                
                    <a href="index-modules-X.html">X</a>
                    
                
              </ul>
            </div>
            
	      <div class="side_panel doc_panel">
		<p>Tools</p>
		<ul>
		  <li><a href="preferences.html">Preferences</a>
		</ul>
	      </div>
            
          </div>
        </div>
        <div id="centre_column">
          <div id="content_header">
            <div id="title_bar">
              <div id="page_name">
                <h1>perlunicode</h1>
              </div>
              <div id="perl_version">
                Perl 5 version 26.0 documentation
              </div>
              <div class="page_links" id="page_links_top">
                <a href="#" onClick="toolbar.goToTop();return false;">Go to top</a>
		
              </div>
	      <div class="page_links" id="page_links_bottom">
		
                  <a href="#" id="page_index_toggle">Show page index</a> &bull;
		
                <a href="#" id="recent_pages_toggle">Show recent pages</a>		
	      </div>
	      <div id="search_form">
		<form action="search.html" method="GET" id="search">
		  <input type="text" name="q" id="search_box" alt="Search">
		</form>
	      </div>
            </div>
            <div id="breadcrumbs">
                
    <a href="index.html">Home</a> &gt;
    
      
        <a href="index-language.html">Language reference</a> &gt;
      
    
    perlunicode
  

            </div>
          </div>
          <div id="content_body">
	    <!--[if lt IE 7]>
 <div class="noscript">
   <p>
     <strong>It looks like you're using Internet Explorer 6. This is a very old
     browser which does not offer full support for modern websites.</strong>
   </p>
   <p>
     Unfortunately this means that this website will not work on
     your computer.
   </p>
   <p>
     Don't miss out though! To view the site (and get a better experience from
     many other websites), simply upgrade to
     <a href="http://www.microsoft.com/windows/Internet-explorer/default.aspx">Internet
Explorer 8</a>
     or download an alternative browser such as
     <a href="http://www.mozilla.com/en-US/firefox/firefox.html">Firefox</a>,
     <a href="http://www.apple.com/safari/download/">Safari</a>, or
     <a href="http://www.google.co.uk/chrome">Google Chrome</a>.
   </p>
   <p>
     All of these browsers are free. If you're using a PC at work, you may
     need to contact your IT administrator.
   </p>
 </div>
<![endif]-->
	    <noscript>
	      <div class="noscript">
	      <p>
                <strong>Please note: Many features of this site require JavaScript. You appear to have JavaScript disabled,
	        or are running a non-JavaScript capable web browser.</strong>
	      </p>
	      <p>
		To get the best experience, please enable JavaScript or download a modern web browser such as <a href="http://www.microsoft.com/windows/Internet-explorer/default.aspx">Internet Explorer 8</a>, <a href="http://www.mozilla.com/en-US/firefox/firefox.html">Firefox</a>, <a href="http://www.apple.com/safari/download/">Safari</a>, or <a href="http://www.google.co.uk/chrome">Google Chrome</a>.
              </p>
	      </div>
	    </noscript>

	    <div id="recent_pages" class="hud_container">
	      <div id="recent_pages_header" class="hud_header">
		<div id="recent_pages_close" class="hud_close"><a href="#" onClick="recentPages.hide();return false;"></a></div>
		<div id="recent_pages_title" class="hud_title"><span class="hud_span_top">Recently read</span></div>
		<div id="recent_pages_topright" class="hud_topright"></div>
	      </div>
	      <div id="recent_pages_content" class="hud_content">
	      </div>
	      <div id="recent_pages_footer" class="hud_footer">
		<div id="recent_pages_bottomleft" class="hud_bottomleft"></div>
		<div id="recent_pages_bottom" class="hud_bottom"><span class="hud_span_bottom"></span></div>
		<div id="recent_pages_resize" class="hud_resize"></div>
	      </div>
	    </div>
  
	    <div id="from_search"></div>
            <h1>perlunicode</h1>


  <!--    -->
<ul><li><a href="#NAME">NAME</a><li><a href="#DESCRIPTION">DESCRIPTION</a><ul><li><a href="#Important-Caveats">Important Caveats</a><li><a href="#Byte-and-Character-Semantics">Byte and Character Semantics</a><li><a href="#ASCII-Rules-versus-Unicode-Rules">ASCII Rules versus Unicode Rules</a><li><a href="#Extended-Grapheme-Clusters-(Logical-characters)">Extended Grapheme Clusters (Logical characters)</a><li><a href="#Unicode-Character-Properties">Unicode Character Properties</a><li><a href="#User-Defined-Character-Properties">User-Defined Character Properties</a><li><a href="#User-Defined-Case-Mappings-(for-serious-hackers-only)">User-Defined Case Mappings (for serious hackers only)</a><li><a href="#Character-Encodings-for-Input-and-Output">Character Encodings for Input and Output</a><li><a href="#Unicode-Regular-Expression-Support-Level">Unicode Regular Expression Support Level</a><li><a href="#Unicode-Encodings">Unicode Encodings</a><li><a href="#Noncharacter-code-points">Noncharacter code points</a><li><a href="#Beyond-Unicode-code-points">Beyond Unicode code points</a><li><a href="#Security-Implications-of-Unicode">Security Implications of Unicode</a><li><a href="#Unicode-in-Perl-on-EBCDIC">Unicode in Perl on EBCDIC</a><li><a href="#Locales">Locales</a><li><a href="#When-Unicode-Does-Not-Happen">When Unicode Does Not Happen</a><li><a href="#The-%22Unicode-Bug%22">The "Unicode Bug"</a><li><a href="#Forcing-Unicode-in-Perl-(Or-Unforcing-Unicode-in-Perl)">Forcing Unicode in Perl (Or Unforcing Unicode in Perl)</a><li><a href="#Using-Unicode-in-XS">Using Unicode in XS</a><li><a href="#Hacking-Perl-to-work-on-earlier-Unicode-versions-(for-very-serious-hackers-only)">Hacking Perl to work on earlier Unicode versions (for very serious hackers only)</a><li><a href="#Porting-code-from-perl-5.6.X">Porting code from perl-5.6.X</a></ul><li><a href="#BUGS">BUGS</a><ul><li><a href="#Interaction-with-Extensions">Interaction with Extensions</a><li><a href="#Speed">Speed</a></ul><li><a href="#SEE-ALSO">SEE ALSO</a></ul><a name="NAME"></a><h1>NAME</h1>
<p>perlunicode - Unicode support in Perl</p>
<a name="DESCRIPTION"></a><h1>DESCRIPTION</h1>
<p>If you haven't already, before reading this document, you should become
familiar with both <a href="perlunitut.html">perlunitut</a> and <a href="perluniintro.html">perluniintro</a>.</p>
<p>Unicode aims to <b>UNI</b>-fy the en-<b>CODE</b>-ings of all the world's
character sets into a single Standard.   For quite a few of the various
coding standards that existed when Unicode was first created, converting
from each to Unicode essentially meant adding a constant to each code
point in the original standard, and converting back meant just
subtracting that same constant.  For ASCII and ISO-8859-1, the constant
is 0.  For ISO-8859-5, (Cyrillic) the constant is 864; for Hebrew
(ISO-8859-8), it's 1488; Thai (ISO-8859-11), 3424; and so forth.  This
made it easy to do the conversions, and facilitated the adoption of
Unicode.</p>
<p>And it worked; nowadays, those legacy standards are rarely used.  Most
everyone uses Unicode.</p>
<p>Unicode is a comprehensive standard.  It specifies many things outside
the scope of Perl, such as how to display sequences of characters.  For
a full discussion of all aspects of Unicode, see
<a href="http://www.unicode.org">http://www.unicode.org</a>.</p>
<a name="Important-Caveats"></a><h2>Important Caveats</h2>
<p>Even though some of this section may not be understandable to you on
first reading, we think it's important enough to highlight some of the
gotchas before delving further, so here goes:</p>
<p>Unicode support is an extensive requirement. While Perl does not
implement the Unicode standard or the accompanying technical reports
from cover to cover, Perl does support many Unicode features.</p>
<p>Also, the use of Unicode may present security issues that aren't
obvious, see <a href="#Security-Implications-of-Unicode">Security Implications of Unicode</a>.</p>
<ul>
<li><a name="Safest-if-you-use-feature-'unicode_strings'"></a><b>Safest if you <code class="inline"><a class="l_k" href="functions/use.html">use</a> <span class="w">feature</span> <span class="q">&#39;unicode_strings&#39;</span></code>
</b>
<p>In order to preserve backward compatibility, Perl does not turn
on full internal Unicode support unless the pragma
<a href="feature.html#The-'unicode_strings'-feature">use feature &#39;unicode_strings&#39; </a>
is specified.  (This is automatically
selected if you <code class="inline"><a class="l_k" href="functions/use.html">use</a> <span class="n">5.012</span></code>
 or higher.)  Failure to do this can
trigger unexpected surprises.  See <a href="#The-%22Unicode-Bug%22">The Unicode Bug</a> below.</p>
<p>This pragma doesn't affect I/O.  Nor does it change the internal
representation of strings, only their interpretation.  There are still
several places where Unicode isn't fully supported, such as in
filenames.</p>
</li>
<li><a name="Input-and-Output-Layers"></a><b>Input and Output Layers</b>
<p>Use the <code class="inline"><span class="j">:</span><span class="i">encoding</span><span class="s">(</span>...<span class="s">)</span></code>
 layer  to read from and write to
filehandles using the specified encoding.  (See <a href="open.html">open</a>.)</p>
</li>
<li><a name="You-should-convert-your-non-ASCII%2c-non-UTF-8-Perl-scripts-to-be-UTF-8."></a><b>You should convert your non-ASCII, non-UTF-8 Perl scripts to be
UTF-8.</b>
<p>See <a href="encoding.html">encoding</a>.</p>
</li>
<li><a name="use-utf8-still-needed-to-enable-%2fUnicode-Encodings-in-scripts"></a><b><code class="inline"><a class="l_k" href="functions/use.html">use</a> <span class="w">utf8</span></code>
 still needed to enable <a href="#Unicode-Encodings">UTF-8</a> in scripts</b>
<p>If your Perl script is itself encoded in <a href="#Unicode-Encodings">UTF-8</a>,
the <code class="inline"><a class="l_k" href="functions/use.html">use</a> <span class="w">utf8</span></code>
 pragma must be explicitly included to enable
recognition of that (in string or regular expression literals, or in
identifier names).  <b>This is the only time when an explicit <code class="inline"><a class="l_k" href="functions/use.html">use</a>
<span class="w">utf8</span></code>
 is needed.</b>  (See <a href="utf8.html">utf8</a>).</p>
<p>If a Perl script begins with the bytes that form the UTF-8 encoding of
the Unicode BYTE ORDER MARK (<code class="inline"><span class="w">BOM</span></code>
, see <a href="#Unicode-Encodings">Unicode Encodings</a>), those
bytes are completely ignored.</p>
</li>
<li><a name="%2fUnicode-Encodings-scripts-autodetected"></a><b><a href="#Unicode-Encodings">UTF-16</a> scripts autodetected</b>
<p>If a Perl script begins with the Unicode <code class="inline"><span class="w">BOM</span></code>
 (UTF-16LE,
UTF16-BE), or if the script looks like non-<code class="inline"><span class="w">BOM</span></code>
-marked
UTF-16 of either endianness, Perl will correctly read in the script as
the appropriate Unicode encoding.</p>
</li>
</ul>
<a name="Byte-and-Character-Semantics"></a><h2>Byte and Character Semantics</h2>
<p>Before Unicode, most encodings used 8 bits (a single byte) to encode
each character.  Thus a character was a byte, and a byte was a
character, and there could be only 256 or fewer possible characters.
"Byte Semantics" in the title of this section refers to
this behavior.  There was no need to distinguish between "Byte" and
"Character".</p>
<p>Then along comes Unicode which has room for over a million characters
(and Perl allows for even more).  This means that a character may
require more than a single byte to represent it, and so the two terms
are no longer equivalent.  What matter are the characters as whole
entities, and not usually the bytes that comprise them.  That's what the
term "Character Semantics" in the title of this section refers to.</p>
<p>Perl had to change internally to decouple "bytes" from "characters".
It is important that you too change your ideas, if you haven't already,
so that "byte" and "character" no longer mean the same thing in your
mind.</p>
<p>The basic building block of Perl strings has always been a "character".
The changes basically come down to that the implementation no longer
thinks that a character is always just a single byte.</p>
<p>There are various things to note:</p>
<ul>
<li>
<p>String handling functions, for the most part, continue to operate in
terms of characters.  <code class="inline"><a class="l_k" href="functions/length.html">length()</a></code>, for example, returns the number of
characters in a string, just as before.  But that number no longer is
necessarily the same as the number of bytes in the string (there may be
more bytes than characters).  The other such functions include
<code class="inline"><a class="l_k" href="functions/chop.html">chop()</a></code>, <code class="inline"><a class="l_k" href="functions/chomp.html">chomp()</a></code>, <code class="inline"><a class="l_k" href="functions/substr.html">substr()</a></code>, <code class="inline"><a class="l_k" href="functions/pos.html">pos()</a></code>, <code class="inline"><a class="l_k" href="functions/index.html">index()</a></code>, <code class="inline"><a class="l_k" href="functions/rindex.html">rindex()</a></code>,
<code class="inline"><a class="l_k" href="functions/sort.html">sort()</a></code>, <code class="inline"><a class="l_k" href="functions/sprintf.html">sprintf()</a></code>, and <code class="inline"><a class="l_k" href="functions/write.html">write()</a></code>.</p>
<p>The exceptions are:</p>
<ul>
<li>
<p>the bit-oriented <code class="inline"><a class="l_k" href="functions/vec.html">vec</a></code></p>
<p>&#xa0;</p>
</li>
<li>
<p>the byte-oriented <code class="inline"><a class="l_k" href="functions/pack.html">pack</a></code>/<code class="inline"><a class="l_k" href="functions/unpack.html">unpack</a></code> <code class="inline"><span class="q">&quot;C&quot;</span></code>
 format</p>
<p>However, the <code class="inline"><span class="w">W</span></code>
 specifier does operate on whole characters, as does the
<code class="inline"><span class="w">U</span></code>
 specifier.</p>
</li>
<li>
<p>some operators that interact with the platform's operating system</p>
<p>Operators dealing with filenames are examples.</p>
</li>
<li>
<p>when the functions are called from within the scope of the
<code class="inline"><a href="bytes.html">use bytes</a></code> pragma</p>
<p>Likely, you should use this only for debugging anyway.</p>
</li>
</ul>
</li>
<li>
<p>Strings--including hash keys--and regular expression patterns may
contain characters that have ordinal values larger than 255.</p>
<p>If you use a Unicode editor to edit your program, Unicode characters may
occur directly within the literal strings in UTF-8 encoding, or UTF-16.
(The former requires a <code class="inline"><a class="l_k" href="functions/use.html">use</a> <span class="w">utf8</span></code>
, the latter may require a <code class="inline"><span class="w">BOM</span></code>
.)</p>
<p><a href="perluniintro.html#Creating-Unicode">Creating Unicode in perluniintro</a> gives other ways to place non-ASCII
characters in your strings.</p>
</li>
<li>
<p>The <code class="inline"><a class="l_k" href="functions/chr.html">chr()</a></code> and <code class="inline"><a class="l_k" href="functions/ord.html">ord()</a></code> functions work on whole characters.</p>
</li>
<li>
<p>Regular expressions match whole characters.  For example, <code class="inline"><span class="q">&quot;.&quot;</span></code>
 matches
a whole character instead of only a single byte.</p>
</li>
<li>
<p>The <code class="inline"><a class="l_k" href="functions/tr.html">tr///</a></code> operator translates whole characters.  (Note that the
<code class="inline"><a class="l_k" href="functions/tr.html">tr///CU</a></code> functionality has been removed.  For similar functionality to
that, see <code class="inline"><a class="l_k" href="functions/pack.html">pack</a><span class="s">(</span><span class="q">&#39;U0&#39;</span><span class="cm">,</span> ...<span class="s">)</span></code>
 and <code class="inline"><a class="l_k" href="functions/pack.html">pack</a><span class="s">(</span><span class="q">&#39;C0&#39;</span><span class="cm">,</span> ...<span class="s">)</span></code>
).</p>
</li>
<li>
<p><code class="inline"><a class="l_k" href="functions/scalar.html">scalar</a> <a class="l_k" href="functions/reverse.html">reverse</a><span class="s">(</span><span class="s">)</span></code>
 reverses by character rather than by byte.</p>
</li>
<li>
<p>The bit string operators, <code class="inline"><span class="i">&amp; |</span> ^ ~</code>
 and (starting in v5.22)
<code class="inline"><span class="i">&amp;.</span> |. ^.  ~.</code>
 can operate on characters that don't fit into a byte.
However, the current behavior is likely to change.  You should not use
these operators on strings that are encoded in UTF-8.  If you're not
sure about the encoding of a string, downgrade it before using any of
these operators; you can use
<a href="utf8.html#Utility-functions">utf8::utf8_downgrade() </a>.</p>
</li>
</ul>
<p>The bottom line is that Perl has always practiced "Character Semantics",
but with the advent of Unicode, that is now different than "Byte
Semantics".</p>
<a name="ASCII-Rules-versus-Unicode-Rules"></a><h2>ASCII Rules versus Unicode Rules</h2>
<p>Before Unicode, when a character was a byte was a character,
Perl knew only about the 128 characters defined by ASCII, code points 0
through 127 (except for under <a href="perllocale.html">use locale </a>).  That
left the code
points 128 to 255 as unassigned, and available for whatever use a
program might want.  The only semantics they have is their ordinal
numbers, and that they are members of none of the non-negative character
classes.  None are considered to match <code class="inline">\<span class="w">w</span></code>
 for example, but all match
<code class="inline">\<span class="w">W</span></code>
.</p>
<p>Unicode, of course, assigns each of those code points a particular
meaning (along with ones above 255).  To preserve backward
compatibility, Perl only uses the Unicode meanings when there is some
indication that Unicode is what is intended; otherwise the non-ASCII
code points remain treated as if they are unassigned.</p>
<p>Here are the ways that Perl knows that a string should be treated as
Unicode:</p>
<ul>
<li>
<p>Within the scope of <code class="inline"><a class="l_k" href="functions/use.html">use</a> <span class="w">utf8</span></code>
</p>
<p>If the whole program is Unicode (signified by using 8-bit <b>U</b>nicode
<b>T</b>ransformation <b>F</b>ormat), then all strings within it must be
Unicode.</p>
</li>
<li>
<p>Within the scope of
<a href="feature.html#The-'unicode_strings'-feature">use feature &#39;unicode_strings&#39; </a></p>
<p>This pragma was created so you can explicitly tell Perl that operations
executed within its scope are to use Unicode rules.  More operations are
affected with newer perls.  See <a href="#The-%22Unicode-Bug%22">The Unicode Bug</a>.</p>
</li>
<li>
<p>Within the scope of <code class="inline"><a class="l_k" href="functions/use.html">use</a> <span class="n">5.012</span></code>
 or higher</p>
<p>This implicitly turns on <code class="inline"><a class="l_k" href="functions/use.html">use</a> <span class="w">feature</span> <span class="q">&#39;unicode_strings&#39;</span></code>
.</p>
</li>
<li>
<p>Within the scope of
<a href="perllocale.html#Unicode-and-UTF-8">use locale &#39;not_characters&#39; </a>,
or <a href="perllocale.html">use locale </a> and the current
locale is a UTF-8 locale.</p>
<p>The former is defined to imply Unicode handling; and the latter
indicates a Unicode locale, hence a Unicode interpretation of all
strings within it.</p>
</li>
<li>
<p>When the string contains a Unicode-only code point</p>
<p>Perl has never accepted code points above 255 without them being
Unicode, so their use implies Unicode for the whole string.</p>
</li>
<li>
<p>When the string contains a Unicode named code point <code class="inline">\<span class="i">N</span><span class="s">{</span>...<span class="s">}</span></code>
</p>
<p>The <code class="inline">\<span class="i">N</span><span class="s">{</span>...<span class="s">}</span></code>
 construct explicitly refers to a Unicode code point,
even if it is one that is also in ASCII.  Therefore the string
containing it must be Unicode.</p>
</li>
<li>
<p>When the string has come from an external source marked as
Unicode</p>
<p>The <a href="perlrun.html#-C-%5bnumber%2flist%5d">-C </a> command line option can
specify that certain inputs to the program are Unicode, and the values
of this can be read by your Perl code, see <a href="perlvar.html#%24%7b%5eUNICODE%7d">${^UNICODE} in perlvar</a>.</p>
</li>
<li><a name="*-When-the-string-has-been-upgraded-to-UTF-8"></a><b>When the string has been upgraded to UTF-8</b>
<p>The function <a href="utf8.html#Utility-functions">utf8::utf8_upgrade() </a>
can be explicitly used to permanently (unless a subsequent
<code class="inline"><span class="i">utf8::utf8_downgrade</span><span class="s">(</span><span class="s">)</span></code>
 is called) cause a string to be treated as
Unicode.</p>
</li>
<li><a name="*-There-are-additional-methods-for-regular-expression-patterns"></a><b>There are additional methods for regular expression patterns</b>
<p>A pattern that is compiled with the <code class="inline"><span class="q">/u</span></code>
 or <code class="inline"><span class="q">/a</span></code>
 modifiers is
treated as Unicode (though there are some restrictions with <code class="inline"><span class="q">/a</span></code>
).
Under the <code class="inline">/d</code> and <code class="inline"><span class="q">/l</span></code>
 modifiers, there are several other
indications for Unicode; see <a href="perlre.html#Character-set-modifiers">Character set modifiers in perlre</a>.</p>
</li>
</ul>
<p>Note that all of the above are overridden within the scope of
<code class="inline"><a href="bytes.html">use bytes</a></code>; but you should be using this pragma only for
debugging.</p>
<p>Note also that some interactions with the platform's operating system
never use Unicode rules.</p>
<p>When Unicode rules are in effect:</p>
<ul>
<li>
<p>Case translation operators use the Unicode case translation tables.</p>
<p>Note that <code class="inline"><a class="l_k" href="functions/uc.html">uc()</a></code>, or <code class="inline">\<span class="w">U</span></code>
 in interpolated strings, translates to
uppercase, while <code class="inline"><a class="l_k" href="functions/ucfirst.html">ucfirst</a></code>, or <code class="inline">\<span class="w">u</span></code>
 in interpolated strings,
translates to titlecase in languages that make the distinction (which is
equivalent to uppercase in languages without the distinction).</p>
<p>There is a CPAN module, <code class="inline"><a href="http://search.cpan.org/perldoc/Unicode::Casing">Unicode::Casing</a></code>, which allows you to
define your own mappings to be used in <code class="inline"><a class="l_k" href="functions/lc.html">lc()</a></code>, <code class="inline"><a class="l_k" href="functions/lcfirst.html">lcfirst()</a></code>, <code class="inline"><a class="l_k" href="functions/uc.html">uc()</a></code>,
<code class="inline"><a class="l_k" href="functions/ucfirst.html">ucfirst()</a></code>, and <code class="inline"><a class="l_k" href="functions/fc.html">fc</a></code> (or their double-quoted string inlined versions
such as <code class="inline">\<span class="w">U</span></code>
).  (Prior to Perl 5.16, this functionality was partially
provided in the Perl core, but suffered from a number of insurmountable
drawbacks, so the CPAN module was written instead.)</p>
</li>
<li>
<p>Character classes in regular expressions match based on the character
properties specified in the Unicode properties database.</p>
<p><code class="inline">\<span class="w">w</span></code>
 can be used to match a Japanese ideograph, for instance; and
<code class="inline"><span class="s">[</span><span class="s">[</span><span class="co">:</span><span class="w">digit</span><span class="co">:</span><span class="s">]</span><span class="s">]</span></code>
 a Bengali number.</p>
</li>
<li>
<p>Named Unicode properties, scripts, and block ranges may be used (like
bracketed character classes) by using the <code class="inline">\<span class="w">p</span><span class="s">{</span><span class="s">}</span></code>
 "matches property"
construct and the <code class="inline">\<span class="w">P</span><span class="s">{</span><span class="s">}</span></code>
 negation, "doesn't match property".</p>
<p>See <a href="#Unicode-Character-Properties">Unicode Character Properties</a> for more details.</p>
<p>You can define your own character properties and use them
in the regular expression with the <code class="inline">\<span class="w">p</span><span class="s">{</span><span class="s">}</span></code>
 or <code class="inline">\<span class="w">P</span><span class="s">{</span><span class="s">}</span></code>
 construct.
See <a href="#User-Defined-Character-Properties">User-Defined Character Properties</a> for more details.</p>
</li>
</ul>
<a name="Extended-Grapheme-Clusters-(Logical-characters)"></a><h2>Extended Grapheme Clusters (Logical characters)</h2>
<p>Consider a character, say <code class="inline"><span class="w">H</span></code>
.  It could appear with various marks around it,
such as an acute accent, or a circumflex, or various hooks, circles, arrows,
<i>etc.</i>, above, below, to one side or the other, <i>etc</i>.  There are many
possibilities among the world's languages.  The number of combinations is
astronomical, and if there were a character for each combination, it would
soon exhaust Unicode's more than a million possible characters.  So Unicode
took a different approach: there is a character for the base <code class="inline"><span class="w">H</span></code>
, and a
character for each of the possible marks, and these can be variously combined
to get a final logical character.  So a logical character--what appears to be a
single character--can be a sequence of more than one individual characters.
The Unicode standard calls these "extended grapheme clusters" (which
is an improved version of the no-longer much used "grapheme cluster");
Perl furnishes the <code class="inline">\<span class="w">X</span></code>
 regular expression construct to match such
sequences in their entirety.</p>
<p>But Unicode's intent is to unify the existing character set standards and
practices, and several pre-existing standards have single characters that
mean the same thing as some of these combinations, like ISO-8859-1,
which has quite a few of them. For example, <code class="inline"><span class="q">&quot;LATIN CAPITAL LETTER E</span>
<span class="q">WITH ACUTE&quot;</span></code>
 was already in this standard when Unicode came along.
Unicode therefore added it to its repertoire as that single character.
But this character is considered by Unicode to be equivalent to the
sequence consisting of the character <code class="inline"><span class="q">&quot;LATIN CAPITAL LETTER E&quot;</span></code>

followed by the character <code class="inline"><span class="q">&quot;COMBINING ACUTE ACCENT&quot;</span></code>
.</p>
<p><code class="inline"><span class="q">&quot;LATIN CAPITAL LETTER E WITH ACUTE&quot;</span></code>
 is called a "pre-composed"
character, and its equivalence with the "E" and the "COMBINING ACCENT"
sequence is called canonical equivalence.  All pre-composed characters
are said to have a decomposition (into the equivalent sequence), and the
decomposition type is also called canonical.  A string may be comprised
as much as possible of precomposed characters, or it may be comprised of
entirely decomposed characters.  Unicode calls these respectively,
"Normalization Form Composed" (NFC) and "Normalization Form Decomposed".
The <code class="inline"><a href="Unicode/Normalize.html">Unicode::Normalize</a></code> module contains functions that convert
between the two.  A string may also have both composed characters and
decomposed characters; this module can be used to make it all one or the
other.</p>
<p>You may be presented with strings in any of these equivalent forms.
There is currently nothing in Perl 5 that ignores the differences.  So
you'll have to specially hanlde it.  The usual advice is to convert your
inputs to <code class="inline"><span class="w">NFD</span></code>
 before processing further.</p>
<p>For more detailed information, see <a href="http://unicode.org/reports/tr15/">http://unicode.org/reports/tr15/</a>.</p>
<a name="Unicode-Character-Properties"></a><h2>Unicode Character Properties</h2>
<p>(The only time that Perl considers a sequence of individual code
points as a single logical character is in the <code class="inline">\<span class="w">X</span></code>
 construct, already
mentioned above.   Therefore "character" in this discussion means a single
Unicode code point.)</p>
<p>Very nearly all Unicode character properties are accessible through
regular expressions by using the <code class="inline">\<span class="w">p</span><span class="s">{</span><span class="s">}</span></code>
 "matches property" construct
and the <code class="inline">\<span class="w">P</span><span class="s">{</span><span class="s">}</span></code>
 "doesn't match property" for its negation.</p>
<p>For instance, <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Uppercase</span><span class="s">}</span></code>
 matches any single character with the Unicode
<code class="inline"><span class="q">&quot;Uppercase&quot;</span></code>
 property, while <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">L</span><span class="s">}</span></code>
 matches any character with a
<code class="inline"><span class="w">General_Category</span></code>
 of <code class="inline"><span class="q">&quot;L&quot;</span></code>
 (letter) property (see
<a href="#General_Category">General_Category</a> below).  Brackets are not
required for single letter property names, so <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">L</span><span class="s">}</span></code>
 is equivalent to <code class="inline">\<span class="w">pL</span></code>
.</p>
<p>More formally, <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Uppercase</span><span class="s">}</span></code>
 matches any single character whose Unicode
<code class="inline"><span class="w">Uppercase</span></code>
 property value is <code class="inline"><span class="w">True</span></code>
, and <code class="inline">\<span class="i">P</span><span class="s">{</span><span class="w">Uppercase</span><span class="s">}</span></code>
 matches any character
whose <code class="inline"><span class="w">Uppercase</span></code>
 property value is <code class="inline"><span class="w">False</span></code>
, and they could have been written as
<code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Uppercase</span>=<span class="w">True</span><span class="s">}</span></code>
 and <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Uppercase</span>=<span class="w">False</span><span class="s">}</span></code>
, respectively.</p>
<p>This formality is needed when properties are not binary; that is, if they can
take on more values than just <code class="inline"><span class="w">True</span></code>
 and <code class="inline"><span class="w">False</span></code>
.  For example, the
<code class="inline"><span class="w">Bidi_Class</span></code>
 property (see <a href="#Bidirectional-Character-Types">Bidirectional Character Types</a> below),
can take on several different
values, such as <code class="inline"><span class="w">Left</span></code>
, <code class="inline"><span class="w">Right</span></code>
, <code class="inline"><span class="w">Whitespace</span></code>
, and others.  To match these, one needs
to specify both the property name (<code class="inline"><span class="w">Bidi_Class</span></code>
), AND the value being
matched against
(<code class="inline"><span class="w">Left</span></code>
, <code class="inline"><span class="w">Right</span></code>
, <i>etc.</i>).  This is done, as in the examples above, by having the
two components separated by an equal sign (or interchangeably, a colon), like
<code class="inline">\<span class="i">p</span><span class="s">{</span><span class="j">Bidi_Class:</span> <span class="w">Left</span><span class="s">}</span></code>
.</p>
<p>All Unicode-defined character properties may be written in these compound forms
of <code class="inline">\p{<i>property</i>=<i>value</i>}</code> or <code class="inline">\p{<i>property</i>:<i>value</i>}</code>, but Perl provides some
additional properties that are written only in the single form, as well as
single-form short-cuts for all binary properties and certain others described
below, in which you may omit the property name and the equals or colon
separator.</p>
<p>Most Unicode character properties have at least two synonyms (or aliases if you
prefer): a short one that is easier to type and a longer one that is more
descriptive and hence easier to understand.  Thus the <code class="inline"><span class="q">&quot;L&quot;</span></code>
 and
<code class="inline"><span class="q">&quot;Letter&quot;</span></code>
 properties above are equivalent and can be used
interchangeably.  Likewise, <code class="inline"><span class="q">&quot;Upper&quot;</span></code>
 is a synonym for <code class="inline"><span class="q">&quot;Uppercase&quot;</span></code>
,
and we could have written <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Uppercase</span><span class="s">}</span></code>
 equivalently as <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Upper</span><span class="s">}</span></code>
.
Also, there are typically various synonyms for the values the property
can be.   For binary properties, <code class="inline"><span class="q">&quot;True&quot;</span></code>
 has 3 synonyms: <code class="inline"><span class="q">&quot;T&quot;</span></code>
,
<code class="inline"><span class="q">&quot;Yes&quot;</span></code>
, and <code class="inline"><span class="q">&quot;Y&quot;</span></code>
; and <code class="inline"><span class="q">&quot;False&quot;</span></code>
 has correspondingly <code class="inline"><span class="q">&quot;F&quot;</span></code>
,
<code class="inline"><span class="q">&quot;No&quot;</span></code>
, and <code class="inline"><span class="q">&quot;N&quot;</span></code>
.  But be careful.  A short form of a value for one
property may not mean the same thing as the same short form for another.
Thus, for the <code class="inline"><a href="#General_Category">General_Category</a></code> property, <code class="inline"><span class="q">&quot;L&quot;</span></code>
 means
<code class="inline"><span class="q">&quot;Letter&quot;</span></code>
, but for the <a href="#Bidirectional-Character-Types">Bidi_Class </a>
property, <code class="inline"><span class="q">&quot;L&quot;</span></code>
 means <code class="inline"><span class="q">&quot;Left&quot;</span></code>
.  A complete list of properties and
synonyms is in <a href="perluniprops.html">perluniprops</a>.</p>
<p>Upper/lower case differences in property names and values are irrelevant;
thus <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Upper</span><span class="s">}</span></code>
 means the same thing as <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">upper</span><span class="s">}</span></code>
 or even <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">UpPeR</span><span class="s">}</span></code>
.
Similarly, you can add or subtract underscores anywhere in the middle of a
word, so that these are also equivalent to <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">U_p_p_e_r</span><span class="s">}</span></code>
.  And white space
is irrelevant adjacent to non-word characters, such as the braces and the equals
or colon separators, so <code class="inline">\<span class="i">p</span><span class="s">{</span>   <span class="w">Upper</span>  <span class="s">}</span></code>
 and <code class="inline">\<span class="i">p</span><span class="s">{</span> <span class="j">Upper_case :</span> <span class="w">Y</span> <span class="s">}</span></code>
 are
equivalent to these as well.  In fact, white space and even
hyphens can usually be added or deleted anywhere.  So even <code class="inline">\<span class="i">p</span><span class="s">{</span> <span class="w">Up</span>-<span class="w">per</span> case = <span class="w">Yes</span><span class="s">}</span></code>
 is
equivalent.  All this is called "loose-matching" by Unicode.  The few places
where stricter matching is used is in the middle of numbers, and in the Perl
extension properties that begin or end with an underscore.  Stricter matching
cares about white space (except adjacent to non-word characters),
hyphens, and non-interior underscores.</p>
<p>You can also use negation in both <code class="inline">\<span class="w">p</span><span class="s">{</span><span class="s">}</span></code>
 and <code class="inline">\<span class="w">P</span><span class="s">{</span><span class="s">}</span></code>
 by introducing a caret
(<code class="inline">^</code>) between the first brace and the property name: <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">^Tamil</span><span class="s">}</span></code>
 is
equal to <code class="inline">\<span class="i">P</span><span class="s">{</span><span class="w">Tamil</span><span class="s">}</span></code>
.</p>
<p>Almost all properties are immune to case-insensitive matching.  That is,
adding a <code class="inline">/i</code> regular expression modifier does not change what they
match.  There are two sets that are affected.
The first set is
<code class="inline"><span class="w">Uppercase_Letter</span></code>
,
<code class="inline"><span class="w">Lowercase_Letter</span></code>
,
and <code class="inline"><span class="w">Titlecase_Letter</span></code>
,
all of which match <code class="inline"><span class="w">Cased_Letter</span></code>
 under <code class="inline">/i</code> matching.
And the second set is
<code class="inline"><span class="w">Uppercase</span></code>
,
<code class="inline"><span class="w">Lowercase</span></code>
,
and <code class="inline"><span class="w">Titlecase</span></code>
,
all of which match <code class="inline"><span class="w">Cased</span></code>
 under <code class="inline">/i</code> matching.
This set also includes its subsets <code class="inline"><span class="w">PosixUpper</span></code>
 and <code class="inline"><span class="w">PosixLower</span></code>
 both
of which under <code class="inline">/i</code> match <code class="inline"><span class="w">PosixAlpha</span></code>
.
(The difference between these sets is that some things, such as Roman
numerals, come in both upper and lower case so they are <code class="inline"><span class="w">Cased</span></code>
, but
aren't considered letters, so they aren't <code class="inline"><span class="w">Cased_Letter</span></code>
's.)</p>
<p>See <a href="#Beyond-Unicode-code-points">Beyond Unicode code points</a> for special considerations when
matching Unicode properties against non-Unicode code points.</p>
<a name="*General_Category*"></a><h3><b>General_Category</b></h3>
<p>Every Unicode character is assigned a general category, which is the "most
usual categorization of a character" (from
<a href="http://www.unicode.org/reports/tr44">http://www.unicode.org/reports/tr44</a>).</p>
<p>The compound way of writing these is like <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">General_Category</span>=<span class="w">Number</span><span class="s">}</span></code>

(short: <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="j">gc:</span><span class="w">n</span><span class="s">}</span></code>
).  But Perl furnishes shortcuts in which everything up
through the equal or colon separator is omitted.  So you can instead just write
<code class="inline">\<span class="w">pN</span></code>
.</p>
<p>Here are the short and long forms of the values the <code class="inline"><span class="w">General</span> <span class="w">Category</span></code>
 property
can have:</p>
<pre class="verbatim"><ol><li>    <span class="w">Short</span>       <span class="w">Long</span></li><li></li><li>    <span class="w">L</span>           <span class="w">Letter</span></li><li>    <span class="w">LC</span><span class="cm">,</span> <span class="w">L</span><span class="i">&amp;      Cased_Letter</span> <span class="s">(</span><span class="w">that</span> <span class="w">is</span><span class="co">:</span> <span class="s">[</span>\<span class="i">p</span><span class="s">{</span><span class="w">Ll</span><span class="s">}</span>\<span class="i">p</span><span class="s">{</span><span class="w">Lu</span><span class="s">}</span>\<span class="i">p</span><span class="s">{</span><span class="w">Lt</span><span class="s">}</span><span class="s">]</span><span class="s">)</span></li><li>    <span class="w">Lu</span>          <span class="w">Uppercase_Letter</span></li><li>    <span class="w">Ll</span>          <span class="w">Lowercase_Letter</span></li><li>    <span class="w">Lt</span>          <span class="w">Titlecase_Letter</span></li><li>    <span class="w">Lm</span>          <span class="w">Modifier_Letter</span></li><li>    <span class="w">Lo</span>          <span class="w">Other_Letter</span></li><li></li><li>    <span class="w">M</span>           <span class="w">Mark</span></li><li>    <span class="w">Mn</span>          <span class="w">Nonspacing_Mark</span></li><li>    <span class="w">Mc</span>          <span class="w">Spacing_Mark</span></li><li>    <span class="w">Me</span>          <span class="w">Enclosing_Mark</span></li><li></li><li>    <span class="w">N</span>           <span class="w">Number</span></li><li>    <span class="w">Nd</span>          <span class="w">Decimal_Number</span> <span class="s">(</span><span class="w">also</span> <span class="w">Digit</span><span class="s">)</span></li><li>    <span class="w">Nl</span>          <span class="w">Letter_Number</span></li><li>    <span class="w">No</span>          <span class="w">Other_Number</span></li><li></li><li>    <span class="w">P</span>           <span class="w">Punctuation</span> <span class="s">(</span><span class="w">also</span> <span class="w">Punct</span><span class="s">)</span></li><li>    <span class="w">Pc</span>          <span class="w">Connector_Punctuation</span></li><li>    <span class="w">Pd</span>          <span class="w">Dash_Punctuation</span></li><li>    <span class="w">Ps</span>          <span class="w">Open_Punctuation</span></li><li>    <span class="w">Pe</span>          <span class="w">Close_Punctuation</span></li><li>    <span class="w">Pi</span>          <span class="i">Initial_Punctuation</span></li><li>                <span class="s">(</span><span class="w">may</span> <span class="w">behave</span> <span class="w">like</span> <span class="w">Ps</span> or <span class="w">Pe</span> <span class="w">depending</span> <span class="w">on</span> <span class="w">usage</span><span class="s">)</span></li><li>    <span class="w">Pf</span>          <span class="i">Final_Punctuation</span></li><li>                <span class="s">(</span><span class="w">may</span> <span class="w">behave</span> <span class="w">like</span> <span class="w">Ps</span> or <span class="w">Pe</span> <span class="w">depending</span> <span class="w">on</span> <span class="w">usage</span><span class="s">)</span></li><li>    <span class="w">Po</span>          <span class="w">Other_Punctuation</span></li><li></li><li>    <span class="w">S</span>           <span class="w">Symbol</span></li><li>    <span class="w">Sm</span>          <span class="w">Math_Symbol</span></li><li>    <span class="w">Sc</span>          <span class="w">Currency_Symbol</span></li><li>    <span class="w">Sk</span>          <span class="w">Modifier_Symbol</span></li><li>    <span class="w">So</span>          <span class="w">Other_Symbol</span></li><li></li><li>    <span class="w">Z</span>           <span class="w">Separator</span></li><li>    <span class="w">Zs</span>          <span class="w">Space_Separator</span></li><li>    <span class="w">Zl</span>          <span class="w">Line_Separator</span></li><li>    <span class="w">Zp</span>          <span class="w">Paragraph_Separator</span></li><li></li><li>    <span class="w">C</span>           <span class="w">Other</span></li><li>    <span class="w">Cc</span>          <span class="w">Control</span> <span class="s">(</span><span class="w">also</span> <span class="w">Cntrl</span><span class="s">)</span></li><li>    <span class="w">Cf</span>          <span class="w">Format</span></li><li>    <span class="w">Cs</span>          <span class="w">Surrogate</span></li><li>    <span class="w">Co</span>          <span class="w">Private_Use</span></li><li>    <span class="w">Cn</span>          <span class="w">Unassigned</span></li></ol></pre><p>Single-letter properties match all characters in any of the
two-letter sub-properties starting with the same letter.
<code class="inline"><span class="w">LC</span></code>
 and <code class="inline"><span class="w">L</span><span class="i">&amp;</span></code>
 are special: both are aliases for the set consisting of everything matched by <code class="inline"><span class="w">Ll</span></code>
, <code class="inline"><span class="w">Lu</span></code>
, and <code class="inline"><span class="w">Lt</span></code>
.</p>
<a name="*Bidirectional-Character-Types*"></a><h3><b>Bidirectional Character Types</b></h3>
<p>Because scripts differ in their directionality (Hebrew and Arabic are
written right to left, for example) Unicode supplies a <code class="inline"><span class="w">Bidi_Class</span></code>
 property.
Some of the values this property can have are:</p>
<pre class="verbatim"><ol><li>    <span class="w">Value</span>       <span class="w">Meaning</span></li><li></li><li>    <span class="w">L</span>           <span class="w">Left</span>-<span class="w">to</span>-<span class="w">Right</span></li><li>    <span class="w">LRE</span>         <span class="w">Left</span>-<span class="w">to</span>-<span class="w">Right</span> <span class="w">Embedding</span></li><li>    <span class="w">LRO</span>         <span class="w">Left</span>-<span class="w">to</span>-<span class="w">Right</span> <span class="w">Override</span></li><li>    <span class="w">R</span>           <span class="w">Right</span>-<span class="w">to</span>-<span class="w">Left</span></li><li>    <span class="w">AL</span>          <span class="w">Arabic</span> <span class="w">Letter</span></li><li>    <span class="w">RLE</span>         <span class="w">Right</span>-<span class="w">to</span>-<span class="w">Left</span> <span class="w">Embedding</span></li><li>    <span class="w">RLO</span>         <span class="w">Right</span>-<span class="w">to</span>-<span class="w">Left</span> <span class="w">Override</span></li><li>    <span class="w">PDF</span>         <span class="w">Pop</span> <span class="w">Directional</span> <span class="w">Format</span></li><li>    <span class="w">EN</span>          <span class="w">European</span> <span class="w">Number</span></li><li>    <span class="w">ES</span>          <span class="w">European</span> <span class="w">Separator</span></li><li>    <span class="w">ET</span>          <span class="w">European</span> <span class="w">Terminator</span></li><li>    <span class="w">AN</span>          <span class="w">Arabic</span> <span class="w">Number</span></li><li>    <span class="w">CS</span>          <span class="w">Common</span> <span class="w">Separator</span></li><li>    <span class="w">NSM</span>         <span class="w">Non</span>-<span class="w">Spacing</span> <span class="w">Mark</span></li><li>    <span class="w">BN</span>          <span class="w">Boundary</span> <span class="w">Neutral</span></li><li>    <span class="w">B</span>           <span class="w">Paragraph</span> <span class="w">Separator</span></li><li>    <span class="w">S</span>           <span class="w">Segment</span> <span class="w">Separator</span></li><li>    <span class="w">WS</span>          <span class="w">Whitespace</span></li><li>    <span class="w">ON</span>          <span class="w">Other</span> <span class="w">Neutrals</span></li></ol></pre><p>This property is always written in the compound form.
For example, <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="j">Bidi_Class:</span><span class="w">R</span><span class="s">}</span></code>
 matches characters that are normally
written right to left.  Unlike the
<code class="inline"><a href="#General_Category">General_Category</a></code> property, this
property can have more values added in a future Unicode release.  Those
listed above comprised the complete set for many Unicode releases, but
others were added in Unicode 6.3; you can always find what the
current ones are in <a href="perluniprops.html">perluniprops</a>.  And
<a href="http://www.unicode.org/reports/tr9/">http://www.unicode.org/reports/tr9/</a> describes how to use them.</p>
<a name="*Scripts*"></a><h3><b>Scripts</b></h3>
<p>The world's languages are written in many different scripts.  This sentence
(unless you're reading it in translation) is written in Latin, while Russian is
written in Cyrillic, and Greek is written in, well, Greek; Japanese mainly in
Hiragana or Katakana.  There are many more.</p>
<p>The Unicode <code class="inline"><span class="w">Script</span></code>
 and <code class="inline"><span class="w">Script_Extensions</span></code>
 properties give what
script a given character is in.  The <code class="inline"><span class="w">Script_Extensions</span></code>
 property is an
improved version of <code class="inline"><span class="w">Script</span></code>
, as demonstrated below.  Either property
can be specified with the compound form like
<code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Script</span>=<span class="w">Hebrew</span><span class="s">}</span></code>
 (short: <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">sc</span>=<span class="w">hebr</span><span class="s">}</span></code>
), or
<code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Script_Extensions</span>=<span class="w">Javanese</span><span class="s">}</span></code>
 (short: <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">scx</span>=<span class="w">java</span><span class="s">}</span></code>
).
In addition, Perl furnishes shortcuts for all
<code class="inline"><span class="w">Script_Extensions</span></code>
 property names.  You can omit everything up through
the equals (or colon), and simply write <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Latin</span><span class="s">}</span></code>
 or <code class="inline">\<span class="i">P</span><span class="s">{</span><span class="w">Cyrillic</span><span class="s">}</span></code>
.
(This is not true for <code class="inline"><span class="w">Script</span></code>
, which is required to be
written in the compound form.  Prior to Perl v5.26, the single form
returned the plain old <code class="inline"><span class="w">Script</span></code>
 version, but was changed because
<code class="inline"><span class="w">Script_Extensions</span></code>
 gives better results.)</p>
<p>The difference between these two properties involves characters that are
used in multiple scripts.  For example the digits '0' through '9' are
used in many parts of the world.  These are placed in a script named
<code class="inline"><span class="w">Common</span></code>
.  Other characters are used in just a few scripts.  For
example, the <code class="inline"><span class="q">&quot;KATAKANA-HIRAGANA DOUBLE HYPHEN&quot;</span></code>
 is used in both Japanese
scripts, Katakana and Hiragana, but nowhere else.  The <code class="inline"><span class="w">Script</span></code>

property places all characters that are used in multiple scripts in the
<code class="inline"><span class="w">Common</span></code>
 script, while the <code class="inline"><span class="w">Script_Extensions</span></code>
 property places those
that are used in only a few scripts into each of those scripts; while
still using <code class="inline"><span class="w">Common</span></code>
 for those used in many scripts.  Thus both these
match:</p>
<pre class="verbatim"><ol><li> <span class="q">&quot;0&quot;</span> =~ <span class="q">/\p{sc=Common}/</span>     <span class="c"># Matches</span></li><li> <span class="q">&quot;0&quot;</span> =~ <span class="q">/\p{scx=Common}/</span>    <span class="c"># Matches</span></li></ol></pre><p>and only the first of these match:</p>
<pre class="verbatim"><ol><li> <span class="q">&quot;\N{KATAKANA-HIRAGANA DOUBLE HYPHEN}&quot;</span> =~ <span class="q">/\p{sc=Common}  # Matches</span></li><li> <span class="q"> &quot;\N{KATAKANA-HIRAGANA DOUBLE HYPHEN}&quot; =~ /</span>\<span class="i">p</span><span class="s">{</span><span class="w">scx</span>=<span class="w">Common</span><span class="s">}</span> <span class="c"># No match</span></li></ol></pre><p>And only the last two of these match:</p>
<pre class="verbatim"><ol><li> <span class="q">&quot;\N{KATAKANA-HIRAGANA DOUBLE HYPHEN}&quot;</span> =~ <span class="q">/\p{sc=Hiragana}  # No match</span></li><li> <span class="q"> &quot;\N{KATAKANA-HIRAGANA DOUBLE HYPHEN}&quot; =~ /</span>\<span class="i">p</span><span class="s">{</span><span class="w">sc</span>=<span class="w">Katakana</span><span class="s">}</span>  <span class="c"># No match</span></li><li> <span class="q">&quot;\N{KATAKANA-HIRAGANA DOUBLE HYPHEN}&quot;</span> =~ <span class="q">/\p{scx=Hiragana} # Matches</span></li><li> <span class="q"> &quot;\N{KATAKANA-HIRAGANA DOUBLE HYPHEN}&quot; =~ /</span>\<span class="i">p</span><span class="s">{</span><span class="w">scx</span>=<span class="w">Katakana</span><span class="s">}</span> <span class="c"># Matches</span></li></ol></pre><p><code class="inline"><span class="w">Script_Extensions</span></code>
 is thus an improved <code class="inline"><span class="w">Script</span></code>
, in which there are
fewer characters in the <code class="inline"><span class="w">Common</span></code>
 script, and correspondingly more in
other scripts.  It is new in Unicode version 6.0, and its data are likely
to change significantly in later releases, as things get sorted out.
New code should probably be using <code class="inline"><span class="w">Script_Extensions</span></code>
 and not plain
<code class="inline"><span class="w">Script</span></code>
.  If you compile perl with a Unicode release that doesn't have
<code class="inline"><span class="w">Script_Extensions</span></code>
, the single form Perl extensions will instead refer
to the plain <code class="inline"><span class="w">Script</span></code>
 property.  If you compile with a version of
Unicode that doesn't have the <code class="inline"><span class="w">Script</span></code>
 property, these extensions will
not be defined at all.</p>
<p>(Actually, besides <code class="inline"><span class="w">Common</span></code>
, the <code class="inline"><span class="w">Inherited</span></code>
 script, contains
characters that are used in multiple scripts.  These are modifier
characters which inherit the script value
of the controlling character.  Some of these are used in many scripts,
and so go into <code class="inline"><span class="w">Inherited</span></code>
 in both <code class="inline"><span class="w">Script</span></code>
 and <code class="inline"><span class="w">Script_Extensions</span></code>
.
Others are used in just a few scripts, so are in <code class="inline"><span class="w">Inherited</span></code>
 in
<code class="inline"><span class="w">Script</span></code>
, but not in <code class="inline"><span class="w">Script_Extensions</span></code>
.)</p>
<p>It is worth stressing that there are several different sets of digits in
Unicode that are equivalent to 0-9 and are matchable by <code class="inline">\<span class="w">d</span></code>
 in a
regular expression.  If they are used in a single language only, they
are in that language's <code class="inline"><span class="w">Script</span></code>
 and <code class="inline"><span class="w">Script_Extensions</span></code>
.  If they are
used in more than one script, they will be in <code class="inline"><span class="w">sc</span>=<span class="w">Common</span></code>
, but only
if they are used in many scripts should they be in <code class="inline"><span class="w">scx</span>=<span class="w">Common</span></code>
.</p>
<p>The explanation above has omitted some detail; refer to UAX#24 "Unicode
Script Property": <a href="http://www.unicode.org/reports/tr24">http://www.unicode.org/reports/tr24</a>.</p>
<p>A complete list of scripts and their shortcuts is in <a href="perluniprops.html">perluniprops</a>.</p>
<a name="*Use-of-the-%22Is%22-Prefix*"></a><h3><b>Use of the <code class="inline"><span class="q">&quot;Is&quot;</span></code>
 Prefix</b></h3>
<p>For backward compatibility (with Perl 5.6), all properties writable
without using the compound form mentioned
so far may have <code class="inline"><span class="w">Is</span></code>
 or <code class="inline"><span class="w">Is_</span></code>
 prepended to their name, so <code class="inline">\<span class="i">P</span><span class="s">{</span><span class="w">Is_Lu</span><span class="s">}</span></code>
, for
example, is equal to <code class="inline">\<span class="i">P</span><span class="s">{</span><span class="w">Lu</span><span class="s">}</span></code>
, and <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="j">IsScript:</span><span class="w">Arabic</span><span class="s">}</span></code>
 is equal to
<code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Arabic</span><span class="s">}</span></code>
.</p>
<a name="*Blocks*"></a><h3><b>Blocks</b></h3>
<p>In addition to <b>scripts</b>, Unicode also defines <b>blocks</b> of
characters.  The difference between scripts and blocks is that the
concept of scripts is closer to natural languages, while the concept
of blocks is more of an artificial grouping based on groups of Unicode
characters with consecutive ordinal values. For example, the <code class="inline"><span class="q">&quot;Basic Latin&quot;</span></code>

block is all the characters whose ordinals are between 0 and 127, inclusive; in
other words, the ASCII characters.  The <code class="inline"><span class="q">&quot;Latin&quot;</span></code>
 script contains some letters
from this as well as several other blocks, like <code class="inline"><span class="q">&quot;Latin-1 Supplement&quot;</span></code>
,
<code class="inline"><span class="q">&quot;Latin Extended-A&quot;</span></code>
, <i>etc.</i>, but it does not contain all the characters from
those blocks. It does not, for example, contain the digits 0-9, because
those digits are shared across many scripts, and hence are in the
<code class="inline"><span class="w">Common</span></code>
 script.</p>
<p>For more about scripts versus blocks, see UAX#24 "Unicode Script Property":
<a href="http://www.unicode.org/reports/tr24">http://www.unicode.org/reports/tr24</a></p>
<p>The <code class="inline"><span class="w">Script_Extensions</span></code>
 or <code class="inline"><span class="w">Script</span></code>
 properties are likely to be the
ones you want to use when processing
natural language; the <code class="inline"><span class="w">Block</span></code>
 property may occasionally be useful in working
with the nuts and bolts of Unicode.</p>
<p>Block names are matched in the compound form, like <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="j">Block:</span> <span class="w">Arrows</span><span class="s">}</span></code>
 or
<code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Blk</span>=<span class="w">Hebrew</span><span class="s">}</span></code>
.  Unlike most other properties, only a few block names have a
Unicode-defined short name.</p>
<p>Perl also defines single form synonyms for the block property in cases
where these do not conflict with something else.  But don't use any of
these, because they are unstable.  Since these are Perl extensions, they
are subordinate to official Unicode property names; Unicode doesn't know
nor care about Perl's extensions.  It may happen that a name that
currently means the Perl extension will later be changed without warning
to mean a different Unicode property in a future version of the perl
interpreter that uses a later Unicode release, and your code would no
longer work.  The extensions are mentioned here for completeness:  Take
the block name and prefix it with one of: <code class="inline"><span class="w">In</span></code>
 (for example
<code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Blk</span>=<span class="w">Arrows</span><span class="s">}</span></code>
 can currently be written as <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">In_Arrows</span><span class="s">}</span></code>
); or
sometimes <code class="inline"><span class="w">Is</span></code>
 (like <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Is_Arrows</span><span class="s">}</span></code>
); or sometimes no prefix at all
(<code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Arrows</span><span class="s">}</span></code>
).  As of this writing (Unicode 9.0) there are no
conflicts with using the <code class="inline"><span class="w">In_</span></code>
 prefix, but there are plenty with the
other two forms.  For example, <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Is_Hebrew</span><span class="s">}</span></code>
 and <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Hebrew</span><span class="s">}</span></code>
 mean
<code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Script_Extensions</span>=<span class="w">Hebrew</span><span class="s">}</span></code>
 which is NOT the same thing as
<code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Blk</span>=<span class="w">Hebrew</span><span class="s">}</span></code>
.  Our
advice used to be to use the <code class="inline"><span class="w">In_</span></code>
 prefix as a single form way of
specifying a block.  But Unicode 8.0 added properties whose names begin
with <code class="inline"><span class="w">In</span></code>
, and it's now clear that it's only luck that's so far
prevented a conflict.  Using <code class="inline"><span class="w">In</span></code>
 is only marginally less typing than
<code class="inline"><span class="j">Blk:</span></code>
, and the latter's meaning is clearer anyway, and guaranteed to
never conflict.  So don't take chances.  Use <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Blk</span>=<span class="w">foo</span><span class="s">}</span></code>
 for new
code.  And be sure that block is what you really really want to do.  In
most cases scripts are what you want instead.</p>
<p>A complete list of blocks is in <a href="perluniprops.html">perluniprops</a>.</p>
<a name="*Other-Properties*"></a><h3><b>Other Properties</b></h3>
<p>There are many more properties than the very basic ones described here.
A complete list is in <a href="perluniprops.html">perluniprops</a>.</p>
<p>Unicode defines all its properties in the compound form, so all single-form
properties are Perl extensions.  Most of these are just synonyms for the
Unicode ones, but some are genuine extensions, including several that are in
the compound form.  And quite a few of these are actually recommended by Unicode
(in <a href="http://www.unicode.org/reports/tr18">http://www.unicode.org/reports/tr18</a>).</p>
<p>This section gives some details on all extensions that aren't just
synonyms for compound-form Unicode properties
(for those properties, you'll have to refer to the
<a href="http://search.cpan.org/perldoc/http:#%2fwww.unicode.org%2freports%2ftr44">Unicode Standard</a>.</p>
<ul>
<li><a name="*%5cp%7bAll%7d*"></a><b><b><code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">All</span><span class="s">}</span></code>
</b></b>
<p>This matches every possible code point.  It is equivalent to <code class="inline"><a class="l_k" href="functions/qr.html">qr/./s</a></code>.
Unlike all the other non-user-defined <code class="inline">\<span class="w">p</span><span class="s">{</span><span class="s">}</span></code>
 property matches, no
warning is ever generated if this is property is matched against a
non-Unicode code point (see <a href="#Beyond-Unicode-code-points">Beyond Unicode code points</a> below).</p>
</li>
<li><a name="*%5cp%7bAlnum%7d*"></a><b><b><code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Alnum</span><span class="s">}</span></code>
</b></b>
<p>This matches any <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Alphabetic</span><span class="s">}</span></code>
 or <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Decimal_Number</span><span class="s">}</span></code>
 character.</p>
</li>
<li><a name="*%5cp%7bAny%7d*"></a><b><b><code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Any</span><span class="s">}</span></code>
</b></b>
<p>This matches any of the 1_114_112 Unicode code points.  It is a synonym
for <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Unicode</span><span class="s">}</span></code>
.</p>
</li>
<li><a name="*%5cp%7bASCII%7d*"></a><b><b><code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">ASCII</span><span class="s">}</span></code>
</b></b>
<p>This matches any of the 128 characters in the US-ASCII character set,
which is a subset of Unicode.</p>
</li>
<li><a name="*%5cp%7bAssigned%7d*"></a><b><b><code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Assigned</span><span class="s">}</span></code>
</b></b>
<p>This matches any assigned code point; that is, any code point whose <a href="#General_Category">general category</a> is not <code class="inline"><span class="w">Unassigned</span></code>
 (or equivalently, not <code class="inline"><span class="w">Cn</span></code>
).</p>
</li>
<li><a name="*%5cp%7bBlank%7d*"></a><b><b><code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Blank</span><span class="s">}</span></code>
</b></b>
<p>This is the same as <code class="inline">\<span class="w">h</span></code>
 and <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">HorizSpace</span><span class="s">}</span></code>
:  A character that changes the
spacing horizontally.</p>
</li>
<li><a name="*%5cp%7bDecomposition_Type%3a-Non_Canonical%7d*----(Short%3a-%5cp%7bDt%3dNonCanon%7d)"></a><b><b><code class="inline">\<span class="i">p</span><span class="s">{</span><span class="j">Decomposition_Type:</span> <span class="w">Non_Canonical</span><span class="s">}</span></code>
</b>    (Short: <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Dt</span>=<span class="w">NonCanon</span><span class="s">}</span></code>
)</b>
<p>Matches a character that has a non-canonical decomposition.</p>
<p>The <a href="#Extended-Grapheme-Clusters-(Logical-characters)">Extended Grapheme Clusters (Logical characters)</a> section above
talked about canonical decompositions.  However, many more characters
have a different type of decomposition, a "compatible" or
"non-canonical" decomposition.  The sequences that form these
decompositions are not considered canonically equivalent to the
pre-composed character.  An example is the <code class="inline"><span class="q">&quot;SUPERSCRIPT ONE&quot;</span></code>
.  It is
somewhat like a regular digit 1, but not exactly; its decomposition into
the digit 1 is called a "compatible" decomposition, specifically a
"super" decomposition.  There are several such compatibility
decompositions (see <a href="http://www.unicode.org/reports/tr44">http://www.unicode.org/reports/tr44</a>), including
one called "compat", which means some miscellaneous type of
decomposition that doesn't fit into the other decomposition categories
that Unicode has chosen.</p>
<p>Note that most Unicode characters don't have a decomposition, so their
decomposition type is <code class="inline"><span class="q">&quot;None&quot;</span></code>
.</p>
<p>For your convenience, Perl has added the <code class="inline"><span class="w">Non_Canonical</span></code>
 decomposition
type to mean any of the several compatibility decompositions.</p>
</li>
<li><a name="*%5cp%7bGraph%7d*"></a><b><b><code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Graph</span><span class="s">}</span></code>
</b></b>
<p>Matches any character that is graphic.  Theoretically, this means a character
that on a printer would cause ink to be used.</p>
</li>
<li><a name="*%5cp%7bHorizSpace%7d*"></a><b><b><code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">HorizSpace</span><span class="s">}</span></code>
</b></b>
<p>This is the same as <code class="inline">\<span class="w">h</span></code>
 and <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Blank</span><span class="s">}</span></code>
:  a character that changes the
spacing horizontally.</p>
</li>
<li><a name="*%5cp%7bIn%3d*%7d*"></a><b><b><code class="inline">\p{In=*}</code></b></b>
<p>This is a synonym for <code class="inline">\p{Present_In=*}</code></p>
</li>
<li><a name="*%5cp%7bPerlSpace%7d*"></a><b><b><code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">PerlSpace</span><span class="s">}</span></code>
</b></b>
<p>This is the same as <code class="inline">\s</code>, restricted to ASCII, namely <code class="inline"><span class="s">[</span> \<span class="w">f</span>\<span class="w">n</span>\<span class="w">r</span>\<span class="w">t</span><span class="s">]</span></code>

and starting in Perl v5.18, a vertical tab.</p>
<p>Mnemonic: Perl's (original) space</p>
</li>
<li><a name="*%5cp%7bPerlWord%7d*"></a><b><b><code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">PerlWord</span><span class="s">}</span></code>
</b></b>
<p>This is the same as <code class="inline">\<span class="w">w</span></code>
, restricted to ASCII, namely <code class="inline"><span class="s">[</span><span class="w">A</span>-<span class="w">Za</span>-<span class="w">z0</span>-<span class="n">9_</span><span class="s">]</span></code>
</p>
<p>Mnemonic: Perl's (original) word.</p>
</li>
<li><a name="*%5cp%7bPosix...%7d*"></a><b><b><code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Posix</span>...<span class="s">}</span></code>
</b></b>
<p>There are several of these, which are equivalents, using the <code class="inline">\<span class="w">p</span><span class="s">{</span><span class="s">}</span></code>

notation, for Posix classes and are described in
<a href="perlrecharclass.html#POSIX-Character-Classes">POSIX Character Classes in perlrecharclass</a>.</p>
</li>
<li><a name="*%5cp%7bPresent_In%3a-*%7d*----(Short%3a-%5cp%7bIn%3d*%7d)"></a><b><b><code class="inline">\p{Present_In: *}</code></b>    (Short: <code class="inline">\p{In=*}</code>)</b>
<p>This property is used when you need to know in what Unicode version(s) a
character is.</p>
<p>The "*" above stands for some two digit Unicode version number, such as
<code class="inline"><span class="n">1.1</span></code>
 or <code class="inline"><span class="n">4.0</span></code>
; or the "*" can also be <code class="inline"><span class="w">Unassigned</span></code>
.  This property will
match the code points whose final disposition has been settled as of the
Unicode release given by the version number; <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="j">Present_In:</span> <span class="w">Unassigned</span><span class="s">}</span></code>

will match those code points whose meaning has yet to be assigned.</p>
<p>For example, <code class="inline"><span class="w">U</span>+<span class="n">0041</span></code>
 <code class="inline"><span class="q">&quot;LATIN CAPITAL LETTER A&quot;</span></code>
 was present in the very first
Unicode release available, which is <code class="inline"><span class="n">1.1</span></code>
, so this property is true for all
valid "*" versions.  On the other hand, <code class="inline">U+1EFF</code> was not assigned until version
5.1 when it became <code class="inline"><span class="q">&quot;LATIN SMALL LETTER Y WITH LOOP&quot;</span></code>
, so the only "*" that
would match it are 5.1, 5.2, and later.</p>
<p>Unicode furnishes the <code class="inline"><span class="w">Age</span></code>
 property from which this is derived.  The problem
with Age is that a strict interpretation of it (which Perl takes) has it
matching the precise release a code point's meaning is introduced in.  Thus
<code class="inline"><span class="w">U</span>+<span class="n">0041</span></code>
 would match only 1.1; and <code class="inline">U+1EFF</code> only 5.1.  This is not usually what
you want.</p>
<p>Some non-Perl implementations of the Age property may change its meaning to be
the same as the Perl <code class="inline"><span class="w">Present_In</span></code>
 property; just be aware of that.</p>
<p>Another confusion with both these properties is that the definition is not
that the code point has been <i>assigned</i>, but that the meaning of the code point
has been <i>determined</i>.  This is because 66 code points will always be
unassigned, and so the <code class="inline"><span class="w">Age</span></code>
 for them is the Unicode version in which the decision
to make them so was made.  For example, <code class="inline"><span class="w">U</span>+<span class="w">FDD0</span></code>
 is to be permanently
unassigned to a character, and the decision to do that was made in version 3.1,
so <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Age</span>=<span class="n">3.1</span><span class="s">}</span></code>
 matches this character, as also does <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="j">Present_In:</span> <span class="n">3.1</span><span class="s">}</span></code>
 and up.</p>
</li>
<li><a name="*%5cp%7bPrint%7d*"></a><b><b><code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Print</span><span class="s">}</span></code>
</b></b>
<p>This matches any character that is graphical or blank, except controls.</p>
</li>
<li><a name="*%5cp%7bSpacePerl%7d*"></a><b><b><code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">SpacePerl</span><span class="s">}</span></code>
</b></b>
<p>This is the same as <code class="inline">\s</code>, including beyond ASCII.</p>
<p>Mnemonic: Space, as modified by Perl.  (It doesn't include the vertical tab
until v5.18, which both the Posix standard and Unicode consider white space.)</p>
</li>
<li><a name="*%5cp%7bTitle%7d*-and--*%5cp%7bTitlecase%7d*"></a><b><b><code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Title</span><span class="s">}</span></code>
</b> and  <b><code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Titlecase</span><span class="s">}</span></code>
</b></b>
<p>Under case-sensitive matching, these both match the same code points as
<code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">General</span> <span class="w">Category</span>=<span class="w">Titlecase_Letter</span><span class="s">}</span></code>
 (<code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">gc</span>=<a class="l_k" href="functions/lt.html">lt</a><span class="s">}</span></code>
).  The difference
is that under <code class="inline">/i</code> caseless matching, these match the same as
<code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Cased</span><span class="s">}</span></code>
, whereas <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">gc</span>=<a class="l_k" href="functions/lt.html">lt</a><span class="s">}</span></code>
 matches <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Cased_Letter</span></code>
).</p>
</li>
<li><a name="*%5cp%7bUnicode%7d*"></a><b><b><code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Unicode</span><span class="s">}</span></code>
</b></b>
<p>This matches any of the 1_114_112 Unicode code points.
<code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Any</span><span class="s">}</span></code>
.</p>
</li>
<li><a name="*%5cp%7bVertSpace%7d*"></a><b><b><code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">VertSpace</span><span class="s">}</span></code>
</b></b>
<p>This is the same as <code class="inline">\<span class="w">v</span></code>
:  A character that changes the spacing vertically.</p>
</li>
<li><a name="*%5cp%7bWord%7d*"></a><b><b><code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Word</span><span class="s">}</span></code>
</b></b>
<p>This is the same as <code class="inline">\<span class="w">w</span></code>
, including over 100_000 characters beyond ASCII.</p>
</li>
<li><a name="*%5cp%7bXPosix...%7d*"></a><b><b><code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">XPosix</span>...<span class="s">}</span></code>
</b></b>
<p>There are several of these, which are the standard Posix classes
extended to the full Unicode range.  They are described in
<a href="perlrecharclass.html#POSIX-Character-Classes">POSIX Character Classes in perlrecharclass</a>.</p>
</li>
</ul>
<a name="User-Defined-Character-Properties"></a><h2>User-Defined Character Properties</h2>
<p>You can define your own binary character properties by defining subroutines
whose names begin with <code class="inline"><span class="q">&quot;In&quot;</span></code>
 or <code class="inline"><span class="q">&quot;Is&quot;</span></code>
.  (The experimental feature
<a href="perlre.html#(%3f%5b-%5d)">(?[ ]) in perlre</a> provides an alternative which allows more complex
definitions.)  The subroutines can be defined in any
package.  The user-defined properties can be used in the regular expression
<code class="inline">\<span class="w">p</span><span class="s">{</span><span class="s">}</span></code>
 and <code class="inline">\<span class="w">P</span><span class="s">{</span><span class="s">}</span></code>
 constructs; if you are using a user-defined property from a
package other than the one you are in, you must specify its package in the
<code class="inline">\<span class="w">p</span><span class="s">{</span><span class="s">}</span></code>
 or <code class="inline">\<span class="w">P</span><span class="s">{</span><span class="s">}</span></code>
 construct.</p>
<pre class="verbatim"><ol><li>    <span class="c"># assuming property Is_Foreign defined in Lang::</span></li><li><a name="package-main"></a>    package <span class="i">main</span><span class="sc">;</span>  <span class="c"># property package name required</span></li><li>    if <span class="s">(</span><span class="i">$txt</span> =~ <span class="q">/\p{Lang::IsForeign}+/</span><span class="s">)</span> <span class="s">{</span> ... <span class="s">}</span></li><li></li><li><a name="package-Lang"></a>    package <span class="i">Lang</span><span class="sc">;</span>  <span class="c"># property package name not required</span></li><li>    if <span class="s">(</span><span class="i">$txt</span> =~ <span class="q">/\p{IsForeign}+/</span><span class="s">)</span> <span class="s">{</span> ... <span class="s">}</span></li></ol></pre><p>Note that the effect is compile-time and immutable once defined.
However, the subroutines are passed a single parameter, which is 0 if
case-sensitive matching is in effect and non-zero if caseless matching
is in effect.  The subroutine may return different values depending on
the value of the flag, and one set of values will immutably be in effect
for all case-sensitive matches, and the other set for all case-insensitive
matches.</p>
<p>Note that if the regular expression is tainted, then Perl will die rather
than calling the subroutine when the name of the subroutine is
determined by the tainted data.</p>
<p>The subroutines must return a specially-formatted string, with one
or more newline-separated lines.  Each line must be one of the following:</p>
<ul>
<li>
<p>A single hexadecimal number denoting a code point to include.</p>
</li>
<li>
<p>Two hexadecimal numbers separated by horizontal whitespace (space or
tabular characters) denoting a range of code points to include.</p>
</li>
<li>
<p>Something to include, prefixed by <code class="inline"><span class="q">&quot;+&quot;</span></code>
: a built-in character
property (prefixed by <code class="inline"><span class="q">&quot;utf8::&quot;</span></code>
) or a fully qualified (including package
name) user-defined character property,
to represent all the characters in that property; two hexadecimal code
points for a range; or a single hexadecimal code point.</p>
</li>
<li>
<p>Something to exclude, prefixed by <code class="inline"><span class="q">&quot;-&quot;</span></code>
: an existing character
property (prefixed by <code class="inline"><span class="q">&quot;utf8::&quot;</span></code>
) or a fully qualified (including package
name) user-defined character property,
to represent all the characters in that property; two hexadecimal code
points for a range; or a single hexadecimal code point.</p>
</li>
<li>
<p>Something to negate, prefixed <code class="inline"><span class="q">&quot;!&quot;</span></code>
: an existing character
property (prefixed by <code class="inline"><span class="q">&quot;utf8::&quot;</span></code>
) or a fully qualified (including package
name) user-defined character property,
to represent all the characters in that property; two hexadecimal code
points for a range; or a single hexadecimal code point.</p>
</li>
<li>
<p>Something to intersect with, prefixed by <code class="inline"><span class="q">&quot;&amp;&quot;</span></code>
: an existing character
property (prefixed by <code class="inline"><span class="q">&quot;utf8::&quot;</span></code>
) or a fully qualified (including package
name) user-defined character property,
for all the characters except the characters in the property; two
hexadecimal code points for a range; or a single hexadecimal code point.</p>
</li>
</ul>
<p>For example, to define a property that covers both the Japanese
syllabaries (hiragana and katakana), you can define</p>
<pre class="verbatim"><ol><li><a name="InKana"></a>    sub <span class="m">InKana</span> <span class="s">{</span></li><li>        <a class="l_k" href="functions/return.html">return</a> <span class="h">&lt;&lt;END</span><span class="sc">;</span></li><li><span class="hh">    3040\t309F</span></li><li><span class="hh">    30A0\t30FF</span></li><li><span class="hh">    END</span></li><li><span class="hh">    }</span></li></ol></pre><p>Imagine that the here-doc end marker is at the beginning of the line.
Now you can use <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">InKana</span><span class="s">}</span></code>
 and <code class="inline">\<span class="i">P</span><span class="s">{</span><span class="w">InKana</span><span class="s">}</span></code>
.</p>
<p>You could also have used the existing block property names:</p>
<pre class="verbatim"><ol><li><a name="InKana"></a>    sub <span class="m">InKana</span> <span class="s">{</span></li><li>        <a class="l_k" href="functions/return.html">return</a> <span class="h">&lt;&lt;&#39;END&#39;</span><span class="sc">;</span></li><li><span class="hh">    +utf8::InHiragana</span></li><li><span class="hh">    +utf8::InKatakana</span></li><li><span class="hh">    END</span></li><li><span class="hh">    }</span></li></ol></pre><p>Suppose you wanted to match only the allocated characters,
not the raw block ranges: in other words, you want to remove
the unassigned characters:</p>
<pre class="verbatim"><ol><li><a name="InKana"></a>    sub <span class="m">InKana</span> <span class="s">{</span></li><li>        <a class="l_k" href="functions/return.html">return</a> <span class="h">&lt;&lt;&#39;END&#39;</span><span class="sc">;</span></li><li><span class="hh">    +utf8::InHiragana</span></li><li><span class="hh">    +utf8::InKatakana</span></li><li><span class="hh">    -utf8::IsCn</span></li><li><span class="hh">    END</span></li><li><span class="hh">    }</span></li></ol></pre><p>The negation is useful for defining (surprise!) negated classes.</p>
<pre class="verbatim"><ol><li><a name="InNotKana"></a>    sub <span class="m">InNotKana</span> <span class="s">{</span></li><li>        <a class="l_k" href="functions/return.html">return</a> <span class="h">&lt;&lt;&#39;END&#39;</span><span class="sc">;</span></li><li><span class="hh">    !utf8::InHiragana</span></li><li><span class="hh">    -utf8::InKatakana</span></li><li><span class="hh">    +utf8::IsCn</span></li><li><span class="hh">    END</span></li><li><span class="hh">    }</span></li></ol></pre><p>This will match all non-Unicode code points, since every one of them is
not in Kana.  You can use intersection to exclude these, if desired, as
this modified example shows:</p>
<pre class="verbatim"><ol><li><a name="InNotKana"></a>    sub <span class="m">InNotKana</span> <span class="s">{</span></li><li>        <a class="l_k" href="functions/return.html">return</a> <span class="h">&lt;&lt;&#39;END&#39;</span><span class="sc">;</span></li><li><span class="hh">    !utf8::InHiragana</span></li><li><span class="hh">    -utf8::InKatakana</span></li><li><span class="hh">    +utf8::IsCn</span></li><li><span class="hh">    &amp;utf8::Any</span></li><li><span class="hh">    END</span></li><li><span class="hh">    }</span></li></ol></pre><p><code class="inline"><span class="i">&amp;utf8::Any</span></code>
 must be the last line in the definition.</p>
<p>Intersection is used generally for getting the common characters matched
by two (or more) classes.  It's important to remember not to use <code class="inline"><span class="q">&quot;&amp;&quot;</span></code>
 for
the first set; that would be intersecting with nothing, resulting in an
empty set.</p>
<p>Unlike non-user-defined <code class="inline">\<span class="w">p</span><span class="s">{</span><span class="s">}</span></code>
 property matches, no warning is ever
generated if these properties are matched against a non-Unicode code
point (see <a href="#Beyond-Unicode-code-points">Beyond Unicode code points</a> below).</p>
<a name="User-Defined-Case-Mappings-(for-serious-hackers-only)"></a><h2>User-Defined Case Mappings (for serious hackers only)</h2>
<p><b>This feature has been removed as of Perl 5.16.</b>
The CPAN module <code class="inline"><a href="http://search.cpan.org/perldoc/Unicode::Casing">Unicode::Casing</a></code> provides better functionality without
the drawbacks that this feature had.  If you are using a Perl earlier
than 5.16, this feature was most fully documented in the 5.14 version of
this pod:
<a href="http://perldoc.perl.org/5.14.0/perlunicode.html#User-Defined-Case-Mappings-%28for-serious-hackers-only%29">http://perldoc.perl.org/5.14.0/perlunicode.html#User-Defined-Case-Mappings-%28for-serious-hackers-only%29</a></p>
<a name="Character-Encodings-for-Input-and-Output"></a><h2>Character Encodings for Input and Output</h2>
<p>See <a href="Encode.html">Encode</a>.</p>
<a name="Unicode-Regular-Expression-Support-Level"></a><h2>Unicode Regular Expression Support Level</h2>
<p>The following list of Unicode supported features for regular expressions describes
all features currently directly supported by core Perl.  The references
to "Level <i>N</i>" and the section numbers refer to
<a href="http://search.cpan.org/perldoc/http:#%2fwww.unicode.org%2freports%2ftr18">UTS#18 Unicode Regular Expressions</a>,
version 13, November 2013.</p>
<a name="Level-1---Basic-Unicode-Support"></a><h3>Level 1 - Basic Unicode Support</h3>
<pre class="verbatim"><ol><li> <span class="w">RL1</span><span class="n">.1</span>   <span class="w">Hex</span> <span class="w">Notation</span>                     - <span class="w">Done</span>          <span class="s">[</span><span class="n">1</span><span class="s">]</span></li><li> <span class="w">RL1</span><span class="n">.2</span>   <span class="w">Properties</span>                       - <span class="w">Done</span>          <span class="s">[</span><span class="n">2</span><span class="s">]</span></li><li> <span class="w">RL1</span><span class="n">.2</span><span class="w">a</span>  <span class="w">Compatibility</span> <span class="w">Properties</span>         - <span class="w">Done</span>          <span class="s">[</span><span class="n">3</span><span class="s">]</span></li><li> <span class="w">RL1</span><span class="n">.3</span>   <span class="w">Subtraction</span> <a class="l_k" href="functions/and.html">and</a> <span class="w">Intersection</span>     - <span class="w">Experimental</span>  <span class="s">[</span><span class="n">4</span><span class="s">]</span></li><li> <span class="w">RL1</span><span class="n">.4</span>   <span class="w">Simple</span> <span class="w">Word</span> <span class="w">Boundaries</span>           - <span class="w">Done</span>          <span class="s">[</span><span class="n">5</span><span class="s">]</span></li><li> <span class="w">RL1</span><span class="n">.5</span>   <span class="w">Simple</span> <span class="w">Loose</span> <span class="w">Matches</span>             - <span class="w">Done</span>          <span class="s">[</span><span class="n">6</span><span class="s">]</span></li><li> <span class="w">RL1</span><span class="n">.6</span>   <span class="w">Line</span> <span class="w">Boundaries</span>                  - <span class="w">Partial</span>       <span class="s">[</span><span class="n">7</span><span class="s">]</span></li><li> <span class="w">RL1</span><span class="n">.7</span>   <span class="w">Supplementary</span> <span class="w">Code</span> <span class="w">Points</span>        - <span class="w">Done</span>          <span class="s">[</span><span class="n">8</span><span class="s">]</span></li></ol></pre><ul>
<li><a name="%5b1%5d-%5cN%7bU%2b...%7d-and-%5cx%7b...%7d"></a><b>[1] <code class="inline">\<span class="i">N</span><span class="s">{</span><span class="w">U</span>+...<span class="s">}</span></code>
 and <code class="inline">\<span class="i">x</span><span class="s">{</span>...<span class="s">}</span></code>
</b>
</li>
<li><a name="%5b2%5d-%5cp%7b...%7d-%5cP%7b...%7d.--This-requirement-is-for-a-minimal-list-of-properties.--Perl-supports-these-and-all-other-Unicode-character-properties%2c-as-R2.7-asks-(see-the-%2f%22Unicode-Character-Properties%22-manpage-above)."></a><b>[2]
<code class="inline">\<span class="i">p</span><span class="s">{</span>...<span class="s">}</span></code>
 <code class="inline">\<span class="i">P</span><span class="s">{</span>...<span class="s">}</span></code>
.  This requirement is for a minimal list of
properties.  Perl supports these and all other Unicode character
properties, as R2.7 asks (see <a href="#Unicode-Character-Properties">Unicode Character Properties</a> above).</b>
</li>
<li><a name="%5b3%5d-Perl-has-%5cd-%5cD-%5cs-%5cS-%5cw-%5cW-%5cX-%5b%3a_prop_%3a%5d-%5b%3a%5e_prop_%3a%5d%2c-plus-all-the-properties-specified-by-the-http%3a%2f%2fwww.unicode.org%2freports%2ftr18%2f%23Compatibility_Properties-manpage.--These-are-described-above-in-the-%2fOther-Properties-manpage"></a><b>[3]
Perl has <code class="inline">\<span class="w">d</span></code>
 <code class="inline">\<span class="w">D</span></code>
 <code class="inline">\s</code> <code class="inline">\<span class="w">S</span></code>
 <code class="inline">\<span class="w">w</span></code>
 <code class="inline">\<span class="w">W</span></code>
 <code class="inline">\<span class="w">X</span></code>
 <code class="inline">[:<i>prop</i>:]</code>
<code class="inline">[:^<i>prop</i>:]</code>, plus all the properties specified by
<a href="http://www.unicode.org/reports/tr18/#Compatibility_Properties">http://www.unicode.org/reports/tr18/#Compatibility_Properties</a>.  These
are described above in <a href="#Other-Properties">Other Properties</a></b>
</li>
<li><a name="%5b4%5d"></a><b>[4]</b>
<p>The experimental feature <code class="inline"><span class="q">&quot;(?[...])&quot;</span></code>
 starting in v5.18 accomplishes
this.</p>
<p>See <a href="perlre.html#(%3f%5b-%5d)">(?[ ]) in perlre</a>.  If you don't want to use an experimental
feature, you can use one of the following:</p>
<ul>
<li><a name="*-Regular-expression-lookahead"></a><b>Regular expression lookahead</b>
<p>You can mimic class subtraction using lookahead.
For example, what UTS#18 might write as</p>
<pre class="verbatim"><ol><li>    <span class="s">[</span><span class="s">{</span><span class="w">Block</span>=<span class="w">Greek</span><span class="s">}</span>-<span class="s">[</span><span class="s">{</span><span class="w">UNASSIGNED</span><span class="s">}</span><span class="s">]</span><span class="s">]</span></li></ol></pre><p>in Perl can be written as:</p>
<pre class="verbatim"><ol><li>    <span class="s">(</span><span class="q">?!\p{Unassigned})\p{Block=Greek}</span></li><li>    <span class="q">    (?</span>=\<span class="i">p</span><span class="s">{</span><span class="w">Assigned</span><span class="s">}</span><span class="s">)</span>\<span class="i">p</span><span class="s">{</span><span class="w">Block</span>=<span class="w">Greek</span><span class="s">}</span></li></ol></pre><p>But in this particular example, you probably really want</p>
<pre class="verbatim"><ol><li>    \<span class="i">p</span><span class="s">{</span><span class="w">Greek</span><span class="s">}</span></li></ol></pre><p>which will match assigned characters known to be part of the Greek script.</p>
</li>
<li>
<p>CPAN module <code class="inline"><a href="http://search.cpan.org/perldoc/Unicode::Regex::Set">Unicode::Regex::Set</a></code></p>
<p>It does implement the full UTS#18 grouping, intersection, union, and
removal (subtraction) syntax.</p>
</li>
<li>
<p><a href="#User-Defined-Character-Properties">User-Defined Character Properties</a></p>
<p><code class="inline"><span class="q">&quot;+&quot;</span></code>
 for union, <code class="inline"><span class="q">&quot;-&quot;</span></code>
 for removal (set-difference), <code class="inline"><span class="q">&quot;&amp;&quot;</span></code>
 for intersection</p>
</li>
</ul>
</li>
<li><a name="%5b5%5d-%5cb-%5cB-meet-most%2c-but-not-all%2c-the-details-of-this-requirement%2c-but-%5cb%7bwb%7d-and-%5cB%7bwb%7d-do%2c-as-well-as-the-stricter-R2.3."></a><b>[5]
<code class="inline">\<span class="w">b</span></code>
 <code class="inline">\<span class="w">B</span></code>
 meet most, but not all, the details of this requirement, but
<code class="inline">\<span class="i">b</span><span class="s">{</span><span class="w">wb</span><span class="s">}</span></code>
 and <code class="inline">\<span class="i">B</span><span class="s">{</span><span class="w">wb</span><span class="s">}</span></code>
 do, as well as the stricter R2.3.</b>
</li>
<li><a name="%5b6%5d"></a><b>[6]</b>
<p>Note that Perl does Full case-folding in matching, not Simple:</p>
<p>For example <code class="inline"><span class="w">U</span>+<span class="n">1</span><span class="w">F88</span></code>
 is equivalent to <code class="inline"><span class="w">U</span>+<span class="n">1</span><span class="w">F00</span> <span class="w">U</span>+<span class="n">03</span><span class="w">B9</span></code>
, instead of just
<code class="inline"><span class="w">U</span>+<span class="n">1</span><span class="w">F80</span></code>
.  This difference matters mainly for certain Greek capital
letters with certain modifiers: the Full case-folding decomposes the
letter, while the Simple case-folding would map it to a single
character.</p>
</li>
<li><a name="%5b7%5d"></a><b>[7]</b>
<p>The reason this is considered to be only partially implemented is that
Perl has <a href="perlrebackslash.html#%5cb%7blb%7d">qr/\b{lb}/</a> and
<code class="inline"><a href="http://search.cpan.org/perldoc/Unicode::LineBreak">Unicode::LineBreak</a></code> that are conformant with
<a href="http://search.cpan.org/perldoc/http:#%2fwww.unicode.org%2freports%2ftr14">UAX#14 Unicode Line Breaking Algorithm</a>.
The regular expression construct provides default behavior, while the
heavier-weight module provides customizable line breaking.</p>
<p>But Perl treats <code class="inline">\<span class="w">n</span></code>
 as the start- and end-line
delimiter, whereas Unicode specifies more characters that should be
so-interpreted.</p>
<p>These are:</p>
<pre class="verbatim"><ol><li> <span class="w">VT</span>   <span class="w">U</span>+<span class="n">000</span><span class="w">B</span>  <span class="s">(</span>\<span class="w">v</span> <span class="w">in</span> <span class="w">C</span><span class="s">)</span></li><li> <span class="w">FF</span>   <span class="w">U</span>+<span class="n">000</span><span class="w">C</span>  <span class="s">(</span>\<span class="w">f</span><span class="s">)</span></li><li> <span class="w">CR</span>   <span class="w">U</span>+<span class="n">000</span><span class="w">D</span>  <span class="s">(</span>\<span class="w">r</span><span class="s">)</span></li><li> <span class="w">NEL</span>  <span class="w">U</span>+<span class="n">0085</span></li><li> <span class="w">LS</span>   <span class="w">U</span>+<span class="n">2028</span></li><li> <span class="w">PS</span>   <span class="w">U</span>+<span class="n">2029</span></li></ol></pre><p><code class="inline">^</code> and <code class="inline"><span class="i">$</span></code>
 in regular expression patterns are supposed to match all
these, but don't.
These characters also don't, but should, affect <code class="inline">&lt;&gt;</code>
 <code class="inline"><span class="i">$.</span></code>
, and
script line numbers.</p>
<p>Also, lines should not be split within <code class="inline"><span class="w">CRLF</span></code>
 (i.e. there is no
empty line between <code class="inline">\<span class="w">r</span></code>
 and <code class="inline">\<span class="w">n</span></code>
).  For <code class="inline"><span class="w">CRLF</span></code>
, try the <code class="inline"><span class="j">:</span><span class="w">crlf</span></code>

layer (see <a href="PerlIO.html">PerlIO</a>).</p>
</li>
<li><a name="%5b8%5d-UTF-8%2fUTF-EBDDIC-used-in-Perl-allows-not-only-U%2b10000-to-U%2b10FFFF-but-also-beyond-U%2b10FFFF"></a><b>[8]
UTF-8/UTF-EBDDIC used in Perl allows not only <code class="inline"><span class="w">U</span>+<span class="n">10000</span></code>
 to
<code class="inline"><span class="w">U</span>+<span class="n">10</span><span class="w">FFFF</span></code>
 but also beyond <code class="inline"><span class="w">U</span>+<span class="n">10</span><span class="w">FFFF</span></code>
</b>
</li>
</ul>
<a name="Level-2---Extended-Unicode-Support"></a><h3>Level 2 - Extended Unicode Support</h3>
<pre class="verbatim"><ol><li> <span class="w">RL2</span><span class="n">.1</span>   <span class="w">Canonical</span> <span class="w">Equivalents</span>           - <span class="w">Retracted</span>     <span class="s">[</span><span class="n">9</span><span class="s">]</span></li><li>                                           <span class="w">by</span> <span class="w">Unicode</span></li><li> <span class="w">RL2</span><span class="n">.2</span>   <span class="w">Extended</span> <span class="w">Grapheme</span> <span class="w">Clusters</span>      - <span class="w">Partial</span>       <span class="s">[</span><span class="n">10</span><span class="s">]</span></li><li> <span class="w">RL2</span><span class="n">.3</span>   <span class="w">Default</span> <span class="w">Word</span> <span class="w">Boundaries</span>         - <span class="w">Done</span>          <span class="s">[</span><span class="n">11</span><span class="s">]</span></li><li> <span class="w">RL2</span><span class="n">.4</span>   <span class="w">Default</span> <span class="w">Case</span> <span class="w">Conversion</span>         - <span class="w">Done</span></li><li> <span class="w">RL2</span><span class="n">.5</span>   <span class="w">Name</span> <span class="w">Properties</span>                 - <span class="w">Done</span></li><li> <span class="w">RL2</span><span class="n">.6</span>   <span class="w">Wildcard</span> <span class="w">Properties</span>             - <span class="w">Missing</span></li><li> <span class="w">RL2</span><span class="n">.7</span>   <span class="w">Full</span> <span class="w">Properties</span>                 - <span class="w">Done</span></li></ol></pre><ul>
<li><a name="%5b9%5d-Unicode-has-rewritten-this-portion-of-UTS%2318-to-say-that-getting-canonical-equivalence-(see-UAX%2315-http%3a%2f%2fwww.unicode.org%2freports%2ftr15)-is-basically-to-be-done-at-the-programmer-level.--Use-NFD-to-write-both-your-regular-expressions-and-text-to-match-them-against-(you-can-use-the-Unicode%3a%3aNormalize-manpage)."></a><b>[9]
Unicode has rewritten this portion of UTS#18 to say that getting
canonical equivalence (see UAX#15
<a href="http://search.cpan.org/perldoc/http:#%2fwww.unicode.org%2freports%2ftr15">Unicode Normalization Forms</a>)
is basically to be done at the programmer level.  Use NFD to write
both your regular expressions and text to match them against (you
can use <a href="Unicode/Normalize.html">Unicode::Normalize</a>).</b>
</li>
<li><a name="%5b10%5d-Perl-has-%5cX-and-%5cb%7bgcb%7d-but-we-don't-have-a-%22Grapheme-Cluster-Mode%22."></a><b>[10]
Perl has <code class="inline">\<span class="w">X</span></code>
 and <code class="inline">\<span class="i">b</span><span class="s">{</span><span class="w">gcb</span><span class="s">}</span></code>
 but we don't have a "Grapheme Cluster Mode".</b>
</li>
<li><a name="%5b11%5d-see-http%3a%2f%2fwww.unicode.org%2freports%2ftr29%2c"></a><b>[11] see
<a href="http://search.cpan.org/perldoc/http:#%2fwww.unicode.org%2freports%2ftr29">UAX#29 Unicode Text Segmentation</a>,</b>
</li>
</ul>
<a name="Level-3---Tailored-Support"></a><h3>Level 3 - Tailored Support</h3>
<pre class="verbatim"><ol><li> <span class="w">RL3</span><span class="n">.1</span>   <span class="w">Tailored</span> <span class="w">Punctuation</span>            - <span class="w">Missing</span></li><li> <span class="w">RL3</span><span class="n">.2</span>   <span class="w">Tailored</span> <span class="w">Grapheme</span> <span class="w">Clusters</span>      - <span class="w">Missing</span>       <span class="s">[</span><span class="n">12</span><span class="s">]</span></li><li> <span class="w">RL3</span><span class="n">.3</span>   <span class="w">Tailored</span> <span class="w">Word</span> <span class="w">Boundaries</span>        - <span class="w">Missing</span></li><li> <span class="w">RL3</span><span class="n">.4</span>   <span class="w">Tailored</span> <span class="w">Loose</span> <span class="w">Matches</span>          - <span class="w">Retracted</span> <span class="w">by</span> <span class="w">Unicode</span></li><li> <span class="w">RL3</span><span class="n">.5</span>   <span class="w">Tailored</span> <span class="w">Ranges</span>                 - <span class="w">Retracted</span> <span class="w">by</span> <span class="w">Unicode</span></li><li> <span class="w">RL3</span><span class="n">.6</span>   <span class="w">Context</span> <span class="w">Matching</span>                - <span class="w">Missing</span>       <span class="s">[</span><span class="n">13</span><span class="s">]</span></li><li> <span class="w">RL3</span><span class="n">.7</span>   <span class="w">Incremental</span> <span class="w">Matches</span>             - <span class="w">Missing</span></li><li> <span class="w">RL3</span><span class="n">.8</span>   <span class="w">Unicode</span> <span class="w">Set</span> <span class="w">Sharing</span>             - <span class="w">Unicode</span> <span class="w">is</span> <span class="w">proposing</span></li><li>                                           <span class="w">to</span> <span class="w">retract</span> <span class="w">this</span></li><li> <span class="w">RL3</span><span class="n">.9</span>   <span class="w">Possible</span> <span class="w">Match</span> <span class="w">Sets</span>             - <span class="w">Missing</span></li><li> <span class="w">RL3</span><span class="n">.10</span>  <span class="w">Folded</span> <span class="w">Matching</span>                 - <span class="w">Retracted</span> <span class="w">by</span> <span class="w">Unicode</span></li><li> <span class="w">RL3</span><span class="n">.11</span>  <span class="w">Submatchers</span>                     - <span class="w">Missing</span></li></ol></pre><ul>
<li><a name="%5b12%5d-Perl-has-the-Unicode%3a%3aCollate-manpage%2c-but-it-isn't-integrated-with-regular-expressions.--See-http%3a%2f%2fwww.unicode.org%2freports%2ftr10."></a><b>[12]
Perl has <a href="Unicode/Collate.html">Unicode::Collate</a>, but it isn't integrated with regular
expressions.  See
<a href="http://search.cpan.org/perldoc/http:#%2fwww.unicode.org%2freports%2ftr10">UTS#10 Unicode Collation Algorithms</a>.</b>
</li>
<li><a name="%5b13%5d-Perl-has-(%3f%3c%3dx)-and-(%3f%3dx)%2c-but-lookaheads-or-lookbehinds-should-see-outside-of-the-target-substring"></a><b>[13]
Perl has <code class="inline"><span class="s">(</span><span class="q">?&lt;=x)</span></code>
 and <code class="inline"><span class="s">(</span><span class="q">?=x)</span></code>
, but lookaheads or lookbehinds should
see outside of the target substring</b>
</li>
</ul>
<a name="Unicode-Encodings"></a><h2>Unicode Encodings</h2>
<p>Unicode characters are assigned to <i>code points</i>, which are abstract
numbers.  To use these numbers, various encodings are needed.</p>
<ul>
<li>
<p>UTF-8</p>
<p>UTF-8 is a variable-length (1 to 4 bytes), byte-order independent
encoding.  In most of Perl's documentation, including elsewhere in this
document, the term "UTF-8" means also "UTF-EBCDIC".  But in this section,
"UTF-8" refers only to the encoding used on ASCII platforms.  It is a
superset of 7-bit US-ASCII, so anything encoded in ASCII has the
identical representation when encoded in UTF-8.</p>
<p>The following table is from Unicode 3.2.</p>
<pre class="verbatim"><ol><li> <span class="w">Code</span> <span class="w">Points</span>            <span class="n">1</span><span class="w">st</span> <span class="w">Byte</span>  <span class="n">2</span><span class="w">nd</span> <span class="w">Byte</span>  <span class="n">3</span><span class="w">rd</span> <span class="w">Byte</span> <span class="n">4</span><span class="w">th</span> <span class="w">Byte</span></li><li></li><li>   <span class="w">U</span>+<span class="n">0000</span>..<span class="w">U</span>+<span class="n">007</span><span class="w">F</span>       <span class="n">00</span>..<span class="n">7</span><span class="w">F</span></li><li>   <span class="w">U</span>+<span class="n">0080</span>..<span class="w">U</span>+<span class="n">07</span><span class="w">FF</span>     * <span class="w">C2</span>..<span class="w">DF</span>    <span class="n">80</span>..<span class="w">BF</span></li><li>   <span class="w">U</span>+<span class="n">0800</span>..<span class="w">U</span>+<span class="n">0</span><span class="w">FFF</span>       <span class="w">E0</span>      * <span class="w">A0</span>..<span class="w">BF</span>    <span class="n">80</span>..<span class="w">BF</span></li><li>   <span class="w">U</span>+<span class="n">1000</span>..<span class="w">U</span>+<span class="w">CFFF</span>       <span class="w">E1</span>..<span class="w">EC</span>    <span class="n">80</span>..<span class="w">BF</span>    <span class="n">80</span>..<span class="w">BF</span></li><li>   <span class="w">U</span>+<span class="w">D000</span>..<span class="w">U</span>+<span class="w">D7FF</span>       <span class="w">ED</span>        <span class="n">80</span>..<span class="n">9</span><span class="w">F</span>    <span class="n">80</span>..<span class="w">BF</span></li><li>   <span class="w">U</span>+<span class="w">D800</span>..<span class="w">U</span>+<span class="w">DFFF</span>       +++++ <span class="w">utf16</span> <span class="w">surrogates</span><span class="cm">,</span> <a class="l_k" href="functions/not.html">not</a> <span class="w">legal</span> <span class="w">utf8</span> +++++</li><li>   <span class="w">U</span>+<span class="w">E000</span>..<span class="w">U</span>+<span class="w">FFFF</span>       <span class="w">EE</span>..<span class="w">EF</span>    <span class="n">80</span>..<span class="w">BF</span>    <span class="n">80</span>..<span class="w">BF</span></li><li>  <span class="w">U</span>+<span class="n">10000</span>..<span class="w">U</span>+<span class="n">3</span><span class="w">FFFF</span>      <span class="w">F0</span>      * <span class="n">90</span>..<span class="w">BF</span>    <span class="n">80</span>..<span class="w">BF</span>    <span class="n">80</span>..<span class="w">BF</span></li><li>  <span class="w">U</span>+<span class="n">40000</span>..<span class="w">U</span>+<span class="w">FFFFF</span>      <span class="w">F1</span>..<span class="w">F3</span>    <span class="n">80</span>..<span class="w">BF</span>    <span class="n">80</span>..<span class="w">BF</span>    <span class="n">80</span>..<span class="w">BF</span></li><li> <span class="w">U</span>+<span class="n">100000</span>..<span class="w">U</span>+<span class="n">10</span><span class="w">FFFF</span>     <span class="w">F4</span>        <span class="n">80</span>..<span class="n">8</span><span class="w">F</span>    <span class="n">80</span>..<span class="w">BF</span>    <span class="n">80</span>..<span class="w">BF</span></li></ol></pre><p>Note the gaps marked by "*" before several of the byte entries above.  These are
caused by legal UTF-8 avoiding non-shortest encodings: it is technically
possible to UTF-8-encode a single code point in different ways, but that is
explicitly forbidden, and the shortest possible encoding should always be used
(and that is what Perl does).</p>
<p>Another way to look at it is via bits:</p>
<pre class="verbatim"><ol><li>                <span class="w">Code</span> <span class="w">Points</span>  <span class="n">1</span><span class="w">st</span> <span class="w">Byte</span>  <span class="n">2</span><span class="w">nd</span> <span class="w">Byte</span>  <span class="n">3</span><span class="w">rd</span> <span class="w">Byte</span>  <span class="n">4</span><span class="w">th</span> <span class="w">Byte</span></li><li></li><li>                   <span class="n">0</span><span class="w">aaaaaaa</span>  <span class="n">0</span><span class="w">aaaaaaa</span></li><li>           <span class="n">00000</span><span class="w">bbbbbaaaaaa</span>  <span class="n">110</span><span class="w">bbbbb</span>  <span class="n">10</span><span class="w">aaaaaa</span></li><li>           <span class="w">ccccbbbbbbaaaaaa</span>  <span class="n">1110</span><span class="w">cccc</span>  <span class="n">10</span><span class="w">bbbbbb</span>  <span class="n">10</span><span class="w">aaaaaa</span></li><li> <span class="n">00000</span><span class="w">dddccccccbbbbbbaaaaaa</span>  <span class="n">11110</span><span class="w">ddd</span>  <span class="n">10</span><span class="w">cccccc</span>  <span class="n">10</span><span class="w">bbbbbb</span>  <span class="n">10</span><span class="w">aaaaaa</span></li></ol></pre><p>As you can see, the continuation bytes all begin with <code class="inline"><span class="q">&quot;10&quot;</span></code>
, and the
leading bits of the start byte tell how many bytes there are in the
encoded character.</p>
<p>The original UTF-8 specification allowed up to 6 bytes, to allow
encoding of numbers up to <code class="inline"><span class="n">0x7FFF_FFFF</span></code>
.  Perl continues to allow those,
and has extended that up to 13 bytes to encode code points up to what
can fit in a 64-bit word.  However, Perl will warn if you output any of
these as being non-portable; and under strict UTF-8 input protocols,
they are forbidden.  In addition, it is deprecated to use a code point
larger than what a signed integer variable on your system can hold.  On
32-bit ASCII systems, this means <code class="inline"><span class="n">0x7FFF_FFFF</span></code>
 is the legal maximum
going forward (much higher on 64-bit systems).</p>
</li>
<li>
<p>UTF-EBCDIC</p>
<p>Like UTF-8, but EBCDIC-safe, in the way that UTF-8 is ASCII-safe.
This means that all the basic characters (which includes all
those that have ASCII equivalents (like <code class="inline"><span class="q">&quot;A&quot;</span></code>
, <code class="inline"><span class="q">&quot;0&quot;</span></code>
, <code class="inline"><span class="q">&quot;%&quot;</span></code>
, <i>etc.</i>)
are the same in both EBCDIC and UTF-EBCDIC.)</p>
<p>UTF-EBCDIC is used on EBCDIC platforms.  It generally requires more
bytes to represent a given code point than UTF-8 does; the largest
Unicode code points take 5 bytes to represent (instead of 4 in UTF-8),
and, extended for 64-bit words, it uses 14 bytes instead of 13 bytes in
UTF-8.</p>
</li>
<li>
<p>UTF-16, UTF-16BE, UTF-16LE, Surrogates, and <code class="inline"><span class="w">BOM</span></code>
's (Byte Order Marks)</p>
<p>The followings items are mostly for reference and general Unicode
knowledge, Perl doesn't use these constructs internally.</p>
<p>Like UTF-8, UTF-16 is a variable-width encoding, but where
UTF-8 uses 8-bit code units, UTF-16 uses 16-bit code units.
All code points occupy either 2 or 4 bytes in UTF-16: code points
<code class="inline"><span class="w">U</span>+<span class="n">0000</span>..<span class="w">U</span>+<span class="w">FFFF</span></code>
 are stored in a single 16-bit unit, and code
points <code class="inline">U+10000..U+10FFFF</code> in two 16-bit units.  The latter case is
using <i>surrogates</i>, the first 16-bit unit being the <i>high
surrogate</i>, and the second being the <i>low surrogate</i>.</p>
<p>Surrogates are code points set aside to encode the <code class="inline">U+10000..U+10FFFF</code>
range of Unicode code points in pairs of 16-bit units.  The <i>high
surrogates</i> are the range <code class="inline"><span class="w">U</span>+<span class="w">D800</span>..<span class="w">U</span>+<span class="w">DBFF</span></code>
 and the <i>low surrogates</i>
are the range <code class="inline"><span class="w">U</span>+<span class="w">DC00</span>..<span class="w">U</span>+<span class="w">DFFF</span></code>
.  The surrogate encoding is</p>
<pre class="verbatim"><ol><li>    <span class="i">$hi</span> = <span class="s">(</span><span class="i">$uni</span> - <span class="n">0x10000</span><span class="s">)</span> / <span class="n">0x400</span> + <span class="n">0xD800</span><span class="sc">;</span></li><li>    <span class="i">$lo</span> = <span class="s">(</span><span class="i">$uni</span> - <span class="n">0x10000</span><span class="s">)</span> % <span class="n">0x400</span> + <span class="n">0xDC00</span><span class="sc">;</span></li></ol></pre><p>and the decoding is</p>
<pre class="verbatim"><ol><li>    <span class="i">$uni</span> = <span class="n">0x10000</span> + <span class="s">(</span><span class="i">$hi</span> - <span class="n">0xD800</span><span class="s">)</span> * <span class="n">0x400</span> + <span class="s">(</span><span class="i">$lo</span> - <span class="n">0xDC00</span><span class="s">)</span><span class="sc">;</span></li></ol></pre><p>Because of the 16-bitness, UTF-16 is byte-order dependent.  UTF-16
itself can be used for in-memory computations, but if storage or
transfer is required either UTF-16BE (big-endian) or UTF-16LE
(little-endian) encodings must be chosen.</p>
<p>This introduces another problem: what if you just know that your data
is UTF-16, but you don't know which endianness?  Byte Order Marks, or
<code class="inline"><span class="w">BOM</span></code>
's, are a solution to this.  A special character has been reserved
in Unicode to function as a byte order marker: the character with the
code point <code class="inline"><span class="w">U</span>+<span class="w">FEFF</span></code>
 is the <code class="inline"><span class="w">BOM</span></code>
.</p>
<p>The trick is that if you read a <code class="inline"><span class="w">BOM</span></code>
, you will know the byte order,
since if it was written on a big-endian platform, you will read the
bytes <code class="inline">0xFE 0xFF</code>, but if it was written on a little-endian platform,
you will read the bytes <code class="inline">0xFF 0xFE</code>.  (And if the originating platform
was writing in ASCII platform UTF-8, you will read the bytes
<code class="inline">0xEF 0xBB 0xBF</code>.)</p>
<p>The way this trick works is that the character with the code point
<code class="inline"><span class="w">U</span>+<span class="w">FFFE</span></code>
 is not supposed to be in input streams, so the
sequence of bytes <code class="inline">0xFF 0xFE</code> is unambiguously "<code class="inline"><span class="w">BOM</span></code>
, represented in
little-endian format" and cannot be <code class="inline"><span class="w">U</span>+<span class="w">FFFE</span></code>
, represented in big-endian
format".</p>
<p>Surrogates have no meaning in Unicode outside their use in pairs to
represent other code points.  However, Perl allows them to be
represented individually internally, for example by saying
<code class="inline"><a class="l_k" href="functions/chr.html">chr(0xD801)</a></code>, so that all code points, not just those valid for open
interchange, are
representable.  Unicode does define semantics for them, such as their
<code class="inline"><a href="#General_Category">General_Category</a></code> is <code class="inline"><span class="q">&quot;Cs&quot;</span></code>
.  But because their use is somewhat dangerous,
Perl will warn (using the warning category <code class="inline"><span class="q">&quot;surrogate&quot;</span></code>
, which is a
sub-category of <code class="inline"><span class="q">&quot;utf8&quot;</span></code>
) if an attempt is made
to do things like take the lower case of one, or match
case-insensitively, or to output them.  (But don't try this on Perls
before 5.14.)</p>
</li>
<li>
<p>UTF-32, UTF-32BE, UTF-32LE</p>
<p>The UTF-32 family is pretty much like the UTF-16 family, except that
the units are 32-bit, and therefore the surrogate scheme is not
needed.  UTF-32 is a fixed-width encoding.  The <code class="inline"><span class="w">BOM</span></code>
 signatures are
<code class="inline">0x00 0x00 0xFE 0xFF</code> for BE and <code class="inline">0xFF 0xFE 0x00 0x00</code> for LE.</p>
</li>
<li>
<p>UCS-2, UCS-4</p>
<p>Legacy, fixed-width encodings defined by the ISO 10646 standard.  UCS-2 is a 16-bit
encoding.  Unlike UTF-16, UCS-2 is not extensible beyond <code class="inline"><span class="w">U</span>+<span class="w">FFFF</span></code>
,
because it does not use surrogates.  UCS-4 is a 32-bit encoding,
functionally identical to UTF-32 (the difference being that
UCS-4 forbids neither surrogates nor code points larger than <code class="inline"><span class="n">0x10_FFFF</span></code>
).</p>
</li>
<li>
<p>UTF-7</p>
<p>A seven-bit safe (non-eight-bit) encoding, which is useful if the
transport or storage is not eight-bit safe.  Defined by RFC 2152.</p>
</li>
</ul>
<a name="Noncharacter-code-points"></a><h2>Noncharacter code points</h2>
<p>66 code points are set aside in Unicode as "noncharacter code points".
These all have the <code class="inline"><span class="w">Unassigned</span></code>
 (<code class="inline"><span class="w">Cn</span></code>
) <code class="inline"><a href="#General_Category">General_Category</a></code>, and
no character will ever be assigned to any of them.  They are the 32 code
points between <code class="inline"><span class="w">U</span>+<span class="w">FDD0</span></code>
 and <code class="inline"><span class="w">U</span>+<span class="w">FDEF</span></code>
 inclusive, and the 34 code
points:</p>
<pre class="verbatim"><ol><li> <span class="w">U</span>+<span class="w">FFFE</span>   <span class="w">U</span>+<span class="w">FFFF</span></li><li> <span class="w">U</span>+<span class="n">1</span><span class="w">FFFE</span>  <span class="w">U</span>+<span class="n">1</span><span class="w">FFFF</span></li><li> <span class="w">U</span>+<span class="n">2</span><span class="w">FFFE</span>  <span class="w">U</span>+<span class="n">2</span><span class="w">FFFF</span></li><li> ...</li><li> <span class="w">U</span>+<span class="w">EFFFE</span>  <span class="w">U</span>+<span class="w">EFFFF</span></li><li> <span class="w">U</span>+<span class="w">FFFFE</span>  <span class="w">U</span>+<span class="w">FFFFF</span></li><li> <span class="w">U</span>+<span class="n">10</span><span class="w">FFFE</span> <span class="w">U</span>+<span class="n">10</span><span class="w">FFFF</span></li></ol></pre><p>Until Unicode 7.0, the noncharacters were "<b>forbidden</b> for use in open
interchange of Unicode text data", so that code that processed those
streams could use these code points as sentinels that could be mixed in
with character data, and would always be distinguishable from that data.
(Emphasis above and in the next paragraph are added in this document.)</p>
<p>Unicode 7.0 changed the wording so that they are "<b>not recommended</b> for
use in open interchange of Unicode text data".  The 7.0 Standard goes on
to say:</p>
<p>This change was made because it was found that various commercial tools
like editors, or for things like source code control, had been written
so that they would not handle program files that used these code points,
effectively precluding their use almost entirely!  And that was never
the intent.  They've always been meant to be usable within an
application, or cooperating set of applications, at will.</p>
<p>If you're writing code, such as an editor, that is supposed to be able
to handle any Unicode text data, then you shouldn't be using these code
points yourself, and instead allow them in the input.  If you need
sentinels, they should instead be something that isn't legal Unicode.
For UTF-8 data, you can use the bytes 0xC1 and 0xC2 as sentinels, as
they never appear in well-formed UTF-8.  (There are equivalents for
UTF-EBCDIC).  You can also store your Unicode code points in integer
variables and use negative values as sentinels.</p>
<p>If you're not writing such a tool, then whether you accept noncharacters
as input is up to you (though the Standard recommends that you not).  If
you do strict input stream checking with Perl, these code points
continue to be forbidden.  This is to maintain backward compatibility
(otherwise potential security holes could open up, as an unsuspecting
application that was written assuming the noncharacters would be
filtered out before getting to it, could now, without warning, start
getting them).  To do strict checking, you can use the layer
<code class="inline"><span class="j">:</span><span class="i">encoding</span><span class="s">(</span><span class="q">&#39;UTF-8&#39;</span><span class="s">)</span></code>
.</p>
<p>Perl continues to warn (using the warning category <code class="inline"><span class="q">&quot;nonchar&quot;</span></code>
, which
is a sub-category of <code class="inline"><span class="q">&quot;utf8&quot;</span></code>
) if an attempt is made to output
noncharacters.</p>
<a name="Beyond-Unicode-code-points"></a><h2>Beyond Unicode code points</h2>
<p>The maximum Unicode code point is <code class="inline"><span class="w">U</span>+<span class="n">10</span><span class="w">FFFF</span></code>
, and Unicode only defines
operations on code points up through that.  But Perl works on code
points up to the maximum permissible unsigned number available on the
platform.  However, Perl will not accept these from input streams unless
lax rules are being used, and will warn (using the warning category
<code class="inline"><span class="q">&quot;non_unicode&quot;</span></code>
, which is a sub-category of <code class="inline"><span class="q">&quot;utf8&quot;</span></code>
) if any are output.</p>
<p>Since Unicode rules are not defined on these code points, if a
Unicode-defined operation is done on them, Perl uses what we believe are
sensible rules, while generally warning, using the <code class="inline"><span class="q">&quot;non_unicode&quot;</span></code>

category.  For example, <code class="inline"><a class="l_k" href="functions/uc.html">uc("\x{11_0000}")</a></code> will generate such a
warning, returning the input parameter as its result, since Perl defines
the uppercase of every non-Unicode code point to be the code point
itself.  (All the case changing operations, not just uppercasing, work
this way.)</p>
<p>The situation with matching Unicode properties in regular expressions,
the <code class="inline">\<span class="w">p</span><span class="s">{</span><span class="s">}</span></code>
 and <code class="inline">\<span class="w">P</span><span class="s">{</span><span class="s">}</span></code>
 constructs, against these code points is not as
clear cut, and how these are handled has changed as we've gained
experience.</p>
<p>One possibility is to treat any match against these code points as
undefined.  But since Perl doesn't have the concept of a match being
undefined, it converts this to failing or <code class="inline"><span class="w">FALSE</span></code>
.  This is almost, but
not quite, what Perl did from v5.14 (when use of these code points
became generally reliable) through v5.18.  The difference is that Perl
treated all <code class="inline">\<span class="w">p</span><span class="s">{</span><span class="s">}</span></code>
 matches as failing, but all <code class="inline">\<span class="w">P</span><span class="s">{</span><span class="s">}</span></code>
 matches as
succeeding.</p>
<p>One problem with this is that it leads to unexpected, and confusing
results in some cases:</p>
<pre class="verbatim"><ol><li> <a class="l_k" href="functions/chr.html">chr</a><span class="s">(</span><span class="n">0x110000</span><span class="s">)</span> =~ \<span class="i">p</span><span class="s">{</span><span class="w">ASCII_Hex_Digit</span>=<span class="w">True</span><span class="s">}</span>      <span class="c"># Failed on &lt;= v5.18</span></li><li> <a class="l_k" href="functions/chr.html">chr</a><span class="s">(</span><span class="n">0x110000</span><span class="s">)</span> =~ \<span class="i">p</span><span class="s">{</span><span class="w">ASCII_Hex_Digit</span>=<span class="w">False</span><span class="s">}</span>     <span class="c"># Failed! on &lt;= v5.18</span></li></ol></pre><p>That is, it treated both matches as undefined, and converted that to
false (raising a warning on each).  The first case is the expected
result, but the second is likely counterintuitive: "How could both be
false when they are complements?"  Another problem was that the
implementation optimized many Unicode property matches down to already
existing simpler, faster operations, which don't raise the warning.  We
chose to not forgo those optimizations, which help the vast majority of
matches, just to generate a warning for the unlikely event that an
above-Unicode code point is being matched against.</p>
<p>As a result of these problems, starting in v5.20, what Perl does is
to treat non-Unicode code points as just typical unassigned Unicode
characters, and matches accordingly.  (Note: Unicode has atypical
unassigned code points.  For example, it has noncharacter code points,
and ones that, when they do get assigned, are destined to be written
Right-to-left, as Arabic and Hebrew are.  Perl assumes that no
non-Unicode code point has any atypical properties.)</p>
<p>Perl, in most cases, will raise a warning when matching an above-Unicode
code point against a Unicode property when the result is <code class="inline"><span class="w">TRUE</span></code>
 for
<code class="inline">\<span class="w">p</span><span class="s">{</span><span class="s">}</span></code>
, and <code class="inline"><span class="w">FALSE</span></code>
 for <code class="inline">\<span class="w">P</span><span class="s">{</span><span class="s">}</span></code>
.  For example:</p>
<pre class="verbatim"><ol><li> <a class="l_k" href="functions/chr.html">chr</a><span class="s">(</span><span class="n">0x110000</span><span class="s">)</span> =~ \<span class="i">p</span><span class="s">{</span><span class="w">ASCII_Hex_Digit</span>=<span class="w">True</span><span class="s">}</span>      <span class="c"># Fails, no warning</span></li><li> <a class="l_k" href="functions/chr.html">chr</a><span class="s">(</span><span class="n">0x110000</span><span class="s">)</span> =~ \<span class="i">p</span><span class="s">{</span><span class="w">ASCII_Hex_Digit</span>=<span class="w">False</span><span class="s">}</span>     <span class="c"># Succeeds, with warning</span></li></ol></pre><p>In both these examples, the character being matched is non-Unicode, so
Unicode doesn't define how it should match.  It clearly isn't an ASCII
hex digit, so the first example clearly should fail, and so it does,
with no warning.  But it is arguable that the second example should have
an undefined, hence <code class="inline"><span class="w">FALSE</span></code>
, result.  So a warning is raised for it.</p>
<p>Thus the warning is raised for many fewer cases than in earlier Perls,
and only when what the result is could be arguable.  It turns out that
none of the optimizations made by Perl (or are ever likely to be made)
cause the warning to be skipped, so it solves both problems of Perl's
earlier approach.  The most commonly used property that is affected by
this change is <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Unassigned</span><span class="s">}</span></code>
 which is a short form for
<code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">General_Category</span>=<span class="w">Unassigned</span><span class="s">}</span></code>
.  Starting in v5.20, all non-Unicode
code points are considered <code class="inline"><span class="w">Unassigned</span></code>
.  In earlier releases the
matches failed because the result was considered undefined.</p>
<p>The only place where the warning is not raised when it might ought to
have been is if optimizations cause the whole pattern match to not even
be attempted.  For example, Perl may figure out that for a string to
match a certain regular expression pattern, the string has to contain
the substring <code class="inline"><span class="q">&quot;foobar&quot;</span></code>
.  Before attempting the match, Perl may look
for that substring, and if not found, immediately fail the match without
actually trying it; so no warning gets generated even if the string
contains an above-Unicode code point.</p>
<p>This behavior is more "Do what I mean" than in earlier Perls for most
applications.  But it catches fewer issues for code that needs to be
strictly Unicode compliant.  Therefore there is an additional mode of
operation available to accommodate such code.  This mode is enabled if a
regular expression pattern is compiled within the lexical scope where
the <code class="inline"><span class="q">&quot;non_unicode&quot;</span></code>
 warning class has been made fatal, say by:</p>
<pre class="verbatim"><ol><li> <a class="l_k" href="functions/use.html">use</a> <span class="w">warnings</span> <span class="w">FATAL</span> <span class="cm">=&gt;</span> <span class="q">&quot;non_unicode&quot;</span></li></ol></pre><p>(see <a href="warnings.html">warnings</a>).  In this mode of operation, Perl will raise the
warning for all matches against a non-Unicode code point (not just the
arguable ones), and it skips the optimizations that might cause the
warning to not be output.  (It currently still won't warn if the match
isn't even attempted, like in the <code class="inline"><span class="q">&quot;foobar&quot;</span></code>
 example above.)</p>
<p>In summary, Perl now normally treats non-Unicode code points as typical
Unicode unassigned code points for regular expression matches, raising a
warning only when it is arguable what the result should be.  However, if
this warning has been made fatal, it isn't skipped.</p>
<p>There is one exception to all this.  <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">All</span><span class="s">}</span></code>
 looks like a Unicode
property, but it is a Perl extension that is defined to be true for all
possible code points, Unicode or not, so no warning is ever generated
when matching this against a non-Unicode code point.  (Prior to v5.20,
it was an exact synonym for <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Any</span><span class="s">}</span></code>
, matching code points <code class="inline"><span class="n">0</span></code>

through <code class="inline"><span class="n">0x10FFFF</span></code>
.)</p>
<a name="Security-Implications-of-Unicode"></a><h2>Security Implications of Unicode</h2>
<p>First, read
<a href="http://search.cpan.org/perldoc/http:#%2fwww.unicode.org%2freports%2ftr36">Unicode Security Considerations</a>.</p>
<p>Also, note the following:</p>
<ul>
<li>
<p>Malformed UTF-8</p>
<p>UTF-8 is very structured, so many combinations of bytes are invalid.  In
the past, Perl tried to soldier on and make some sense of invalid
combinations, but this can lead to security holes, so now, if the Perl
core needs to process an invalid combination, it will either raise a
fatal error, or will replace those bytes by the sequence that forms the
Unicode REPLACEMENT CHARACTER, for which purpose Unicode created it.</p>
<p>Every code point can be represented by more than one possible
syntactically valid UTF-8 sequence.  Early on, both Unicode and Perl
considered any of these to be valid, but now, all sequences longer
than the shortest possible one are considered to be malformed.</p>
<p>Unicode considers many code points to be illegal, or to be avoided.
Perl generally accepts them, once they have passed through any input
filters that may try to exclude them.  These have been discussed above
(see "Surrogates" under UTF-16 in <a href="#Unicode-Encodings">Unicode Encodings</a>,
<a href="#Noncharacter-code-points">Noncharacter code points</a>, and <a href="#Beyond-Unicode-code-points">Beyond Unicode code points</a>).</p>
</li>
<li>
<p>Regular expression pattern matching may surprise you if you're not
accustomed to Unicode.  Starting in Perl 5.14, several pattern
modifiers are available to control this, called the character set
modifiers.  Details are given in <a href="perlre.html#Character-set-modifiers">Character set modifiers in perlre</a>.</p>
</li>
</ul>
<p>As discussed elsewhere, Perl has one foot (two hooves?) planted in
each of two worlds: the old world of ASCII and single-byte locales, and
the new world of Unicode, upgrading when necessary.
If your legacy code does not explicitly use Unicode, no automatic
switch-over to Unicode should happen.</p>
<a name="Unicode-in-Perl-on-EBCDIC"></a><h2>Unicode in Perl on EBCDIC</h2>
<p>Unicode is supported on EBCDIC platforms.  See <a href="perlebcdic.html">perlebcdic</a>.</p>
<p>Unless ASCII vs. EBCDIC issues are specifically being discussed,
references to UTF-8 encoding in this document and elsewhere should be
read as meaning UTF-EBCDIC on EBCDIC platforms.
See <a href="perlebcdic.html#Unicode-and-UTF">Unicode and UTF in perlebcdic</a>.</p>
<p>Because UTF-EBCDIC is so similar to UTF-8, the differences are mostly
hidden from you; <code class="inline"><a class="l_k" href="functions/use.html">use</a> <span class="w">utf8</span></code>
 (and NOT something like
<code class="inline"><a class="l_k" href="functions/use.html">use</a> <span class="w">utfebcdic</span></code>
) declares the the script is in the platform's
"native" 8-bit encoding of Unicode.  (Similarly for the <code class="inline"><span class="q">&quot;:utf8&quot;</span></code>

layer.)</p>
<a name="Locales"></a><h2>Locales</h2>
<p>See <a href="perllocale.html#Unicode-and-UTF-8">Unicode and UTF-8 in perllocale</a></p>
<a name="When-Unicode-Does-Not-Happen"></a><h2>When Unicode Does Not Happen</h2>
<p>There are still many places where Unicode (in some encoding or
another) could be given as arguments or received as results, or both in
Perl, but it is not, in spite of Perl having extensive ways to input and
output in Unicode, and a few other "entry points" like the <code class="inline"><span class="i">@ARGV</span></code>

array (which can sometimes be interpreted as UTF-8).</p>
<p>The following are such interfaces.  Also, see <a href="#The-%22Unicode-Bug%22">The Unicode Bug</a>.
For all of these interfaces Perl
currently (as of v5.16.0) simply assumes byte strings both as arguments
and results, or UTF-8 strings if the (deprecated) <code class="inline"><span class="w">encoding</span></code>
 pragma has been used.</p>
<p>One reason that Perl does not attempt to resolve the role of Unicode in
these situations is that the answers are highly dependent on the operating
system and the file system(s).  For example, whether filenames can be
in Unicode and in exactly what kind of encoding, is not exactly a
portable concept.  Similarly for <code class="inline"><a class="l_k" href="functions/qx.html">qx</a></code> and <code class="inline"><a class="l_k" href="functions/system.html">system</a></code>: how well will the
"command-line interface" (and which of them?) handle Unicode?</p>
<ul>
<li>
<p><code class="inline"><a class="l_k" href="functions/chdir.html">chdir</a></code>, <code class="inline"><a class="l_k" href="functions/chmod.html">chmod</a></code>, <code class="inline"><a class="l_k" href="functions/chown.html">chown</a></code>, <code class="inline"><a class="l_k" href="functions/chroot.html">chroot</a></code>, <code class="inline"><a class="l_k" href="functions/exec.html">exec</a></code>, <code class="inline"><a class="l_k" href="functions/link.html">link</a></code>, <code class="inline"><a class="l_k" href="functions/lstat.html">lstat</a></code>, <code class="inline"><a class="l_k" href="functions/mkdir.html">mkdir</a></code>,
<code class="inline"><a class="l_k" href="functions/rename.html">rename</a></code>, <code class="inline"><a class="l_k" href="functions/rmdir.html">rmdir</a></code>, <code class="inline"><a class="l_k" href="functions/stat.html">stat</a></code>, <code class="inline"><a class="l_k" href="functions/symlink.html">symlink</a></code>, <code class="inline"><a class="l_k" href="functions/truncate.html">truncate</a></code>, <code class="inline"><a class="l_k" href="functions/unlink.html">unlink</a></code>, <code class="inline"><a class="l_k" href="functions/utime.html">utime</a></code>, <code class="inline"><a class="l_k" href="functions/-X.html">-X</a></code></p>
</li>
<li>
<p><code class="inline"><span class="i">%ENV</span></code>
</p>
</li>
<li>
<p><code class="inline"><a class="l_k" href="functions/glob.html">glob</a></code> (aka the <code class="inline"><span class="q">&lt;*&gt;</span></code>
)</p>
</li>
<li>
<p><code class="inline"><a class="l_k" href="functions/open.html">open</a></code>, <code class="inline"><a class="l_k" href="functions/opendir.html">opendir</a></code>, <code class="inline"><a class="l_k" href="functions/sysopen.html">sysopen</a></code></p>
</li>
<li>
<p><code class="inline"><a class="l_k" href="functions/qx.html">qx</a></code> (aka the backtick operator), <code class="inline"><a class="l_k" href="functions/system.html">system</a></code></p>
</li>
<li>
<p><code class="inline"><a class="l_k" href="functions/readdir.html">readdir</a></code>, <code class="inline"><a class="l_k" href="functions/readlink.html">readlink</a></code></p>
</li>
</ul>
<a name="The-%22Unicode-Bug%22"></a><h2>The "Unicode Bug"</h2>
<p>The term, "Unicode bug" has been applied to an inconsistency with the
code points in the <code class="inline">Latin-1 Supplement</code> block, that is, between
128 and 255.  Without a locale specified, unlike all other characters or
code points, these characters can have very different semantics
depending on the rules in effect.  (Characters whose code points are
above 255 force Unicode rules; whereas the rules for ASCII characters
are the same under both ASCII and Unicode rules.)</p>
<p>Under Unicode rules, these upper-Latin1 characters are interpreted as
Unicode code points, which means they have the same semantics as Latin-1
(ISO-8859-1) and C1 controls.</p>
<p>As explained in <a href="#ASCII-Rules-versus-Unicode-Rules">ASCII Rules versus Unicode Rules</a>, under ASCII rules,
they are considered to be unassigned characters.</p>
<p>This can lead to unexpected results.  For example, a string's
semantics can suddenly change if a code point above 255 is appended to
it, which changes the rules from ASCII to Unicode.  As an
example, consider the following program and its output:</p>
<pre class="verbatim"><ol><li> <span class="i">$ perl</span> -<a class="l_k" href="functions/le.html">le</a><span class="q">&#39;</span></li><li>     <span class="q">     no feature &quot;unicode_strings&quot;;</span></li><li>     <span class="q">     $s1 = &quot;\xC2&quot;;</span></li><li>     <span class="q">     $s2 = &quot;\x{2660}&quot;;</span></li><li>     <span class="q">     for ($s1, $s2, $s1.$s2) {</span></li><li>         <span class="q">         print /\w/ || 0;</span></li><li>     <span class="q">     }</span></li><li> <span class="q"> &#39;</span></li><li> <span class="n">0</span></li><li> <span class="n">0</span></li><li> <span class="n">1</span></li></ol></pre><p>If there's no <code class="inline">\<span class="w">w</span></code>
 in <code class="inline"><span class="w">s1</span></code>
 nor in <code class="inline"><span class="w">s2</span></code>
, why does their concatenation
have one?</p>
<p>This anomaly stems from Perl's attempt to not disturb older programs that
didn't use Unicode, along with Perl's desire to add Unicode support
seamlessly.  But the result turned out to not be seamless.  (By the way,
you can choose to be warned when things like this happen.  See
<code class="inline"><a href="encoding/warnings.html">encoding::warnings</a></code>.)</p>
<p><a href="feature.html#The-'unicode_strings'-feature">use feature &#39;unicode_strings&#39; </a>
was added, starting in Perl v5.12, to address this problem.  It affects
these things:</p>
<ul>
<li>
<p>Changing the case of a scalar, that is, using <code class="inline"><a class="l_k" href="functions/uc.html">uc()</a></code>, <code class="inline"><a class="l_k" href="functions/ucfirst.html">ucfirst()</a></code>, <code class="inline"><a class="l_k" href="functions/lc.html">lc()</a></code>,
and <code class="inline"><a class="l_k" href="functions/lcfirst.html">lcfirst()</a></code>, or <code class="inline">\<span class="w">L</span></code>
, <code class="inline">\<span class="w">U</span></code>
, <code class="inline">\<span class="w">u</span></code>
 and <code class="inline">\<span class="w">l</span></code>
 in double-quotish
contexts, such as regular expression substitutions.</p>
<p>Under <code class="inline"><span class="w">unicode_strings</span></code>
 starting in Perl 5.12.0, Unicode rules are
generally used.  See <a href="functions/lc.html">lc</a> for details on how this works
in combination with various other pragmas.</p>
</li>
<li>
<p>Using caseless (<code class="inline">/i</code>) regular expression matching.</p>
<p>Starting in Perl 5.14.0, regular expressions compiled within
the scope of <code class="inline"><span class="w">unicode_strings</span></code>
 use Unicode rules
even when executed or compiled into larger
regular expressions outside the scope.</p>
</li>
<li>
<p>Matching any of several properties in regular expressions.</p>
<p>These properties are <code class="inline">\<span class="w">b</span></code>
 (without braces), <code class="inline">\<span class="w">B</span></code>
 (without braces),
<code class="inline">\s</code>, <code class="inline">\<span class="w">S</span></code>
, <code class="inline">\<span class="w">w</span></code>
, <code class="inline">\<span class="w">W</span></code>
, and all the Posix character classes
<i>except</i> <code class="inline"><span class="s">[</span><span class="s">[</span><span class="co">:</span><span class="w">ascii</span><span class="co">:</span><span class="s">]</span><span class="s">]</span></code>
.</p>
<p>Starting in Perl 5.14.0, regular expressions compiled within
the scope of <code class="inline"><span class="w">unicode_strings</span></code>
 use Unicode rules
even when executed or compiled into larger
regular expressions outside the scope.</p>
</li>
<li>
<p>In <code class="inline"><a class="l_k" href="functions/quotemeta.html">quotemeta</a></code> or its inline equivalent <code class="inline">\<span class="w">Q</span></code>
.</p>
<p>Starting in Perl 5.16.0, consistent quoting rules are used within the
scope of <code class="inline"><span class="w">unicode_strings</span></code>
, as described in <a href="functions/quotemeta.html">quotemeta</a>.
Prior to that, or outside its scope, no code points above 127 are quoted
in UTF-8 encoded strings, but in byte encoded strings, code points
between 128-255 are always quoted.</p>
</li>
<li>
<p>In the <code class="inline">..</code>
 or <a href="perlop.html#Range-Operators">range</a> operator.</p>
<p>Starting in Perl 5.26.0, the range operator on strings treats their lengths
consistently within the scope of <code class="inline"><span class="w">unicode_strings</span></code>
. Prior to that, or
outside its scope, it could produce strings whose length in characters
exceeded that of the right-hand side, where the right-hand side took up more
bytes than the correct range endpoint.</p>
</li>
</ul>
<p>You can see from the above that the effect of <code class="inline"><span class="w">unicode_strings</span></code>

increased over several Perl releases.  (And Perl's support for Unicode
continues to improve; it's best to use the latest available release in
order to get the most complete and accurate results possible.)  Note that
<code class="inline"><span class="w">unicode_strings</span></code>
 is automatically chosen if you <code class="inline"><a class="l_k" href="functions/use.html">use</a> <span class="n">5.012</span></code>
 or
higher.</p>
<p>For Perls earlier than those described above, or when a string is passed
to a function outside the scope of <code class="inline"><span class="w">unicode_strings</span></code>
, see the next section.</p>
<a name="Forcing-Unicode-in-Perl-(Or-Unforcing-Unicode-in-Perl)"></a><h2>Forcing Unicode in Perl (Or Unforcing Unicode in Perl)</h2>
<p>Sometimes (see <a href="#When-Unicode-Does-Not-Happen">When Unicode Does Not Happen</a> or <a href="#The-%22Unicode-Bug%22">The Unicode Bug</a>)
there are situations where you simply need to force a byte
string into UTF-8, or vice versa.  The standard module <a href="Encode.html">Encode</a> can be
used for this, or the low-level calls
<a href="utf8.html#Utility-functions">utf8::upgrade($bytestring) </a> and
<a href="utf8.html#Utility-functions">utf8::downgrade($utf8string[, FAIL_OK]) </a>.</p>
<p>Note that <code class="inline"><span class="i">utf8::downgrade</span><span class="s">(</span><span class="s">)</span></code>
 can fail if the string contains characters
that don't fit into a byte.</p>
<p>Calling either function on a string that already is in the desired state is a
no-op.</p>
<p><a href="#ASCII-Rules-versus-Unicode-Rules">ASCII Rules versus Unicode Rules</a> gives all the ways that a string is
made to use Unicode rules.</p>
<a name="Using-Unicode-in-XS"></a><h2>Using Unicode in XS</h2>
<p>See <a href="perlguts.html#Unicode-Support">Unicode Support in perlguts</a> for an introduction to Unicode at
the XS level, and <a href="perlapi.html#Unicode-Support">Unicode Support in perlapi</a> for the API details.</p>
<a name="Hacking-Perl-to-work-on-earlier-Unicode-versions-(for-very-serious-hackers-only)"></a><h2>Hacking Perl to work on earlier Unicode versions (for very serious hackers only)</h2>
<p>Perl by default comes with the latest supported Unicode version built-in, but
the goal is to allow you to change to use any earlier one.  In Perls
v5.20 and v5.22, however, the earliest usable version is Unicode 5.1.
Perl v5.18 and v5.24 are able to handle all earlier versions.</p>
<p>Download the files in the desired version of Unicode from the Unicode web
site <a href="http://www.unicode.org">http://www.unicode.org</a>).  These should replace the existing files in
<i>lib/unicore</i> in the Perl source tree.  Follow the instructions in
<i>README.perl</i> in that directory to change some of their names, and then build
perl (see <a href="http://search.cpan.org/perldoc/INSTALL">INSTALL</a>).</p>
<a name="Porting-code-from-perl-5.6.X"></a><h2>Porting code from perl-5.6.X</h2>
<p>Perls starting in 5.8 have a different Unicode model from 5.6. In 5.6 the
programmer was required to use the <code class="inline"><span class="w">utf8</span></code>
 pragma to declare that a
given scope expected to deal with Unicode data and had to make sure that
only Unicode data were reaching that scope. If you have code that is
working with 5.6, you will need some of the following adjustments to
your code. The examples are written such that the code will continue to
work under 5.6, so you should be safe to try them out.</p>
<ul>
<li>
<p>A filehandle that should read or write UTF-8</p>
<pre class="verbatim"><ol><li>  <a class="l_k" href="functions/if.html">if</a> <span class="s">(</span><span class="i">$]</span> &gt; <span class="n">5.008</span><span class="s">)</span> <span class="s">{</span></li><li>    <a class="l_k" href="functions/binmode.html">binmode</a> <span class="i">$fh</span><span class="cm">,</span> <span class="q">&quot;:encoding(UTF-8)&quot;</span><span class="sc">;</span></li><li>  <span class="s">}</span></li></ol></pre></li>
<li>
<p>A scalar that is going to be passed to some extension</p>
<p>Be it <code class="inline"><span class="w">Compress::Zlib</span></code>
, <code class="inline"><span class="w">Apache::Request</span></code>
 or any extension that has no
mention of Unicode in the manpage, you need to make sure that the
UTF8 flag is stripped off. Note that at the time of this writing
(January 2012) the mentioned modules are not UTF-8-aware. Please
check the documentation to verify if this is still true.</p>
<pre class="verbatim"><ol><li>  <a class="l_k" href="functions/if.html">if</a> <span class="s">(</span><span class="i">$]</span> &gt; <span class="n">5.008</span><span class="s">)</span> <span class="s">{</span></li><li>    <a class="l_k" href="functions/require.html">require</a> <span class="w">Encode</span><span class="sc">;</span></li><li>    <span class="i">$val</span> = <span class="i">Encode::encode</span><span class="s">(</span><span class="q">&quot;UTF-8&quot;</span><span class="cm">,</span> <span class="i">$val</span><span class="s">)</span><span class="sc">;</span> <span class="c"># make octets</span></li><li>  <span class="s">}</span></li></ol></pre></li>
<li>
<p>A scalar we got back from an extension</p>
<p>If you believe the scalar comes back as UTF-8, you will most likely
want the UTF8 flag restored:</p>
<pre class="verbatim"><ol><li>  <a class="l_k" href="functions/if.html">if</a> <span class="s">(</span><span class="i">$]</span> &gt; <span class="n">5.008</span><span class="s">)</span> <span class="s">{</span></li><li>    <a class="l_k" href="functions/require.html">require</a> <span class="w">Encode</span><span class="sc">;</span></li><li>    <span class="i">$val</span> = <span class="i">Encode::decode</span><span class="s">(</span><span class="q">&quot;UTF-8&quot;</span><span class="cm">,</span> <span class="i">$val</span><span class="s">)</span><span class="sc">;</span></li><li>  <span class="s">}</span></li></ol></pre></li>
<li>
<p>Same thing, if you are really sure it is UTF-8</p>
<pre class="verbatim"><ol><li>  <a class="l_k" href="functions/if.html">if</a> <span class="s">(</span><span class="i">$]</span> &gt; <span class="n">5.008</span><span class="s">)</span> <span class="s">{</span></li><li>    <a class="l_k" href="functions/require.html">require</a> <span class="w">Encode</span><span class="sc">;</span></li><li>    <span class="i">Encode::_utf8_on</span><span class="s">(</span><span class="i">$val</span><span class="s">)</span><span class="sc">;</span></li><li>  <span class="s">}</span></li></ol></pre></li>
<li>
<p>A wrapper for <a href="http://search.cpan.org/perldoc/DBI">DBI</a> <code class="inline"><span class="w">fetchrow_array</span></code>
 and <code class="inline"><span class="w">fetchrow_hashref</span></code>
</p>
<p>When the database contains only UTF-8, a wrapper function or method is
a convenient way to replace all your <code class="inline"><span class="w">fetchrow_array</span></code>
 and
<code class="inline"><span class="w">fetchrow_hashref</span></code>
 calls. A wrapper function will also make it easier to
adapt to future enhancements in your database driver. Note that at the
time of this writing (January 2012), the DBI has no standardized way
to deal with UTF-8 data. Please check the <a href="http://search.cpan.org/perldoc/DBI">DBI documentation</a> to verify if
that is still true.</p>
<pre class="verbatim"><ol><li><a name="fetchrow"></a>  sub <span class="m">fetchrow</span> <span class="s">{</span></li><li>    <span class="c"># $what is one of fetchrow_{array,hashref}</span></li><li>    <a class="l_k" href="functions/my.html">my</a><span class="s">(</span><span class="i">$self</span><span class="cm">,</span> <span class="i">$sth</span><span class="cm">,</span> <span class="i">$what</span><span class="s">)</span> = <span class="i">@_</span><span class="sc">;</span></li><li>    <a class="l_k" href="functions/if.html">if</a> <span class="s">(</span><span class="i">$]</span> &lt; <span class="n">5.008</span><span class="s">)</span> <span class="s">{</span></li><li>      <a class="l_k" href="functions/return.html">return</a> <span class="i">$sth</span><span class="i">-&gt;$what</span><span class="sc">;</span></li><li>    <span class="s">}</span> <a class="l_k" href="functions/else.html">else</a> <span class="s">{</span></li><li>      <a class="l_k" href="functions/require.html">require</a> <span class="w">Encode</span><span class="sc">;</span></li><li>      <a class="l_k" href="functions/if.html">if</a> <span class="s">(</span><a class="l_k" href="functions/wantarray.html">wantarray</a><span class="s">)</span> <span class="s">{</span></li><li>        <a class="l_k" href="functions/my.html">my</a> <span class="i">@arr</span> = <span class="i">$sth</span><span class="i">-&gt;$what</span><span class="sc">;</span></li><li>        <a class="l_k" href="functions/for.html">for</a> <span class="s">(</span><span class="i">@arr</span><span class="s">)</span> <span class="s">{</span></li><li>          <a class="l_k" href="functions/defined.html">defined</a> &amp;&amp; <span class="q">/[^\000-\177]/</span> &amp;&amp; <span class="i">Encode::_utf8_on</span><span class="s">(</span><span class="i">$_</span><span class="s">)</span><span class="sc">;</span></li><li>        <span class="s">}</span></li><li>        <a class="l_k" href="functions/return.html">return</a> <span class="i">@arr</span><span class="sc">;</span></li><li>      <span class="s">}</span> <a class="l_k" href="functions/else.html">else</a> <span class="s">{</span></li><li>        <a class="l_k" href="functions/my.html">my</a> <span class="i">$ret</span> = <span class="i">$sth</span><span class="i">-&gt;$what</span><span class="sc">;</span></li><li>        <a class="l_k" href="functions/if.html">if</a> <span class="s">(</span><a class="l_k" href="functions/ref.html">ref</a> <span class="i">$ret</span><span class="s">)</span> <span class="s">{</span></li><li>          <a class="l_k" href="functions/for.html">for</a> <a class="l_k" href="functions/my.html">my</a> <span class="i">$k</span> <span class="s">(</span><a class="l_k" href="functions/keys.html">keys</a> <span class="i">%$ret</span><span class="s">)</span> <span class="s">{</span></li><li>            <a class="l_k" href="functions/defined.html">defined</a></li><li>            &amp;&amp; <span class="q">/[^\000-\177]/</span></li><li>            &amp;&amp; <span class="i">Encode::_utf8_on</span><span class="s">(</span><span class="i">$_</span><span class="s">)</span> <a class="l_k" href="functions/for.html">for</a> <span class="i">$ret</span>-&gt;{<span class="i">$k</span>}<span class="sc">;</span></li><li>          <span class="s">}</span></li><li>          <a class="l_k" href="functions/return.html">return</a> <span class="i">$ret</span><span class="sc">;</span></li><li>        <span class="s">}</span> <a class="l_k" href="functions/else.html">else</a> <span class="s">{</span></li><li>          <a class="l_k" href="functions/defined.html">defined</a> &amp;&amp; <span class="q">/[^\000-\177]/</span> &amp;&amp; <span class="i">Encode::_utf8_on</span><span class="s">(</span><span class="i">$_</span><span class="s">)</span> <a class="l_k" href="functions/for.html">for</a> <span class="i">$ret</span><span class="sc">;</span></li><li>          <a class="l_k" href="functions/return.html">return</a> <span class="i">$ret</span><span class="sc">;</span></li><li>        <span class="s">}</span></li><li>      <span class="s">}</span></li><li>    <span class="s">}</span></li><li>  <span class="s">}</span></li></ol></pre></li>
<li>
<p>A large scalar that you know can only contain ASCII</p>
<p>Scalars that contain only ASCII and are marked as UTF-8 are sometimes
a drag to your program. If you recognize such a situation, just remove
the UTF8 flag:</p>
<pre class="verbatim"><ol><li>  <span class="i">utf8::downgrade</span><span class="s">(</span><span class="i">$val</span><span class="s">)</span> <a class="l_k" href="functions/if.html">if</a> <span class="i">$]</span> &gt; <span class="n">5.008</span><span class="sc">;</span></li></ol></pre></li>
</ul>
<a name="BUGS"></a><h1>BUGS</h1>
<p>See also <a href="#The-%22Unicode-Bug%22">The Unicode Bug</a> above.</p>
<a name="Interaction-with-Extensions"></a><h2>Interaction with Extensions</h2>
<p>When Perl exchanges data with an extension, the extension should be
able to understand the UTF8 flag and act accordingly. If the
extension doesn't recognize that flag, it's likely that the extension
will return incorrectly-flagged data.</p>
<p>So if you're working with Unicode data, consult the documentation of
every module you're using if there are any issues with Unicode data
exchange. If the documentation does not talk about Unicode at all,
suspect the worst and probably look at the source to learn how the
module is implemented. Modules written completely in Perl shouldn't
cause problems. Modules that directly or indirectly access code written
in other programming languages are at risk.</p>
<p>For affected functions, the simple strategy to avoid data corruption is
to always make the encoding of the exchanged data explicit. Choose an
encoding that you know the extension can handle. Convert arguments passed
to the extensions to that encoding and convert results back from that
encoding. Write wrapper functions that do the conversions for you, so
you can later change the functions when the extension catches up.</p>
<p>To provide an example, let's say the popular <code class="inline"><span class="w">Foo::Bar::escape_html</span></code>

function doesn't deal with Unicode data yet. The wrapper function
would convert the argument to raw UTF-8 and convert the result back to
Perl's internal representation like so:</p>
<pre class="verbatim"><ol><li><a name="my_escape_html"></a>    sub <span class="m">my_escape_html ($)</span> <span class="s">{</span></li><li>        <a class="l_k" href="functions/my.html">my</a><span class="s">(</span><span class="i">$what</span><span class="s">)</span> = <a class="l_k" href="functions/shift.html">shift</a><span class="sc">;</span></li><li>        <a class="l_k" href="functions/return.html">return</a> <a class="l_k" href="functions/unless.html">unless</a> <a class="l_k" href="functions/defined.html">defined</a> <span class="i">$what</span><span class="sc">;</span></li><li>        <span class="i">Encode::decode</span><span class="s">(</span><span class="q">&quot;UTF-8&quot;</span><span class="cm">,</span> <span class="i">Foo::Bar::escape_html</span><span class="s">(</span></li><li>                                     <span class="i">Encode::encode</span><span class="s">(</span><span class="q">&quot;UTF-8&quot;</span><span class="cm">,</span> <span class="i">$what</span><span class="s">)</span><span class="s">)</span><span class="s">)</span><span class="sc">;</span></li><li>    <span class="s">}</span></li></ol></pre><p>Sometimes, when the extension does not convert data but just stores
and retrieves it, you will be able to use the otherwise
dangerous <a href="Encode.html#_utf8_on">Encode::_utf8_on() </a> function. Let's say
the popular <code class="inline"><span class="w">Foo::Bar</span></code>
 extension, written in C, provides a <code class="inline"><span class="w">param</span></code>

method that lets you store and retrieve data according to these prototypes:</p>
<pre class="verbatim"><ol><li>    <span class="i">$self</span><span class="i">-&gt;param</span><span class="s">(</span><span class="i">$name</span><span class="cm">,</span> <span class="i">$value</span><span class="s">)</span><span class="sc">;</span>            <span class="c"># set a scalar</span></li><li>    <span class="i">$value</span> = <span class="i">$self</span><span class="i">-&gt;param</span><span class="s">(</span><span class="i">$name</span><span class="s">)</span><span class="sc">;</span>           <span class="c"># retrieve a scalar</span></li></ol></pre><p>If it does not yet provide support for any encoding, one could write a
derived class with such a <code class="inline"><span class="w">param</span></code>
 method:</p>
<pre class="verbatim"><ol><li><a name="param"></a>    sub <span class="m">param</span> <span class="s">{</span></li><li>      <a class="l_k" href="functions/my.html">my</a><span class="s">(</span><span class="i">$self</span><span class="cm">,</span><span class="i">$name</span><span class="cm">,</span><span class="i">$value</span><span class="s">)</span> = <span class="i">@_</span><span class="sc">;</span></li><li>      <span class="i">utf8::upgrade</span><span class="s">(</span><span class="i">$name</span><span class="s">)</span><span class="sc">;</span>     <span class="c"># make sure it is UTF-8 encoded</span></li><li>      if <span class="s">(</span><a class="l_k" href="functions/defined.html">defined</a> <span class="i">$value</span><span class="s">)</span> <span class="s">{</span></li><li>        <span class="i">utf8::upgrade</span><span class="s">(</span><span class="i">$value</span><span class="s">)</span><span class="sc">;</span>  <span class="c"># make sure it is UTF-8 encoded</span></li><li>        <a class="l_k" href="functions/return.html">return</a> <span class="i">$self</span><span class="i">-&gt;SUPER::param</span><span class="s">(</span><span class="i">$name</span><span class="cm">,</span><span class="i">$value</span><span class="s">)</span><span class="sc">;</span></li><li>      <span class="s">}</span> else <span class="s">{</span></li><li>        <a class="l_k" href="functions/my.html">my</a> <span class="i">$ret</span> = <span class="i">$self</span><span class="i">-&gt;SUPER::param</span><span class="s">(</span><span class="i">$name</span><span class="s">)</span><span class="sc">;</span></li><li>        <span class="i">Encode::_utf8_on</span><span class="s">(</span><span class="i">$ret</span><span class="s">)</span><span class="sc">;</span> <span class="c"># we know, it is UTF-8 encoded</span></li><li>        <a class="l_k" href="functions/return.html">return</a> <span class="i">$ret</span><span class="sc">;</span></li><li>      <span class="s">}</span></li><li>    <span class="s">}</span></li></ol></pre><p>Some extensions provide filters on data entry/exit points, such as
<code class="inline"><span class="w">DB_File::filter_store_key</span></code>
 and family. Look out for such filters in
the documentation of your extensions; they can make the transition to
Unicode data much easier.</p>
<a name="Speed"></a><h2>Speed</h2>
<p>Some functions are slower when working on UTF-8 encoded strings than
on byte encoded strings.  All functions that need to hop over
characters such as <code class="inline"><a class="l_k" href="functions/length.html">length()</a></code>, <code class="inline"><a class="l_k" href="functions/substr.html">substr()</a></code> or <code class="inline"><a class="l_k" href="functions/index.html">index()</a></code>, or matching
regular expressions can work <b>much</b> faster when the underlying data are
byte-encoded.</p>
<p>In Perl 5.8.0 the slowness was often quite spectacular; in Perl 5.8.1
a caching scheme was introduced which improved the situation.  In general,
operations with UTF-8 encoded strings are still slower. As an example,
the Unicode properties (character classes) like <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">Nd</span><span class="s">}</span></code>
 are known to
be quite a bit slower (5-20 times) than their simpler counterparts
like <code class="inline"><span class="s">[</span><span class="n">0</span>-<span class="n">9</span><span class="s">]</span></code>
 (then again, there are hundreds of Unicode characters matching
<code class="inline"><span class="w">Nd</span></code>
 compared with the 10 ASCII characters matching <code class="inline"><span class="s">[</span><span class="n">0</span>-<span class="n">9</span><span class="s">]</span></code>
).</p>
<a name="SEE-ALSO"></a><h1>SEE ALSO</h1>
<p><a href="perlunitut.html">perlunitut</a>, <a href="perluniintro.html">perluniintro</a>, <a href="perluniprops.html">perluniprops</a>, <a href="Encode.html">Encode</a>, <a href="open.html">open</a>, <a href="utf8.html">utf8</a>, <a href="bytes.html">bytes</a>,
<a href="perlretut.html">perlretut</a>, <a href="perlvar.html#%24%7b%5eUNICODE%7d">${^UNICODE} in perlvar</a>,
<a href="http://www.unicode.org/reports/tr44">http://www.unicode.org/reports/tr44</a>).</p>




  <div id="page_index" class="hud_container">
    <div id="page_index_header" class="hud_header">
      <div id="page_index_close" class="hud_close"><a href="#" onClick="pageIndex.hide();return false;"></a></div>
      <div id="page_index_title" class="hud_title"><span class="hud_span_top">Page index</span></div>
      <div id="page_index_topright" class="hud_topright"></div>
    </div>
    <div id="page_index_content" class="hud_content">
      <ul><li><a href="#NAME">NAME</a><li><a href="#DESCRIPTION">DESCRIPTION</a><ul><li><a href="#Important-Caveats">Important Caveats</a><li><a href="#Byte-and-Character-Semantics">Byte and Character Semantics</a><li><a href="#ASCII-Rules-versus-Unicode-Rules">ASCII Rules versus Unicode Rules</a><li><a href="#Extended-Grapheme-Clusters-(Logical-characters)">Extended Grapheme Clusters (Logical characters)</a><li><a href="#Unicode-Character-Properties">Unicode Character Properties</a><li><a href="#User-Defined-Character-Properties">User-Defined Character Properties</a><li><a href="#User-Defined-Case-Mappings-(for-serious-hackers-only)">User-Defined Case Mappings (for serious hackers only)</a><li><a href="#Character-Encodings-for-Input-and-Output">Character Encodings for Input and Output</a><li><a href="#Unicode-Regular-Expression-Support-Level">Unicode Regular Expression Support Level</a><li><a href="#Unicode-Encodings">Unicode Encodings</a><li><a href="#Noncharacter-code-points">Noncharacter code points</a><li><a href="#Beyond-Unicode-code-points">Beyond Unicode code points</a><li><a href="#Security-Implications-of-Unicode">Security Implications of Unicode</a><li><a href="#Unicode-in-Perl-on-EBCDIC">Unicode in Perl on EBCDIC</a><li><a href="#Locales">Locales</a><li><a href="#When-Unicode-Does-Not-Happen">When Unicode Does Not Happen</a><li><a href="#The-%22Unicode-Bug%22">The "Unicode Bug"</a><li><a href="#Forcing-Unicode-in-Perl-(Or-Unforcing-Unicode-in-Perl)">Forcing Unicode in Perl (Or Unforcing Unicode in Perl)</a><li><a href="#Using-Unicode-in-XS">Using Unicode in XS</a><li><a href="#Hacking-Perl-to-work-on-earlier-Unicode-versions-(for-very-serious-hackers-only)">Hacking Perl to work on earlier Unicode versions (for very serious hackers only)</a><li><a href="#Porting-code-from-perl-5.6.X">Porting code from perl-5.6.X</a></ul><li><a href="#BUGS">BUGS</a><ul><li><a href="#Interaction-with-Extensions">Interaction with Extensions</a><li><a href="#Speed">Speed</a></ul><li><a href="#SEE-ALSO">SEE ALSO</a></ul>
    </div>
    <div id="page_index_footer" class="hud_footer">
      <div id="page_index_bottomleft" class="hud_bottomleft"></div>
      <div id="page_index_bottom" class="hud_bottom"><span class="hud_span_bottom"></span></div>
      <div id="page_index_resize" class="hud_resize"></div>
    </div>
  </div>


	    &nbsp;
          </div>
          <div id="content_footer">
          </div>
        </div>
        <div class="clear"></div>
      </div>
      
    <div id="footer">
      <div id="footer_content">
        <div id="footer_strapline">
          perldoc.perl.org - Official documentation for the Perl programming language
        </div>
        <div id="footer_links">
          <div id="address">
            <p class="name">Contact details</p>
            <p class="address">
	      Site maintained by <a href="mailto:jj@jonallen.info">Jon Allen (JJ)</a><br>
	    </p>
            <p class="contact">
              Documentation maintained by the <a href="http://lists.cpan.org/showlist.cgi?name=perl5-porters">Perl 5 Porters</a>
            </p>
          </div>
          <ul class="f1">
            <li>Manual
              <ul class="f2">
                <li><a href="index-overview.html">Overview</a>
                <li><a href="index-tutorials.html">Tutorials</a>
                <li><a href="index-faq.html">FAQs</a>
                <li><a href="index-history.html">Changes</a>
              </ul>
            <li>Reference
              <ul class="f2">
                <li><a href="index-language.html">Language</a>
                <li><a href="index-functions.html">Functions</a>
                <li><a href="perlop.html">Operators</a>
                <li><a href="perlvar.html">Variables</a>
              </ul>
            <li>Modules
              <ul class="f2">
                <li><a href="index-modules-A.html">Modules</a>
                <li><a href="index-pragmas.html">Pragmas</a>
                <li><a href="index-utilities.html">Utilities</a>
              </ul>
            <li>Misc
              <ul class="f2">
                <li><a href="index-licence.html">License</a>
                <li><a href="index-internals.html">Internals</a>
                <li><a href="index-platforms.html">Platforms</a>
              </ul>          </ul>
          <div class="clear"></div>
        </div>
      </div>
      <div id="footer_end">
      </div>
    </div>
      
    </div>
      <script language="JavaScript" type="text/javascript" src="static/exploreperl.js"></script>
      <script language="JavaScript" src="static/combined-20100403.js" type="text/javascript"></script>
<script language="JavaScript" type="text/javascript">
  perldoc.setPath(0);
  perldoc.pageName    = 'perlunicode';
  perldoc.pageAddress = 'perlunicode.html';
  perldoc.contentPage = 1;
  explorePerl.render();
  explorePerl.addEvents('explore_anchor');
</script>
    
  </body>
</html>