File: perluniintro.html

package info (click to toggle)
perl-doc-html 5.26.0-4
  • links: PTS, VCS
  • area: main
  • in suites: sid
  • size: 39,400 kB
  • sloc: xml: 36; makefile: 7
file content (1238 lines) | stat: -rw-r--r-- 94,325 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
"http://www.w3.org/TR/html4/loose.dtd">
<html>
<head>
  <title>perluniintro - perldoc.perl.org</title>
  <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
  <meta http-equiv="Content-Language" content="en-gb">
  <link rel="search" type="application/opensearchdescription+xml" title="Search perldoc.perl.org" href="/static/search.xml"/>
  <link href="static/css-20100830.css" rel="stylesheet" rev="stylesheet" type="text/css" media="screen">
  <link href="static/exploreperl.css" rel="stylesheet" rev="stylesheet" type="text/css">
</head>

<body onLoad="perldoc.startup();" onPageShow="if (event.persisted) perldoc.startup();">
    <div id="page">
      
      <div id="header">
	<div id="homepage_link">
	  <a href="index.html"></a>
	</div>
	<div id="strapline">
	  Perl Programming Documentation
	</div>
	<div id="download_link" class="download">
	  <a href="http://www.perl.org/get.html">Download Perl</a>
	</div>
	<div id="explore_link" class="download">
	  <a id="explore_anchor" href="#">Explore</a>
	</div>
      </div>
      
      <div id="body">
        <div id="left_column">
          <div class="side_group">
            
	    <div class="side_panel doc_panel">
              <p>Manual</p>
              <ul>
                <li><a href="index-overview.html">Overview</a>
                <li><a href="index-tutorials.html">Tutorials</a>
                <li><a href="index-faq.html">FAQs</a>
                <li><a href="index-history.html">History / Changes</a>
                <li><a href="index-licence.html">License</a>
              </ul>
            </div>
            <div class="side_panel doc_panel">
              <p>Reference</p>
              <ul>
                <li><a href="index-language.html">Language</a>
                <li><a href="index-functions.html">Functions</a>
                <li><a href="perlop.html">Operators</a>
                <li><a href="perlvar.html">Special Variables</a>
                <li><a href="index-pragmas.html">Pragmas</a>
                <li><a href="index-utilities.html">Utilities</a>
                <li><a href="index-internals.html">Internals</a>
                <li><a href="index-platforms.html">Platform Specific</a>
              </ul>
            </div>
            <div class="side_panel doc_panel">
              <p>Modules</p>
              <ul>
		<li>
		
                
                  
		    
		  
		
                  
		    
		  
		
                  
		    
		  
		
                  
		    
		  
		
                  
		    
		  
		
                  
		    
		  
		
                  
		    
		  
		
                  
		    
		  
		
                  
		    
		  
		
                  
		
                  
		
                  
		    
		  
		
                  
		    
		  
		
                  
		    
		  
		
                  
		    
		  
		
                  
		    
		  
		
                  
		
                  
		
                  
		    
		  
		
                  
		    
		  
		
                  
		    
		  
		
                  
		
                  
		
                  
		    
		  
		
                  
		
                  
		
		
                    <a href="index-modules-A.html">A</a>
                    
                      
                        &bull;
                      
                    
                
                    <a href="index-modules-B.html">B</a>
                    
                      
                        &bull;
                      
                    
                
                    <a href="index-modules-C.html">C</a>
                    
                      
                        &bull;
                      
                    
                
                    <a href="index-modules-D.html">D</a>
                    
                      
                        &bull;
                      
                    
                
                    <a href="index-modules-E.html">E</a>
                    
                      
                        <li>
                      
                    
                
                    <a href="index-modules-F.html">F</a>
                    
                      
                        &bull;
                      
                    
                
                    <a href="index-modules-G.html">G</a>
                    
                      
                        &bull;
                      
                    
                
                    <a href="index-modules-H.html">H</a>
                    
                      
                        &bull;
                      
                    
                
                    <a href="index-modules-I.html">I</a>
                    
                      
                        &bull;
                      
                    
                
                    <a href="index-modules-L.html">L</a>
                    
                      
                        <li>
                      
                    
                
                    <a href="index-modules-M.html">M</a>
                    
                      
                        &bull;
                      
                    
                
                    <a href="index-modules-N.html">N</a>
                    
                      
                        &bull;
                      
                    
                
                    <a href="index-modules-O.html">O</a>
                    
                      
                        &bull;
                      
                    
                
                    <a href="index-modules-P.html">P</a>
                    
                      
                        &bull;
                      
                    
                
                    <a href="index-modules-S.html">S</a>
                    
                      
                        <li>
                      
                    
                
                    <a href="index-modules-T.html">T</a>
                    
                      
                        &bull;
                      
                    
                
                    <a href="index-modules-U.html">U</a>
                    
                      
                        &bull;
                      
                    
                
                    <a href="index-modules-X.html">X</a>
                    
                
              </ul>
            </div>
            
	      <div class="side_panel doc_panel">
		<p>Tools</p>
		<ul>
		  <li><a href="preferences.html">Preferences</a>
		</ul>
	      </div>
            
          </div>
        </div>
        <div id="centre_column">
          <div id="content_header">
            <div id="title_bar">
              <div id="page_name">
                <h1>perluniintro</h1>
              </div>
              <div id="perl_version">
                Perl 5 version 26.0 documentation
              </div>
              <div class="page_links" id="page_links_top">
                <a href="#" onClick="toolbar.goToTop();return false;">Go to top</a>
		
              </div>
	      <div class="page_links" id="page_links_bottom">
		
                  <a href="#" id="page_index_toggle">Show page index</a> &bull;
		
                <a href="#" id="recent_pages_toggle">Show recent pages</a>		
	      </div>
	      <div id="search_form">
		<form action="search.html" method="GET" id="search">
		  <input type="text" name="q" id="search_box" alt="Search">
		</form>
	      </div>
            </div>
            <div id="breadcrumbs">
                
    <a href="index.html">Home</a> &gt;
    
      
        <a href="index-language.html">Language reference</a> &gt;
      
    
    perluniintro
  

            </div>
          </div>
          <div id="content_body">
	    <!--[if lt IE 7]>
 <div class="noscript">
   <p>
     <strong>It looks like you're using Internet Explorer 6. This is a very old
     browser which does not offer full support for modern websites.</strong>
   </p>
   <p>
     Unfortunately this means that this website will not work on
     your computer.
   </p>
   <p>
     Don't miss out though! To view the site (and get a better experience from
     many other websites), simply upgrade to
     <a href="http://www.microsoft.com/windows/Internet-explorer/default.aspx">Internet
Explorer 8</a>
     or download an alternative browser such as
     <a href="http://www.mozilla.com/en-US/firefox/firefox.html">Firefox</a>,
     <a href="http://www.apple.com/safari/download/">Safari</a>, or
     <a href="http://www.google.co.uk/chrome">Google Chrome</a>.
   </p>
   <p>
     All of these browsers are free. If you're using a PC at work, you may
     need to contact your IT administrator.
   </p>
 </div>
<![endif]-->
	    <noscript>
	      <div class="noscript">
	      <p>
                <strong>Please note: Many features of this site require JavaScript. You appear to have JavaScript disabled,
	        or are running a non-JavaScript capable web browser.</strong>
	      </p>
	      <p>
		To get the best experience, please enable JavaScript or download a modern web browser such as <a href="http://www.microsoft.com/windows/Internet-explorer/default.aspx">Internet Explorer 8</a>, <a href="http://www.mozilla.com/en-US/firefox/firefox.html">Firefox</a>, <a href="http://www.apple.com/safari/download/">Safari</a>, or <a href="http://www.google.co.uk/chrome">Google Chrome</a>.
              </p>
	      </div>
	    </noscript>

	    <div id="recent_pages" class="hud_container">
	      <div id="recent_pages_header" class="hud_header">
		<div id="recent_pages_close" class="hud_close"><a href="#" onClick="recentPages.hide();return false;"></a></div>
		<div id="recent_pages_title" class="hud_title"><span class="hud_span_top">Recently read</span></div>
		<div id="recent_pages_topright" class="hud_topright"></div>
	      </div>
	      <div id="recent_pages_content" class="hud_content">
	      </div>
	      <div id="recent_pages_footer" class="hud_footer">
		<div id="recent_pages_bottomleft" class="hud_bottomleft"></div>
		<div id="recent_pages_bottom" class="hud_bottom"><span class="hud_span_bottom"></span></div>
		<div id="recent_pages_resize" class="hud_resize"></div>
	      </div>
	    </div>
  
	    <div id="from_search"></div>
            <h1>perluniintro</h1>


  <!--    -->
<ul><li><a href="#NAME">NAME</a><li><a href="#DESCRIPTION">DESCRIPTION</a><ul><li><a href="#Unicode">Unicode</a><li><a href="#Perl's-Unicode-Support">Perl's Unicode Support</a><li><a href="#Perl's-Unicode-Model">Perl's Unicode Model</a><li><a href="#Unicode-and-EBCDIC">Unicode and EBCDIC</a><li><a href="#Creating-Unicode">Creating Unicode</a><li><a href="#Handling-Unicode">Handling Unicode</a><li><a href="#Legacy-Encodings">Legacy Encodings</a><li><a href="#Unicode-I%2fO">Unicode I/O</a><li><a href="#Displaying-Unicode-As-Text">Displaying Unicode As Text</a><li><a href="#Special-Cases">Special Cases</a><li><a href="#Advanced-Topics">Advanced Topics</a><li><a href="#Miscellaneous">Miscellaneous</a><li><a href="#Questions-With-Answers">Questions With Answers</a><li><a href="#Hexadecimal-Notation">Hexadecimal Notation</a><li><a href="#Further-Resources">Further Resources</a></ul><li><a href="#UNICODE-IN-OLDER-PERLS">UNICODE IN OLDER PERLS</a><li><a href="#SEE-ALSO">SEE ALSO</a><li><a href="#ACKNOWLEDGMENTS">ACKNOWLEDGMENTS</a><li><a href="#AUTHOR%2c-COPYRIGHT%2c-AND-LICENSE">AUTHOR, COPYRIGHT, AND LICENSE</a></ul><a name="NAME"></a><h1>NAME</h1>
<p>perluniintro - Perl Unicode introduction</p>
<a name="DESCRIPTION"></a><h1>DESCRIPTION</h1>
<p>This document gives a general idea of Unicode and how to use Unicode
in Perl.  See <a href="#Further-Resources">Further Resources</a> for references to more in-depth
treatments of Unicode.</p>
<a name="Unicode"></a><h2>Unicode</h2>
<p>Unicode is a character set standard which plans to codify all of the
writing systems of the world, plus many other symbols.</p>
<p>Unicode and ISO/IEC 10646 are coordinated standards that unify
almost all other modern character set standards,
covering more than 80 writing systems and hundreds of languages,
including all commercially-important modern languages.  All characters
in the largest Chinese, Japanese, and Korean dictionaries are also
encoded. The standards will eventually cover almost all characters in
more than 250 writing systems and thousands of languages.
Unicode 1.0 was released in October 1991, and 6.0 in October 2010.</p>
<p>A Unicode <i>character</i> is an abstract entity.  It is not bound to any
particular integer width, especially not to the C language <code class="inline"><span class="w">char</span></code>
.
Unicode is language-neutral and display-neutral: it does not encode the
language of the text, and it does not generally define fonts or other graphical
layout details.  Unicode operates on characters and on text built from
those characters.</p>
<p>Unicode defines characters like <code class="inline"><span class="w">LATIN</span> <span class="w">CAPITAL</span> <span class="w">LETTER</span> <span class="w">A</span></code>
 or <code class="inline"><span class="w">GREEK</span>
<span class="w">SMALL</span> <span class="w">LETTER</span> <span class="w">ALPHA</span></code>
 and unique numbers for the characters, in this
case 0x0041 and 0x03B1, respectively.  These unique numbers are called
<i>code points</i>.  A code point is essentially the position of the
character within the set of all possible Unicode characters, and thus in
Perl, the term <i>ordinal</i> is often used interchangeably with it.</p>
<p>The Unicode standard prefers using hexadecimal notation for the code
points.  If numbers like <code class="inline"><span class="n">0x0041</span></code>
 are unfamiliar to you, take a peek
at a later section, <a href="#Hexadecimal-Notation">Hexadecimal Notation</a>.  The Unicode standard
uses the notation <code class="inline">U+0041 LATIN CAPITAL LETTER A</code>, to give the
hexadecimal code point and the normative name of the character.</p>
<p>Unicode also defines various <i>properties</i> for the characters, like
"uppercase" or "lowercase", "decimal digit", or "punctuation";
these properties are independent of the names of the characters.
Furthermore, various operations on the characters like uppercasing,
lowercasing, and collating (sorting) are defined.</p>
<p>A Unicode <i>logical</i> "character" can actually consist of more than one internal
<i>actual</i> "character" or code point.  For Western languages, this is adequately
modelled by a <i>base character</i> (like <code class="inline"><span class="w">LATIN</span> <span class="w">CAPITAL</span> <span class="w">LETTER</span> <span class="w">A</span></code>
) followed
by one or more <i>modifiers</i> (like <code class="inline"><span class="w">COMBINING</span> <span class="w">ACUTE</span> <span class="w">ACCENT</span></code>
).  This sequence of
base character and modifiers is called a <i>combining character
sequence</i>.  Some non-western languages require more complicated
models, so Unicode created the <i>grapheme cluster</i> concept, which was
later further refined into the <i>extended grapheme cluster</i>.  For
example, a Korean Hangul syllable is considered a single logical
character, but most often consists of three actual
Unicode characters: a leading consonant followed by an interior vowel followed
by a trailing consonant.</p>
<p>Whether to call these extended grapheme clusters "characters" depends on your
point of view. If you are a programmer, you probably would tend towards seeing
each element in the sequences as one unit, or "character".  However from
the user's point of view, the whole sequence could be seen as one
"character" since that's probably what it looks like in the context of the
user's language.  In this document, we take the programmer's point of
view: one "character" is one Unicode code point.</p>
<p>For some combinations of base character and modifiers, there are
<i>precomposed</i> characters.  There is a single character equivalent, for
example, for the sequence <code class="inline"><span class="w">LATIN</span> <span class="w">CAPITAL</span> <span class="w">LETTER</span> <span class="w">A</span></code>
 followed by
<code class="inline"><span class="w">COMBINING</span> <span class="w">ACUTE</span> <span class="w">ACCENT</span></code>
.  It is called  <code class="inline"><span class="w">LATIN</span> <span class="w">CAPITAL</span> <span class="w">LETTER</span> <span class="w">A</span> <span class="w">WITH</span>
<span class="w">ACUTE</span></code>
.  These precomposed characters are, however, only available for
some combinations, and are mainly meant to support round-trip
conversions between Unicode and legacy standards (like ISO 8859).  Using
sequences, as Unicode does, allows for needing fewer basic building blocks
(code points) to express many more potential grapheme clusters.  To
support conversion between equivalent forms, various <i>normalization
forms</i> are also defined.  Thus, <code class="inline"><span class="w">LATIN</span> <span class="w">CAPITAL</span> <span class="w">LETTER</span> <span class="w">A</span> <span class="w">WITH</span> <span class="w">ACUTE</span></code>
 is
in <i>Normalization Form Composed</i>, (abbreviated NFC), and the sequence
<code class="inline"><span class="w">LATIN</span> <span class="w">CAPITAL</span> <span class="w">LETTER</span> <span class="w">A</span></code>
 followed by <code class="inline"><span class="w">COMBINING</span> <span class="w">ACUTE</span> <span class="w">ACCENT</span></code>

represents the same character in <i>Normalization Form Decomposed</i> (NFD).</p>
<p>Because of backward compatibility with legacy encodings, the "a unique
number for every character" idea breaks down a bit: instead, there is
"at least one number for every character".  The same character could
be represented differently in several legacy encodings.  The
converse is not true: some code points do not have an assigned
character.  Firstly, there are unallocated code points within
otherwise used blocks.  Secondly, there are special Unicode control
characters that do not represent true characters.</p>
<p>When Unicode was first conceived, it was thought that all the world's
characters could be represented using a 16-bit word; that is a maximum of
<code class="inline"><span class="n">0x10000</span></code>
 (or 65,536) characters would be needed, from <code class="inline"><span class="n">0x0000</span></code>
 to
<code class="inline"><span class="n">0xFFFF</span></code>
.  This soon proved to be wrong, and since Unicode 2.0 (July
1996), Unicode has been defined all the way up to 21 bits (<code class="inline"><span class="n">0x10FFFF</span></code>
),
and Unicode 3.1 (March 2001) defined the first characters above <code class="inline"><span class="n">0xFFFF</span></code>
.
The first <code class="inline"><span class="n">0x10000</span></code>
 characters are called the <i>Plane 0</i>, or the
<i>Basic Multilingual Plane</i> (BMP).  With Unicode 3.1, 17 (yes,
seventeen) planes in all were defined--but they are nowhere near full of
defined characters, yet.</p>
<p>When a new language is being encoded, Unicode generally will choose a
<code class="inline"><span class="w">block</span></code>
 of consecutive unallocated code points for its characters.  So
far, the number of code points in these blocks has always been evenly
divisible by 16.  Extras in a block, not currently needed, are left
unallocated, for future growth.  But there have been occasions when
a later release needed more code points than the available extras, and a
new block had to allocated somewhere else, not contiguous to the initial
one, to handle the overflow.  Thus, it became apparent early on that
"block" wasn't an adequate organizing principle, and so the <code class="inline"><span class="w">Script</span></code>

property was created.  (Later an improved script property was added as
well, the <code class="inline"><span class="w">Script_Extensions</span></code>
 property.)  Those code points that are in
overflow blocks can still
have the same script as the original ones.  The script concept fits more
closely with natural language: there is <code class="inline"><span class="w">Latin</span></code>
 script, <code class="inline"><span class="w">Greek</span></code>

script, and so on; and there are several artificial scripts, like
<code class="inline"><span class="w">Common</span></code>
 for characters that are used in multiple scripts, such as
mathematical symbols.  Scripts usually span varied parts of several
blocks.  For more information about scripts, see <a href="perlunicode.html#Scripts">Scripts in perlunicode</a>.
The division into blocks exists, but it is almost completely
accidental--an artifact of how the characters have been and still are
allocated.  (Note that this paragraph has oversimplified things for the
sake of this being an introduction.  Unicode doesn't really encode
languages, but the writing systems for them--their scripts; and one
script can be used by many languages.  Unicode also encodes things that
aren't really about languages, such as symbols like <code class="inline"><span class="w">BAGGAGE</span> <span class="w">CLAIM</span></code>
.)</p>
<p>The Unicode code points are just abstract numbers.  To input and
output these abstract numbers, the numbers must be <i>encoded</i> or
<i>serialised</i> somehow.  Unicode defines several <i>character encoding
forms</i>, of which <i>UTF-8</i> is the most popular.  UTF-8 is a
variable length encoding that encodes Unicode characters as 1 to 4
bytes.  Other encodings
include UTF-16 and UTF-32 and their big- and little-endian variants
(UTF-8 is byte-order independent).  The ISO/IEC 10646 defines the UCS-2
and UCS-4 encoding forms.</p>
<p>For more information about encodings--for instance, to learn what
<i>surrogates</i> and <i>byte order marks</i> (BOMs) are--see <a href="perlunicode.html">perlunicode</a>.</p>
<a name="Perl's-Unicode-Support"></a><h2>Perl's Unicode Support</h2>
<p>Starting from Perl v5.6.0, Perl has had the capacity to handle Unicode
natively.  Perl v5.8.0, however, is the first recommended release for
serious Unicode work.  The maintenance release 5.6.1 fixed many of the
problems of the initial Unicode implementation, but for example
regular expressions still do not work with Unicode in 5.6.1.
Perl v5.14.0 is the first release where Unicode support is
(almost) seamlessly integrable without some gotchas. (There are two
exceptions. Firstly, some differences in <a href="functions/quotemeta.html">quotemeta</a>
were fixed starting in Perl 5.16.0. Secondly, some differences in
<a href="perlop.html#Range-Operators">the range operator</a> were fixed starting in
Perl 5.26.0.)</p>
<p>To enable this
seamless support, you should <code class="inline"><a class="l_k" href="functions/use.html">use</a> <span class="w">feature</span> <span class="q">&#39;unicode_strings&#39;</span></code>
 (which is
automatically selected if you <code class="inline"><a class="l_k" href="functions/use.html">use</a> <span class="n">5.012</span></code>
 or higher).  See <a href="feature.html">feature</a>.
(5.14 also fixes a number of bugs and departures from the Unicode
standard.)</p>
<p>Before Perl v5.8.0, the use of <code class="inline"><a class="l_k" href="functions/use.html">use</a> <span class="w">utf8</span></code>
 was used to declare
that operations in the current block or file would be Unicode-aware.
This model was found to be wrong, or at least clumsy: the "Unicodeness"
is now carried with the data, instead of being attached to the
operations.
Starting with Perl v5.8.0, only one case remains where an explicit <code class="inline"><a class="l_k" href="functions/use.html">use</a>
<span class="w">utf8</span></code>
 is needed: if your Perl script itself is encoded in UTF-8, you can
use UTF-8 in your identifier names, and in string and regular expression
literals, by saying <code class="inline"><a class="l_k" href="functions/use.html">use</a> <span class="w">utf8</span></code>
.  This is not the default because
scripts with legacy 8-bit data in them would break.  See <a href="utf8.html">utf8</a>.</p>
<a name="Perl's-Unicode-Model"></a><h2>Perl's Unicode Model</h2>
<p>Perl supports both pre-5.6 strings of eight-bit native bytes, and
strings of Unicode characters.  The general principle is that Perl tries
to keep its data as eight-bit bytes for as long as possible, but as soon
as Unicodeness cannot be avoided, the data is transparently upgraded
to Unicode.  Prior to Perl v5.14.0, the upgrade was not completely
transparent (see <a href="perlunicode.html#The-%22Unicode-Bug%22">The Unicode Bug in perlunicode</a>), and for backwards
compatibility, full transparency is not gained unless <code class="inline"><a class="l_k" href="functions/use.html">use</a> <span class="w">feature</span>
<span class="q">&#39;unicode_strings&#39;</span></code>
 (see <a href="feature.html">feature</a>) or <code class="inline"><a class="l_k" href="functions/use.html">use</a> <span class="n">5.012</span></code>
 (or higher) is
selected.</p>
<p>Internally, Perl currently uses either whatever the native eight-bit
character set of the platform (for example Latin-1) is, defaulting to
UTF-8, to encode Unicode strings. Specifically, if all code points in
the string are <code class="inline"><span class="n">0xFF</span></code>
 or less, Perl uses the native eight-bit
character set.  Otherwise, it uses UTF-8.</p>
<p>A user of Perl does not normally need to know nor care how Perl
happens to encode its internal strings, but it becomes relevant when
outputting Unicode strings to a stream without a PerlIO layer (one with
the "default" encoding).  In such a case, the raw bytes used internally
(the native character set or UTF-8, as appropriate for each string)
will be used, and a "Wide character" warning will be issued if those
strings contain a character beyond 0x00FF.</p>
<p>For example,</p>
<pre class="verbatim"><ol><li>      <span class="w">perl</span> -e <span class="q">&#39;print &quot;\x{DF}\n&quot;, &quot;\x{0100}\x{DF}\n&quot;&#39;</span></li></ol></pre><p>produces a fairly useless mixture of native bytes and UTF-8, as well
as a warning:</p>
<pre class="verbatim"><ol><li>     <span class="w">Wide</span> <span class="w">character</span> <span class="w">in</span> <a class="l_k" href="functions/print.html">print</a> <span class="i">at</span> ...</li></ol></pre><p>To output UTF-8, use the <code class="inline"><span class="j">:</span><span class="w">encoding</span></code>
 or <code class="inline"><span class="j">:</span><span class="w">utf8</span></code>
 output layer.  Prepending</p>
<pre class="verbatim"><ol><li>      <a class="l_k" href="functions/binmode.html">binmode</a><span class="s">(</span><span class="w">STDOUT</span><span class="cm">,</span> <span class="q">&quot;:utf8&quot;</span><span class="s">)</span><span class="sc">;</span></li></ol></pre><p>to this sample program ensures that the output is completely UTF-8,
and removes the program's warning.</p>
<p>You can enable automatic UTF-8-ification of your standard file
handles, default <code class="inline"><a class="l_k" href="functions/open.html">open()</a></code> layer, and <code class="inline"><span class="i">@ARGV</span></code>
 by using either
the <code class="inline">-C</code>
 command line switch or the <code class="inline"><span class="w">PERL_UNICODE</span></code>
 environment
variable, see <a href="perlrun.html">perlrun</a> for the documentation of the <code class="inline">-C</code>
 switch.</p>
<p>Note that this means that Perl expects other software to work the same
way:
if Perl has been led to believe that STDIN should be UTF-8, but then
STDIN coming in from another command is not UTF-8, Perl will likely
complain about the malformed UTF-8.</p>
<p>All features that combine Unicode and I/O also require using the new
PerlIO feature.  Almost all Perl 5.8 platforms do use PerlIO, though:
you can see whether yours is by running "perl -V" and looking for
<code class="inline"><span class="w">useperlio</span>=<span class="w">define</span></code>
.</p>
<a name="Unicode-and-EBCDIC"></a><h2>Unicode and EBCDIC</h2>
<p>Perl 5.8.0 added support for Unicode on EBCDIC platforms.  This support
was allowed to lapse in later releases, but was revived in 5.22.
Unicode support is somewhat more complex to implement since additional
conversions are needed.  See <a href="perlebcdic.html">perlebcdic</a> for more information.</p>
<p>On EBCDIC platforms, the internal Unicode encoding form is UTF-EBCDIC
instead of UTF-8.  The difference is that as UTF-8 is "ASCII-safe" in
that ASCII characters encode to UTF-8 as-is, while UTF-EBCDIC is
"EBCDIC-safe", in that all the basic characters (which includes all
those that have ASCII equivalents (like <code class="inline"><span class="q">&quot;A&quot;</span></code>
, <code class="inline"><span class="q">&quot;0&quot;</span></code>
, <code class="inline"><span class="q">&quot;%&quot;</span></code>
, <i>etc.</i>)
are the same in both EBCDIC and UTF-EBCDIC.  Often, documentation
will use the term "UTF-8" to mean UTF-EBCDIC as well.  This is the case
in this document.</p>
<a name="Creating-Unicode"></a><h2>Creating Unicode</h2>
<p>This section applies fully to Perls starting with v5.22.  Various
caveats for earlier releases are in the <a href="#Earlier-releases-caveats">Earlier releases caveats</a>
subsection below.</p>
<p>To create Unicode characters in literals,
use the <code class="inline">\<span class="i">N</span><span class="s">{</span>...<span class="s">}</span></code>
 notation in double-quoted strings:</p>
<pre class="verbatim"><ol><li> <a class="l_k" href="functions/my.html">my</a> <span class="i">$smiley_from_name</span> = <span class="q">&quot;\N{WHITE SMILING FACE}&quot;</span><span class="sc">;</span></li><li> <a class="l_k" href="functions/my.html">my</a> <span class="i">$smiley_from_code_point</span> = <span class="q">&quot;\N{U+263a}&quot;</span><span class="sc">;</span></li></ol></pre><p>Similarly, they can be used in regular expression literals</p>
<pre class="verbatim"><ol><li> <span class="i">$smiley</span> =~ <span class="q">/\N{WHITE SMILING FACE}/</span><span class="sc">;</span></li><li> <span class="i">$smiley</span> =~ <span class="q">/\N{U+263a}/</span><span class="sc">;</span></li></ol></pre><p>At run-time you can use:</p>
<pre class="verbatim"><ol><li> <a class="l_k" href="functions/use.html">use</a> <span class="w">charnames</span> <span class="s">(</span><span class="s">)</span><span class="sc">;</span></li><li> <a class="l_k" href="functions/my.html">my</a> <span class="i">$hebrew_alef_from_name</span></li><li>                      = <span class="i">charnames::string_vianame</span><span class="s">(</span><span class="q">&quot;HEBREW LETTER ALEF&quot;</span><span class="s">)</span><span class="sc">;</span></li><li> <a class="l_k" href="functions/my.html">my</a> <span class="i">$hebrew_alef_from_code_point</span> = <span class="i">charnames::string_vianame</span><span class="s">(</span><span class="q">&quot;U+05D0&quot;</span><span class="s">)</span><span class="sc">;</span></li></ol></pre><p>Naturally, <code class="inline"><a class="l_k" href="functions/ord.html">ord()</a></code> will do the reverse: it turns a character into
a code point.</p>
<p>There are other runtime options as well.  You can use <code class="inline"><a class="l_k" href="functions/pack.html">pack()</a></code>:</p>
<pre class="verbatim"><ol><li> <a class="l_k" href="functions/my.html">my</a> <span class="i">$hebrew_alef_from_code_point</span> = <a class="l_k" href="functions/pack.html">pack</a><span class="s">(</span><span class="q">&quot;U&quot;</span><span class="cm">,</span> <span class="n">0x05d0</span><span class="s">)</span><span class="sc">;</span></li></ol></pre><p>Or you can use <code class="inline"><a class="l_k" href="functions/chr.html">chr()</a></code>, though it is less convenient in the general
case:</p>
<pre class="verbatim"><ol><li> <span class="i">$hebrew_alef_from_code_point</span> = <a class="l_k" href="functions/chr.html">chr</a><span class="s">(</span><span class="i">utf8::unicode_to_native</span><span class="s">(</span><span class="n">0x05d0</span><span class="s">)</span><span class="s">)</span><span class="sc">;</span></li><li> <span class="i">utf8::upgrade</span><span class="s">(</span><span class="i">$hebrew_alef_from_code_point</span><span class="s">)</span><span class="sc">;</span></li></ol></pre><p>The <code class="inline"><span class="i">utf8::unicode_to_native</span><span class="s">(</span><span class="s">)</span></code>
 and <code class="inline"><span class="i">utf8::upgrade</span><span class="s">(</span><span class="s">)</span></code>
 aren't needed if
the argument is above 0xFF, so the above could have been written as</p>
<pre class="verbatim"><ol><li> <span class="i">$hebrew_alef_from_code_point</span> = <a class="l_k" href="functions/chr.html">chr</a><span class="s">(</span><span class="n">0x05d0</span><span class="s">)</span><span class="sc">;</span></li></ol></pre><p>since 0x5d0 is above 255.</p>
<p><code class="inline">\<span class="w">x</span><span class="s">{</span><span class="s">}</span></code>
 and <code class="inline">\<span class="w">o</span><span class="s">{</span><span class="s">}</span></code>
 can also be used to specify code points at compile
time in double-quotish strings, but, for backward compatibility with
older Perls, the same rules apply as with <code class="inline"><a class="l_k" href="functions/chr.html">chr()</a></code> for code points less
than 256.</p>
<p><code class="inline"><span class="i">utf8::unicode_to_native</span><span class="s">(</span><span class="s">)</span></code>
 is used so that the Perl code is portable
to EBCDIC platforms.  You can omit it if you're <i>really</i> sure no one
will ever want to use your code on a non-ASCII platform.  Starting in
Perl v5.22, calls to it on ASCII platforms are optimized out, so there's
no performance penalty at all in adding it.  Or you can simply use the
other constructs that don't require it.</p>
<p>See <a href="#Further-Resources">Further Resources</a> for how to find all these names and numeric
codes.</p>
<a name="Earlier-releases-caveats"></a><h3>Earlier releases caveats</h3>
<p>On EBCDIC platforms, prior to v5.22, using <code class="inline">\<span class="i">N</span><span class="s">{</span><span class="w">U</span>+...<span class="s">}</span></code>
 doesn't work
properly.</p>
<p>Prior to v5.16, using <code class="inline">\<span class="i">N</span><span class="s">{</span>...<span class="s">}</span></code>
 with a character name (as opposed to a
<code class="inline"><span class="w">U</span>+...</code>
 code point) required a <code class="inline"><a class="l_k" href="functions/use.html">use</a> <span class="w">charnames</span> <span class="co">:</span><span class="w">full</span></code>
.</p>
<p>Prior to v5.14, there were some bugs in <code class="inline">\<span class="i">N</span><span class="s">{</span>...<span class="s">}</span></code>
 with a character name
(as opposed to a <code class="inline"><span class="w">U</span>+...</code>
 code point).</p>
<p><code class="inline"><span class="i">charnames::string_vianame</span><span class="s">(</span><span class="s">)</span></code>
 was introduced in v5.14.  Prior to that,
<code class="inline"><span class="i">charnames::vianame</span><span class="s">(</span><span class="s">)</span></code>
 should work, but only if the argument is of the
form <code class="inline"><span class="q">&quot;U+...&quot;</span></code>
.  Your best bet there for runtime Unicode by character
name is probably:</p>
<pre class="verbatim"><ol><li> <a class="l_k" href="functions/use.html">use</a> <span class="w">charnames</span> <span class="s">(</span><span class="s">)</span><span class="sc">;</span></li><li> <a class="l_k" href="functions/my.html">my</a> <span class="i">$hebrew_alef_from_name</span></li><li>                  = <a class="l_k" href="functions/pack.html">pack</a><span class="s">(</span><span class="q">&quot;U&quot;</span><span class="cm">,</span> <span class="i">charnames::vianame</span><span class="s">(</span><span class="q">&quot;HEBREW LETTER ALEF&quot;</span><span class="s">)</span><span class="s">)</span><span class="sc">;</span></li></ol></pre><a name="Handling-Unicode"></a><h2>Handling Unicode</h2>
<p>Handling Unicode is for the most part transparent: just use the
strings as usual.  Functions like <code class="inline"><a class="l_k" href="functions/index.html">index()</a></code>, <code class="inline"><a class="l_k" href="functions/length.html">length()</a></code>, and
<code class="inline"><a class="l_k" href="functions/substr.html">substr()</a></code> will work on the Unicode characters; regular expressions
will work on the Unicode characters (see <a href="perlunicode.html">perlunicode</a> and <a href="perlretut.html">perlretut</a>).</p>
<p>Note that Perl considers grapheme clusters to be separate characters, so for
example</p>
<pre class="verbatim"><ol><li> <a class="l_k" href="functions/print.html">print</a> <a class="l_k" href="functions/length.html">length</a><span class="s">(</span><span class="q">&quot;\N{LATIN CAPITAL LETTER A}\N{COMBINING ACUTE ACCENT}&quot;</span><span class="s">)</span><span class="cm">,</span></li><li>       <span class="q">&quot;\n&quot;</span><span class="sc">;</span></li></ol></pre><p>will print 2, not 1.  The only exception is that regular expressions
have <code class="inline">\<span class="w">X</span></code>
 for matching an extended grapheme cluster.  (Thus <code class="inline">\<span class="w">X</span></code>
 in a
regular expression would match the entire sequence of both the example
characters.)</p>
<p>Life is not quite so transparent, however, when working with legacy
encodings, I/O, and certain special cases:</p>
<a name="Legacy-Encodings"></a><h2>Legacy Encodings</h2>
<p>When you combine legacy data and Unicode, the legacy data needs
to be upgraded to Unicode.  Normally the legacy data is assumed to be
ISO 8859-1 (or EBCDIC, if applicable).</p>
<p>The <code class="inline"><span class="w">Encode</span></code>
 module knows about many encodings and has interfaces
for doing conversions between those encodings:</p>
<pre class="verbatim"><ol><li>    <a class="l_k" href="functions/use.html">use</a> <span class="w">Encode</span> <span class="q">&#39;decode&#39;</span><span class="sc">;</span></li><li>    <span class="i">$data</span> = <span class="i">decode</span><span class="s">(</span><span class="q">&quot;iso-8859-3&quot;</span><span class="cm">,</span> <span class="i">$data</span><span class="s">)</span><span class="sc">;</span> <span class="c"># convert from legacy</span></li></ol></pre><a name="Unicode-I%2fO"></a><h2>Unicode I/O</h2>
<p>Normally, writing out Unicode data</p>
<pre class="verbatim"><ol><li>    <a class="l_k" href="functions/print.html">print</a> <span class="i">FH</span> <span class="i">$some_string_with_unicode</span><span class="cm">,</span> <span class="q">&quot;\n&quot;</span><span class="sc">;</span></li></ol></pre><p>produces raw bytes that Perl happens to use to internally encode the
Unicode string.  Perl's internal encoding depends on the system as
well as what characters happen to be in the string at the time. If
any of the characters are at code points <code class="inline"><span class="n">0x100</span></code>
 or above, you will get
a warning.  To ensure that the output is explicitly rendered in the
encoding you desire--and to avoid the warning--open the stream with
the desired encoding. Some examples:</p>
<pre class="verbatim"><ol><li>    <a class="l_k" href="functions/open.html">open</a> <span class="w">FH</span><span class="cm">,</span> <span class="q">&quot;&gt;:utf8&quot;</span><span class="cm">,</span> <span class="q">&quot;file&quot;</span><span class="sc">;</span></li><li></li><li>    <a class="l_k" href="functions/open.html">open</a> <span class="w">FH</span><span class="cm">,</span> <span class="q">&quot;&gt;:encoding(ucs2)&quot;</span><span class="cm">,</span>      <span class="q">&quot;file&quot;</span><span class="sc">;</span></li><li>    <a class="l_k" href="functions/open.html">open</a> <span class="w">FH</span><span class="cm">,</span> <span class="q">&quot;&gt;:encoding(UTF-8)&quot;</span><span class="cm">,</span>     <span class="q">&quot;file&quot;</span><span class="sc">;</span></li><li>    <a class="l_k" href="functions/open.html">open</a> <span class="w">FH</span><span class="cm">,</span> <span class="q">&quot;&gt;:encoding(shift_jis)&quot;</span><span class="cm">,</span> <span class="q">&quot;file&quot;</span><span class="sc">;</span></li></ol></pre><p>and on already open streams, use <code class="inline"><a class="l_k" href="functions/binmode.html">binmode()</a></code>:</p>
<pre class="verbatim"><ol><li>    <a class="l_k" href="functions/binmode.html">binmode</a><span class="s">(</span><span class="w">STDOUT</span><span class="cm">,</span> <span class="q">&quot;:utf8&quot;</span><span class="s">)</span><span class="sc">;</span></li><li></li><li>    <a class="l_k" href="functions/binmode.html">binmode</a><span class="s">(</span><span class="w">STDOUT</span><span class="cm">,</span> <span class="q">&quot;:encoding(ucs2)&quot;</span><span class="s">)</span><span class="sc">;</span></li><li>    <a class="l_k" href="functions/binmode.html">binmode</a><span class="s">(</span><span class="w">STDOUT</span><span class="cm">,</span> <span class="q">&quot;:encoding(UTF-8)&quot;</span><span class="s">)</span><span class="sc">;</span></li><li>    <a class="l_k" href="functions/binmode.html">binmode</a><span class="s">(</span><span class="w">STDOUT</span><span class="cm">,</span> <span class="q">&quot;:encoding(shift_jis)&quot;</span><span class="s">)</span><span class="sc">;</span></li></ol></pre><p>The matching of encoding names is loose: case does not matter, and
many encodings have several aliases.  Note that the <code class="inline"><span class="j">:</span><span class="w">utf8</span></code>
 layer
must always be specified exactly like that; it is <i>not</i> subject to
the loose matching of encoding names. Also note that currently <code class="inline"><span class="j">:</span><span class="w">utf8</span></code>
 is unsafe for
input, because it accepts the data without validating that it is indeed valid
UTF-8; you should instead use <code class="inline"><span class="j">:</span><span class="i">encoding</span><span class="s">(</span><span class="w">UTF</span>-<span class="n">8</span><span class="s">)</span></code>
 (with or without a
hyphen).</p>
<p>See <a href="PerlIO.html">PerlIO</a> for the <code class="inline"><span class="j">:</span><span class="w">utf8</span></code>
 layer, <a href="PerlIO/encoding.html">PerlIO::encoding</a> and
<a href="Encode/PerlIO.html">Encode::PerlIO</a> for the <code class="inline"><span class="j">:</span><span class="i">encoding</span><span class="s">(</span><span class="s">)</span></code>
 layer, and
<a href="Encode/Supported.html">Encode::Supported</a> for many encodings supported by the <code class="inline"><span class="w">Encode</span></code>

module.</p>
<p>Reading in a file that you know happens to be encoded in one of the
Unicode or legacy encodings does not magically turn the data into
Unicode in Perl's eyes.  To do that, specify the appropriate
layer when opening files</p>
<pre class="verbatim"><ol><li>    <a class="l_k" href="functions/open.html">open</a><span class="s">(</span><a class="l_k" href="functions/my.html">my</a> <span class="i">$fh</span><span class="cm">,</span><span class="q">&#39;&lt;:encoding(UTF-8)&#39;</span><span class="cm">,</span> <span class="q">&#39;anything&#39;</span><span class="s">)</span><span class="sc">;</span></li><li>    <a class="l_k" href="functions/my.html">my</a> <span class="i">$line_of_unicode</span> = <span class="q">&lt;$fh&gt;</span><span class="sc">;</span></li><li></li><li>    <a class="l_k" href="functions/open.html">open</a><span class="s">(</span><a class="l_k" href="functions/my.html">my</a> <span class="i">$fh</span><span class="cm">,</span><span class="q">&#39;&lt;:encoding(Big5)&#39;</span><span class="cm">,</span> <span class="q">&#39;anything&#39;</span><span class="s">)</span><span class="sc">;</span></li><li>    <a class="l_k" href="functions/my.html">my</a> <span class="i">$line_of_unicode</span> = <span class="q">&lt;$fh&gt;</span><span class="sc">;</span></li></ol></pre><p>The I/O layers can also be specified more flexibly with
the <code class="inline"><a class="l_k" href="functions/open.html">open</a></code> pragma.  See <a href="open.html">open</a>, or look at the following example.</p>
<pre class="verbatim"><ol><li>    <a class="l_k" href="functions/use.html">use</a> <a class="l_k" href="functions/open.html">open</a> <span class="q">&#39;:encoding(UTF-8)&#39;</span><span class="sc">;</span> <span class="c"># input/output default encoding will be</span></li><li>                                 <span class="c"># UTF-8</span></li><li>    <a class="l_k" href="functions/open.html">open</a> <span class="w">X</span><span class="cm">,</span> <span class="q">&quot;&gt;file&quot;</span><span class="sc">;</span></li><li>    <a class="l_k" href="functions/print.html">print</a> <span class="i">X</span> <a class="l_k" href="functions/chr.html">chr</a><span class="s">(</span><span class="n">0x100</span><span class="s">)</span><span class="cm">,</span> <span class="q">&quot;\n&quot;</span><span class="sc">;</span></li><li>    <a class="l_k" href="functions/close.html">close</a> <span class="w">X</span><span class="sc">;</span></li><li>    <a class="l_k" href="functions/open.html">open</a> <span class="w">Y</span><span class="cm">,</span> <span class="q">&quot;&lt;file&quot;</span><span class="sc">;</span></li><li>    <a class="l_k" href="functions/printf.html">printf</a> <span class="q">&quot;%#x\n&quot;</span><span class="cm">,</span> <a class="l_k" href="functions/ord.html">ord</a><span class="s">(</span><span class="q">&lt;Y&gt;</span><span class="s">)</span><span class="sc">;</span> <span class="c"># this should print 0x100</span></li><li>    <a class="l_k" href="functions/close.html">close</a> <span class="w">Y</span><span class="sc">;</span></li></ol></pre><p>With the <code class="inline"><a class="l_k" href="functions/open.html">open</a></code> pragma you can use the <code class="inline"><span class="j">:</span><span class="w">locale</span></code>
 layer</p>
<pre class="verbatim"><ol><li>    BEGIN <span class="s">{</span> <span class="i">$ENV</span>{<span class="w">LC_ALL</span>} = <span class="i">$ENV</span>{<span class="w">LANG</span>} = <span class="q">&#39;ru_RU.KOI8-R&#39;</span> <span class="s">}</span></li><li>    <span class="c"># the :locale will probe the locale environment variables like</span></li><li>    <span class="c"># LC_ALL</span></li><li>    <a class="l_k" href="functions/use.html">use</a> <a class="l_k" href="functions/open.html">open</a> <span class="w">OUT</span> <span class="cm">=&gt;</span> <span class="q">&#39;:locale&#39;</span><span class="sc">;</span> <span class="c"># russki parusski</span></li><li>    <a class="l_k" href="functions/open.html">open</a><span class="s">(</span><span class="w">O</span><span class="cm">,</span> <span class="q">&quot;&gt;koi8&quot;</span><span class="s">)</span><span class="sc">;</span></li><li>    <a class="l_k" href="functions/print.html">print</a> <span class="i">O</span> <a class="l_k" href="functions/chr.html">chr</a><span class="s">(</span><span class="n">0x430</span><span class="s">)</span><span class="sc">;</span> <span class="c"># Unicode CYRILLIC SMALL LETTER A = KOI8-R 0xc1</span></li><li>    <a class="l_k" href="functions/close.html">close</a> <span class="w">O</span><span class="sc">;</span></li><li>    <a class="l_k" href="functions/open.html">open</a><span class="s">(</span><span class="w">I</span><span class="cm">,</span> <span class="q">&quot;&lt;koi8&quot;</span><span class="s">)</span><span class="sc">;</span></li><li>    <a class="l_k" href="functions/printf.html">printf</a> <span class="q">&quot;%#x\n&quot;</span><span class="cm">,</span> <a class="l_k" href="functions/ord.html">ord</a><span class="s">(</span><span class="q">&lt;I&gt;</span><span class="s">)</span><span class="cm">,</span> <span class="q">&quot;\n&quot;</span><span class="sc">;</span> <span class="c"># this should print 0xc1</span></li><li>    <a class="l_k" href="functions/close.html">close</a> <span class="w">I</span><span class="sc">;</span></li></ol></pre><p>These methods install a transparent filter on the I/O stream that
converts data from the specified encoding when it is read in from the
stream.  The result is always Unicode.</p>
<p>The <a href="open.html">open</a> pragma affects all the <code class="inline"><a class="l_k" href="functions/open.html">open()</a></code> calls after the pragma by
setting default layers.  If you want to affect only certain
streams, use explicit layers directly in the <code class="inline"><a class="l_k" href="functions/open.html">open()</a></code> call.</p>
<p>You can switch encodings on an already opened stream by using
<code class="inline"><a class="l_k" href="functions/binmode.html">binmode()</a></code>; see <a href="functions/binmode.html">binmode</a>.</p>
<p>The <code class="inline"><span class="j">:</span><span class="w">locale</span></code>
 does not currently work with
<code class="inline"><a class="l_k" href="functions/open.html">open()</a></code> and <code class="inline"><a class="l_k" href="functions/binmode.html">binmode()</a></code>, only with the <code class="inline"><a class="l_k" href="functions/open.html">open</a></code> pragma.  The
<code class="inline"><span class="j">:</span><span class="w">utf8</span></code>
 and <code class="inline"><span class="j">:</span><span class="i">encoding</span><span class="s">(</span>...<span class="s">)</span></code>
 methods do work with all of <code class="inline"><a class="l_k" href="functions/open.html">open()</a></code>,
<code class="inline"><a class="l_k" href="functions/binmode.html">binmode()</a></code>, and the <code class="inline"><a class="l_k" href="functions/open.html">open</a></code> pragma.</p>
<p>Similarly, you may use these I/O layers on output streams to
automatically convert Unicode to the specified encoding when it is
written to the stream. For example, the following snippet copies the
contents of the file "text.jis" (encoded as ISO-2022-JP, aka JIS) to
the file "text.utf8", encoded as UTF-8:</p>
<pre class="verbatim"><ol><li>    <a class="l_k" href="functions/open.html">open</a><span class="s">(</span><a class="l_k" href="functions/my.html">my</a> <span class="i">$nihongo</span><span class="cm">,</span> <span class="q">&#39;&lt;:encoding(iso-2022-jp)&#39;</span><span class="cm">,</span> <span class="q">&#39;text.jis&#39;</span><span class="s">)</span><span class="sc">;</span></li><li>    <a class="l_k" href="functions/open.html">open</a><span class="s">(</span><a class="l_k" href="functions/my.html">my</a> <span class="i">$unicode</span><span class="cm">,</span> <span class="q">&#39;&gt;:utf8&#39;</span><span class="cm">,</span>                  <span class="q">&#39;text.utf8&#39;</span><span class="s">)</span><span class="sc">;</span></li><li>    while <span class="s">(</span><span class="q">&lt;$nihongo&gt;</span><span class="s">)</span> <span class="s">{</span> <a class="l_k" href="functions/print.html">print</a> <span class="i">$unicode</span> <span class="i">$_</span> <span class="s">}</span></li></ol></pre><p>The naming of encodings, both by the <code class="inline"><a class="l_k" href="functions/open.html">open()</a></code> and by the <code class="inline"><a class="l_k" href="functions/open.html">open</a></code>
pragma allows for flexible names: <code class="inline"><span class="w">koi8</span>-r</code>
 and <code class="inline"><span class="w">KOI8R</span></code>
 will both be
understood.</p>
<p>Common encodings recognized by ISO, MIME, IANA, and various other
standardisation organisations are recognised; for a more detailed
list see <a href="Encode/Supported.html">Encode::Supported</a>.</p>
<p><code class="inline"><a class="l_k" href="functions/read.html">read()</a></code> reads characters and returns the number of characters.
<code class="inline"><a class="l_k" href="functions/seek.html">seek()</a></code> and <code class="inline"><a class="l_k" href="functions/tell.html">tell()</a></code> operate on byte counts, as do <code class="inline"><a class="l_k" href="functions/sysread.html">sysread()</a></code>
and <code class="inline"><a class="l_k" href="functions/sysseek.html">sysseek()</a></code>.</p>
<p>Notice that because of the default behaviour of not doing any
conversion upon input if there is no default layer,
it is easy to mistakenly write code that keeps on expanding a file
by repeatedly encoding the data:</p>
<pre class="verbatim"><ol><li>    <span class="c"># BAD CODE WARNING</span></li><li>    <a class="l_k" href="functions/open.html">open</a> <span class="w">F</span><span class="cm">,</span> <span class="q">&quot;file&quot;</span><span class="sc">;</span></li><li>    <a class="l_k" href="functions/local.html">local</a> <span class="i">$/</span><span class="sc">;</span> <span class="c">## read in the whole file of 8-bit characters</span></li><li>    <span class="i">$t</span> = <span class="q">&lt;F&gt;</span><span class="sc">;</span></li><li>    <a class="l_k" href="functions/close.html">close</a> <span class="w">F</span><span class="sc">;</span></li><li>    <a class="l_k" href="functions/open.html">open</a> <span class="w">F</span><span class="cm">,</span> <span class="q">&quot;&gt;:encoding(UTF-8)&quot;</span><span class="cm">,</span> <span class="q">&quot;file&quot;</span><span class="sc">;</span></li><li>    <a class="l_k" href="functions/print.html">print</a> <span class="i">F</span> <span class="i">$t</span><span class="sc">;</span> <span class="c">## convert to UTF-8 on output</span></li><li>    <a class="l_k" href="functions/close.html">close</a> <span class="w">F</span><span class="sc">;</span></li></ol></pre><p>If you run this code twice, the contents of the <i>file</i> will be twice
UTF-8 encoded.  A <code class="inline"><a class="l_k" href="functions/use.html">use</a> <a class="l_k" href="functions/open.html">open</a> <span class="q">&#39;:encoding(UTF-8)&#39;</span></code>
 would have avoided the
bug, or explicitly opening also the <i>file</i> for input as UTF-8.</p>
<p><b>NOTE</b>: the <code class="inline"><span class="j">:</span><span class="w">utf8</span></code>
 and <code class="inline"><span class="j">:</span><span class="w">encoding</span></code>
 features work only if your
Perl has been built with <a href="PerlIO.html">PerlIO</a>, which is the default
on most systems.</p>
<a name="Displaying-Unicode-As-Text"></a><h2>Displaying Unicode As Text</h2>
<p>Sometimes you might want to display Perl scalars containing Unicode as
simple ASCII (or EBCDIC) text.  The following subroutine converts
its argument so that Unicode characters with code points greater than
255 are displayed as <code class="inline">\<span class="i">x</span><span class="s">{</span>...<span class="s">}</span></code>
, control characters (like <code class="inline">\<span class="w">n</span></code>
) are
displayed as <code class="inline">\<span class="w">x</span>..</code>
, and the rest of the characters as themselves:</p>
<pre class="verbatim"><ol><li><a name="nice_string"></a> sub <span class="m">nice_string</span> <span class="s">{</span></li><li>        <a class="l_k" href="functions/join.html">join</a><span class="s">(</span><span class="q">&quot;&quot;</span><span class="cm">,</span></li><li>        <a class="l_k" href="functions/map.html">map</a> <span class="s">{</span> <span class="i">$_</span> &gt; <span class="n">255</span>                    <span class="c"># if wide character...</span></li><li>              ? <a class="l_k" href="functions/sprintf.html">sprintf</a><span class="s">(</span><span class="q">&quot;\\x{%04X}&quot;</span><span class="cm">,</span> <span class="i">$_</span><span class="s">)</span>  <span class="c"># \x{...}</span></li><li>              <span class="co">:</span> <a class="l_k" href="functions/chr.html">chr</a><span class="s">(</span><span class="i">$_</span><span class="s">)</span> =~ <span class="q">/[[:cntrl:]]/</span>  <span class="c"># else if control character...</span></li><li>                ? <a class="l_k" href="functions/sprintf.html">sprintf</a><span class="s">(</span><span class="q">&quot;\\x%02X&quot;</span><span class="cm">,</span> <span class="i">$_</span><span class="s">)</span>  <span class="c"># \x..</span></li><li>                <span class="co">:</span> <a class="l_k" href="functions/quotemeta.html">quotemeta</a><span class="s">(</span><a class="l_k" href="functions/chr.html">chr</a><span class="s">(</span><span class="i">$_</span><span class="s">)</span><span class="s">)</span>      <span class="c"># else quoted or as themselves</span></li><li>        <span class="s">}</span> <a class="l_k" href="functions/unpack.html">unpack</a><span class="s">(</span><span class="q">&quot;W*&quot;</span><span class="cm">,</span> <span class="i">$_</span>[<span class="n">0</span>]<span class="s">)</span><span class="s">)</span><span class="sc">;</span>           <span class="c"># unpack Unicode characters</span></li><li>   <span class="s">}</span></li></ol></pre><p>For example,</p>
<pre class="verbatim"><ol><li>   <span class="i">nice_string</span><span class="s">(</span><span class="q">&quot;foo\x{100}bar\n&quot;</span><span class="s">)</span></li></ol></pre><p>returns the string</p>
<pre class="verbatim"><ol><li>   <span class="q">&#39;foo\x{0100}bar\x0A&#39;</span></li></ol></pre><p>which is ready to be printed.</p>
<p>(<code class="inline">\\<span class="w">x</span><span class="s">{</span><span class="s">}</span></code>
 is used here instead of <code class="inline">\\<span class="w">N</span><span class="s">{</span><span class="s">}</span></code>
, since it's most likely that
you want to see what the native values are.)</p>
<a name="Special-Cases"></a><h2>Special Cases</h2>
<ul>
<li>
<p>Bit Complement Operator ~ And vec()</p>
<p>The bit complement operator <code class="inline">~</code>
 may produce surprising results if
used on strings containing characters with ordinal values above
255. In such a case, the results are consistent with the internal
encoding of the characters, but not with much else. So don't do
that. Similarly for <code class="inline"><a class="l_k" href="functions/vec.html">vec()</a></code>: you will be operating on the
internally-encoded bit patterns of the Unicode characters, not on
the code point values, which is very probably not what you want.</p>
</li>
<li>
<p>Peeking At Perl's Internal Encoding</p>
<p>Normal users of Perl should never care how Perl encodes any particular
Unicode string (because the normal ways to get at the contents of a
string with Unicode--via input and output--should always be via
explicitly-defined I/O layers). But if you must, there are two
ways of looking behind the scenes.</p>
<p>One way of peeking inside the internal encoding of Unicode characters
is to use <code class="inline">unpack("C*", ...</code> to get the bytes of whatever the string
encoding happens to be, or <code class="inline"><a class="l_k" href="functions/unpack.html">unpack</a><span class="s">(</span><span class="q">&quot;U0..&quot;</span><span class="cm">,</span> ...<span class="s">)</span></code>
 to get the bytes of the
UTF-8 encoding:</p>
<pre class="verbatim"><ol><li>    <span class="c"># this prints  c4 80  for the UTF-8 bytes 0xc4 0x80</span></li><li>    <a class="l_k" href="functions/print.html">print</a> <a class="l_k" href="functions/join.html">join</a><span class="s">(</span><span class="q">&quot; &quot;</span><span class="cm">,</span> <a class="l_k" href="functions/unpack.html">unpack</a><span class="s">(</span><span class="q">&quot;U0(H2)*&quot;</span><span class="cm">,</span> <a class="l_k" href="functions/pack.html">pack</a><span class="s">(</span><span class="q">&quot;U&quot;</span><span class="cm">,</span> <span class="n">0x100</span><span class="s">)</span><span class="s">)</span><span class="s">)</span><span class="cm">,</span> <span class="q">&quot;\n&quot;</span><span class="sc">;</span></li></ol></pre><p>Yet another way would be to use the Devel::Peek module:</p>
<pre class="verbatim"><ol><li>    <span class="w">perl</span> -<span class="w">MDevel::Peek</span> -e <span class="q">&#39;Dump(chr(0x100))&#39;</span></li></ol></pre><p>That shows the <code class="inline"><span class="w">UTF8</span></code>
 flag in FLAGS and both the UTF-8 bytes
and Unicode characters in <code class="inline"><span class="w">PV</span></code>
.  See also later in this document
the discussion about the <code class="inline"><span class="i">utf8::is_utf8</span><span class="s">(</span><span class="s">)</span></code>
 function.</p>
</li>
</ul>
<a name="Advanced-Topics"></a><h2>Advanced Topics</h2>
<ul>
<li>
<p>String Equivalence</p>
<p>The question of string equivalence turns somewhat complicated
in Unicode: what do you mean by "equal"?</p>
<p>(Is <code class="inline"><span class="w">LATIN</span> <span class="w">CAPITAL</span> <span class="w">LETTER</span> <span class="w">A</span> <span class="w">WITH</span> <span class="w">ACUTE</span></code>
 equal to
<code class="inline"><span class="w">LATIN</span> <span class="w">CAPITAL</span> <span class="w">LETTER</span> <span class="w">A</span></code>
?)</p>
<p>The short answer is that by default Perl compares equivalence (<code class="inline">eq</code>
,
<code class="inline">ne</code>
) based only on code points of the characters.  In the above
case, the answer is no (because 0x00C1 != 0x0041).  But sometimes, any
CAPITAL LETTER A's should be considered equal, or even A's of any case.</p>
<p>The long answer is that you need to consider character normalization
and casing issues: see <a href="Unicode/Normalize.html">Unicode::Normalize</a>, Unicode Technical Report #15,
<a href="http://search.cpan.org/perldoc/http:#%2fwww.unicode.org%2funicode%2freports%2ftr15">Unicode Normalization Forms</a> and
sections on case mapping in the <a href="http://search.cpan.org/perldoc/http:#%2fwww.unicode.org">Unicode Standard</a>.</p>
<p>As of Perl 5.8.0, the "Full" case-folding of <i>Case
Mappings/SpecialCasing</i> is implemented, but bugs remain in <code class="inline"><a class="l_k" href="functions/qr.html">qr//i</a></code> with them,
mostly fixed by 5.14, and essentially entirely by 5.18.</p>
</li>
<li>
<p>String Collation</p>
<p>People like to see their strings nicely sorted--or as Unicode
parlance goes, collated.  But again, what do you mean by collate?</p>
<p>(Does <code class="inline"><span class="w">LATIN</span> <span class="w">CAPITAL</span> <span class="w">LETTER</span> <span class="w">A</span> <span class="w">WITH</span> <span class="w">ACUTE</span></code>
 come before or after
<code class="inline"><span class="w">LATIN</span> <span class="w">CAPITAL</span> <span class="w">LETTER</span> <span class="w">A</span> <span class="w">WITH</span> <span class="w">GRAVE</span></code>
?)</p>
<p>The short answer is that by default, Perl compares strings (<code class="inline">lt</code>
,
<code class="inline">le</code>
, <code class="inline">cmp</code>
, <code class="inline">ge</code>
, <code class="inline">gt</code>
) based only on the code points of the
characters.  In the above case, the answer is "after", since
<code class="inline"><span class="n">0x00C1</span></code>
 &gt; <code class="inline"><span class="n">0x00C0</span></code>
.</p>
<p>The long answer is that "it depends", and a good answer cannot be
given without knowing (at the very least) the language context.
See <a href="Unicode/Collate.html">Unicode::Collate</a>, and <i>Unicode Collation Algorithm</i>
<a href="http://www.unicode.org/unicode/reports/tr10/">http://www.unicode.org/unicode/reports/tr10/</a></p>
</li>
</ul>
<a name="Miscellaneous"></a><h2>Miscellaneous</h2>
<ul>
<li>
<p>Character Ranges and Classes</p>
<p>Character ranges in regular expression bracketed character classes ( e.g.,
<code class="inline"><span class="q">/[a-z]/</span></code>
) and in the <code class="inline"><a class="l_k" href="functions/tr.html">tr///</a></code> (also known as <code class="inline"><a class="l_k" href="functions/y.html">y///</a></code>) operator are not
magically Unicode-aware.  What this means is that <code class="inline"><span class="s">[</span><span class="w">A</span>-<span class="w">Za</span>-z<span class="s">]</span></code>
 will not
magically start to mean "all alphabetic letters" (not that it does mean that
even for 8-bit characters; for those, if you are using locales (<a href="perllocale.html">perllocale</a>),
use <code class="inline"><span class="q">/[[:alpha:]]/</span></code>
; and if not, use the 8-bit-aware property <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">alpha</span><span class="s">}</span></code>
).</p>
<p>All the properties that begin with <code class="inline">\<span class="w">p</span></code>
 (and its inverse <code class="inline">\<span class="w">P</span></code>
) are actually
character classes that are Unicode-aware.  There are dozens of them, see
<a href="perluniprops.html">perluniprops</a>.</p>
<p>Starting in v5.22, you can use Unicode code points as the end points of
regular expression pattern character ranges, and the range will include
all Unicode code points that lie between those end points, inclusive.</p>
<pre class="verbatim"><ol><li> <span class="q">qr/ [ \N{U+03} - \N{U+20} ] /xx</span></li></ol></pre><p>includes the code points
<code class="inline">\<span class="i">N</span><span class="s">{</span><span class="w">U</span>+<span class="n">03</span><span class="s">}</span></code>
, <code class="inline">\<span class="i">N</span><span class="s">{</span><span class="w">U</span>+<span class="n">04</span><span class="s">}</span></code>
, ..., <code class="inline">\<span class="i">N</span><span class="s">{</span><span class="w">U</span>+<span class="n">20</span><span class="s">}</span></code>
.</p>
<p>This also works for ranges in <code class="inline"><a class="l_k" href="functions/tr.html">tr///</a></code> starting in Perl v5.24.</p>
</li>
<li>
<p>String-To-Number Conversions</p>
<p>Unicode does define several other decimal--and numeric--characters
besides the familiar 0 to 9, such as the Arabic and Indic digits.
Perl does not support string-to-number conversion for digits other
than ASCII <code class="inline"><span class="n">0</span></code>
 to <code class="inline"><span class="n">9</span></code>
 (and ASCII <code class="inline"><span class="w">a</span></code>
 to <code class="inline"><span class="w">f</span></code>
 for hexadecimal).
To get safe conversions from any Unicode string, use
<a href="Unicode/UCD.html#num()">num() in Unicode::UCD</a>.</p>
</li>
</ul>
<a name="Questions-With-Answers"></a><h2>Questions With Answers</h2>
<ul>
<li>
<p>Will My Old Scripts Break?</p>
<p>Very probably not.  Unless you are generating Unicode characters
somehow, old behaviour should be preserved.  About the only behaviour
that has changed and which could start generating Unicode is the old
behaviour of <code class="inline"><a class="l_k" href="functions/chr.html">chr()</a></code> where supplying an argument more than 255
produced a character modulo 255.  <code class="inline"><a class="l_k" href="functions/chr.html">chr(300)</a></code>, for example, was equal
to <code class="inline"><a class="l_k" href="functions/chr.html">chr(45)</a></code> or "-" (in ASCII), now it is LATIN CAPITAL LETTER I WITH
BREVE.</p>
</li>
<li>
<p>How Do I Make My Scripts Work With Unicode?</p>
<p>Very little work should be needed since nothing changes until you
generate Unicode data.  The most important thing is getting input as
Unicode; for that, see the earlier I/O discussion.
To get full seamless Unicode support, add
<code class="inline"><a class="l_k" href="functions/use.html">use</a> <span class="w">feature</span> <span class="q">&#39;unicode_strings&#39;</span></code>
 (or <code class="inline"><a class="l_k" href="functions/use.html">use</a> <span class="n">5.012</span></code>
 or higher) to your
script.</p>
</li>
<li>
<p>How Do I Know Whether My String Is In Unicode?</p>
<p>You shouldn't have to care.  But you may if your Perl is before 5.14.0
or you haven't specified <code class="inline"><a class="l_k" href="functions/use.html">use</a> <span class="w">feature</span> <span class="q">&#39;unicode_strings&#39;</span></code>
 or <code class="inline"><a class="l_k" href="functions/use.html">use</a>
<span class="n">5.012</span></code>
 (or higher) because otherwise the rules for the code points
in the range 128 to 255 are different depending on
whether the string they are contained within is in Unicode or not.
(See <a href="perlunicode.html#When-Unicode-Does-Not-Happen">When Unicode Does Not Happen in perlunicode</a>.)</p>
<p>To determine if a string is in Unicode, use:</p>
<pre class="verbatim"><ol><li>    <a class="l_k" href="functions/print.html">print</a> <span class="i">utf8::is_utf8</span><span class="s">(</span><span class="i">$string</span><span class="s">)</span> ? <span class="n">1</span> <span class="co">:</span> <span class="n">0</span><span class="cm">,</span> <span class="q">&quot;\n&quot;</span><span class="sc">;</span></li></ol></pre><p>But note that this doesn't mean that any of the characters in the
string are necessary UTF-8 encoded, or that any of the characters have
code points greater than 0xFF (255) or even 0x80 (128), or that the
string has any characters at all.  All the <code class="inline"><span class="i">is_utf8</span><span class="s">(</span><span class="s">)</span></code>
 does is to
return the value of the internal "utf8ness" flag attached to the
<code class="inline"><span class="i">$string</span></code>
.  If the flag is off, the bytes in the scalar are interpreted
as a single byte encoding.  If the flag is on, the bytes in the scalar
are interpreted as the (variable-length, potentially multi-byte) UTF-8 encoded
code points of the characters.  Bytes added to a UTF-8 encoded string are
automatically upgraded to UTF-8.  If mixed non-UTF-8 and UTF-8 scalars
are merged (double-quoted interpolation, explicit concatenation, or
printf/sprintf parameter substitution), the result will be UTF-8 encoded
as if copies of the byte strings were upgraded to UTF-8: for example,</p>
<pre class="verbatim"><ol><li>    <span class="i">$a</span> = <span class="q">&quot;ab\x80c&quot;</span><span class="sc">;</span></li><li>    <span class="i">$b</span> = <span class="q">&quot;\x{100}&quot;</span><span class="sc">;</span></li><li>    <a class="l_k" href="functions/print.html">print</a> <span class="q">&quot;$a = $b\n&quot;</span><span class="sc">;</span></li></ol></pre><p>the output string will be UTF-8-encoded <code class="inline"><span class="w">ab</span>\<span class="w">x80c</span> = \<span class="i">x</span><span class="s">{</span><span class="n">100</span><span class="s">}</span>\<span class="w">n</span></code>
, but
<code class="inline"><span class="i">$a</span></code>
 will stay byte-encoded.</p>
<p>Sometimes you might really need to know the byte length of a string
instead of the character length. For that use the <code class="inline"><span class="w">bytes</span></code>
 pragma
and the <code class="inline"><a class="l_k" href="functions/length.html">length()</a></code> function:</p>
<pre class="verbatim"><ol><li>    <a class="l_k" href="functions/my.html">my</a> <span class="i">$unicode</span> = <a class="l_k" href="functions/chr.html">chr</a><span class="s">(</span><span class="n">0x100</span><span class="s">)</span><span class="sc">;</span></li><li>    <a class="l_k" href="functions/print.html">print</a> <a class="l_k" href="functions/length.html">length</a><span class="s">(</span><span class="i">$unicode</span><span class="s">)</span><span class="cm">,</span> <span class="q">&quot;\n&quot;</span><span class="sc">;</span> <span class="c"># will print 1</span></li><li>    <a class="l_k" href="functions/use.html">use</a> <span class="w">bytes</span><span class="sc">;</span></li><li>    <a class="l_k" href="functions/print.html">print</a> <a class="l_k" href="functions/length.html">length</a><span class="s">(</span><span class="i">$unicode</span><span class="s">)</span><span class="cm">,</span> <span class="q">&quot;\n&quot;</span><span class="sc">;</span> <span class="c"># will print 2</span></li><li>                                  <span class="c"># (the 0xC4 0x80 of the UTF-8)</span></li><li>    <a class="l_k" href="functions/no.html">no</a> <span class="w">bytes</span><span class="sc">;</span></li></ol></pre></li>
<li>
<p>How Do I Find Out What Encoding a File Has?</p>
<p>You might try <a href="Encode/Guess.html">Encode::Guess</a>, but it has a number of limitations.</p>
</li>
<li>
<p>How Do I Detect Data That's Not Valid In a Particular Encoding?</p>
<p>Use the <code class="inline"><span class="w">Encode</span></code>
 package to try converting it.
For example,</p>
<pre class="verbatim"><ol><li>    <a class="l_k" href="functions/use.html">use</a> <span class="w">Encode</span> <span class="q">&#39;decode&#39;</span><span class="sc">;</span></li><li></li><li>    <a class="l_k" href="functions/if.html">if</a> <span class="s">(</span><a class="l_k" href="functions/eval.html">eval</a> <span class="s">{</span> <span class="i">decode</span><span class="s">(</span><span class="q">&#39;UTF-8&#39;</span><span class="cm">,</span> <span class="i">$string</span><span class="cm">,</span> <span class="w">Encode::FB_CROAK</span><span class="s">)</span><span class="sc">;</span> <span class="n">1</span> <span class="s">}</span><span class="s">)</span> <span class="s">{</span></li><li>        <span class="c"># $string is valid UTF-8</span></li><li>    <span class="s">}</span> <a class="l_k" href="functions/else.html">else</a> <span class="s">{</span></li><li>        <span class="c"># $string is not valid UTF-8</span></li><li>    <span class="s">}</span></li></ol></pre><p>Or use <code class="inline"><a class="l_k" href="functions/unpack.html">unpack</a></code> to try decoding it:</p>
<pre class="verbatim"><ol><li>    <a class="l_k" href="functions/use.html">use</a> <span class="w">warnings</span><span class="sc">;</span></li><li>    <span class="i">@chars</span> = <a class="l_k" href="functions/unpack.html">unpack</a><span class="s">(</span><span class="q">&quot;C0U*&quot;</span><span class="cm">,</span> <span class="i">$string_of_bytes_that_I_think_is_utf8</span><span class="s">)</span><span class="sc">;</span></li></ol></pre><p>If invalid, a <code class="inline">Malformed UTF-8 character</code> warning is produced. The "C0" means
"process the string character per character".  Without that, the
<code class="inline"><a class="l_k" href="functions/unpack.html">unpack</a><span class="s">(</span><span class="q">&quot;U*&quot;</span><span class="cm">,</span> ...<span class="s">)</span></code>
 would work in <code class="inline"><span class="w">U0</span></code>
 mode (the default if the format
string starts with <code class="inline"><span class="w">U</span></code>
) and it would return the bytes making up the UTF-8
encoding of the target string, something that will always work.</p>
</li>
<li>
<p>How Do I Convert Binary Data Into a Particular Encoding, Or Vice Versa?</p>
<p>This probably isn't as useful as you might think.
Normally, you shouldn't need to.</p>
<p>In one sense, what you are asking doesn't make much sense: encodings
are for characters, and binary data are not "characters", so converting
"data" into some encoding isn't meaningful unless you know in what
character set and encoding the binary data is in, in which case it's
not just binary data, now is it?</p>
<p>If you have a raw sequence of bytes that you know should be
interpreted via a particular encoding, you can use <code class="inline"><span class="w">Encode</span></code>
:</p>
<pre class="verbatim"><ol><li>    <a class="l_k" href="functions/use.html">use</a> <span class="w">Encode</span> <span class="q">&#39;from_to&#39;</span><span class="sc">;</span></li><li>    <span class="i">from_to</span><span class="s">(</span><span class="i">$data</span><span class="cm">,</span> <span class="q">&quot;iso-8859-1&quot;</span><span class="cm">,</span> <span class="q">&quot;UTF-8&quot;</span><span class="s">)</span><span class="sc">;</span> <span class="c"># from latin-1 to UTF-8</span></li></ol></pre><p>The call to <code class="inline"><span class="i">from_to</span><span class="s">(</span><span class="s">)</span></code>
 changes the bytes in <code class="inline"><span class="i">$data</span></code>
, but nothing
material about the nature of the string has changed as far as Perl is
concerned.  Both before and after the call, the string <code class="inline"><span class="i">$data</span></code>

contains just a bunch of 8-bit bytes. As far as Perl is concerned,
the encoding of the string remains as "system-native 8-bit bytes".</p>
<p>You might relate this to a fictional 'Translate' module:</p>
<pre class="verbatim"><ol><li>   <a class="l_k" href="functions/use.html">use</a> <span class="w">Translate</span><span class="sc">;</span></li><li>   <a class="l_k" href="functions/my.html">my</a> <span class="i">$phrase</span> = <span class="q">&quot;Yes&quot;</span><span class="sc">;</span></li><li>   <span class="i">Translate::from_to</span><span class="s">(</span><span class="i">$phrase</span><span class="cm">,</span> <span class="q">&#39;english&#39;</span><span class="cm">,</span> <span class="q">&#39;deutsch&#39;</span><span class="s">)</span><span class="sc">;</span></li><li>   <span class="c">## phrase now contains &quot;Ja&quot;</span></li></ol></pre><p>The contents of the string changes, but not the nature of the string.
Perl doesn't know any more after the call than before that the
contents of the string indicates the affirmative.</p>
<p>Back to converting data.  If you have (or want) data in your system's
native 8-bit encoding (e.g. Latin-1, EBCDIC, etc.), you can use
pack/unpack to convert to/from Unicode.</p>
<pre class="verbatim"><ol><li>    <span class="i">$native_string</span>  = <a class="l_k" href="functions/pack.html">pack</a><span class="s">(</span><span class="q">&quot;W*&quot;</span><span class="cm">,</span> <a class="l_k" href="functions/unpack.html">unpack</a><span class="s">(</span><span class="q">&quot;U*&quot;</span><span class="cm">,</span> <span class="i">$Unicode_string</span><span class="s">)</span><span class="s">)</span><span class="sc">;</span></li><li>    <span class="i">$Unicode_string</span> = <a class="l_k" href="functions/pack.html">pack</a><span class="s">(</span><span class="q">&quot;U*&quot;</span><span class="cm">,</span> <a class="l_k" href="functions/unpack.html">unpack</a><span class="s">(</span><span class="q">&quot;W*&quot;</span><span class="cm">,</span> <span class="i">$native_string</span><span class="s">)</span><span class="s">)</span><span class="sc">;</span></li></ol></pre><p>If you have a sequence of bytes you <b>know</b> is valid UTF-8,
but Perl doesn't know it yet, you can make Perl a believer, too:</p>
<pre class="verbatim"><ol><li>    <span class="i">$Unicode</span> = <span class="i">$bytes</span><span class="sc">;</span></li><li>    <span class="i">utf8::decode</span><span class="s">(</span><span class="i">$Unicode</span><span class="s">)</span><span class="sc">;</span></li></ol></pre><p>or:</p>
<pre class="verbatim"><ol><li>    <span class="i">$Unicode</span> = <a class="l_k" href="functions/pack.html">pack</a><span class="s">(</span><span class="q">&quot;U0a*&quot;</span><span class="cm">,</span> <span class="i">$bytes</span><span class="s">)</span><span class="sc">;</span></li></ol></pre><p>You can find the bytes that make up a UTF-8 sequence with</p>
<pre class="verbatim"><ol><li>    <span class="i">@bytes</span> = <a class="l_k" href="functions/unpack.html">unpack</a><span class="s">(</span><span class="q">&quot;C*&quot;</span><span class="cm">,</span> <span class="i">$Unicode_string</span><span class="s">)</span></li></ol></pre><p>and you can create well-formed Unicode with</p>
<pre class="verbatim"><ol><li>    <span class="i">$Unicode_string</span> = <a class="l_k" href="functions/pack.html">pack</a><span class="s">(</span><span class="q">&quot;U*&quot;</span><span class="cm">,</span> <span class="n">0xff</span><span class="cm">,</span> ...<span class="s">)</span></li></ol></pre></li>
<li>
<p>How Do I Display Unicode?  How Do I Input Unicode?</p>
<p>See <a href="http://www.alanwood.net/unicode/">http://www.alanwood.net/unicode/</a> and
<a href="http://www.cl.cam.ac.uk/~mgk25/unicode.html">http://www.cl.cam.ac.uk/~mgk25/unicode.html</a></p>
</li>
<li>
<p>How Does Unicode Work With Traditional Locales?</p>
<p>If your locale is a UTF-8 locale, starting in Perl v5.20, Perl works
well for all categories except <code class="inline"><span class="w">LC_COLLATE</span></code>
 dealing with sorting and
the <code class="inline">cmp</code>
 operator.</p>
<p>For other locales, starting in Perl 5.16, you can specify</p>
<pre class="verbatim"><ol><li>    <a class="l_k" href="functions/use.html">use</a> <span class="w">locale</span> <span class="q">&#39;:not_characters&#39;</span><span class="sc">;</span></li></ol></pre><p>to get Perl to work well with them.  The catch is that you
have to translate from the locale character set to/from Unicode
yourself.  See <a href="#Unicode-I%2fO">Unicode I/O</a> above for how to</p>
<pre class="verbatim"><ol><li>    <a class="l_k" href="functions/use.html">use</a> <a class="l_k" href="functions/open.html">open</a> <span class="q">&#39;:locale&#39;</span><span class="sc">;</span></li></ol></pre><p>to accomplish this, but full details are in <a href="perllocale.html#Unicode-and-UTF-8">Unicode and UTF-8 in perllocale</a>, including gotchas that happen if you don't specify
<code class="inline"><span class="j">:</span><span class="w">not_characters</span></code>
.</p>
</li>
</ul>
<a name="Hexadecimal-Notation"></a><h2>Hexadecimal Notation</h2>
<p>The Unicode standard prefers using hexadecimal notation because
that more clearly shows the division of Unicode into blocks of 256 characters.
Hexadecimal is also simply shorter than decimal.  You can use decimal
notation, too, but learning to use hexadecimal just makes life easier
with the Unicode standard.  The <code class="inline"><span class="w">U</span>+<span class="w">HHHH</span></code>
 notation uses hexadecimal,
for example.</p>
<p>The <code class="inline"><span class="n">0</span>x</code>
 prefix means a hexadecimal number, the digits are 0-9 <i>and</i>
a-f (or A-F, case doesn't matter).  Each hexadecimal digit represents
four bits, or half a byte.  <code class="inline"><a class="l_k" href="functions/print.html">print</a> <span class="n">0</span>x...<span class="cm">,</span> <span class="q">&quot;\n&quot;</span></code>
 will show a
hexadecimal number in decimal, and <code class="inline"><a class="l_k" href="functions/printf.html">printf</a> <span class="q">&quot;%x\n&quot;</span><span class="cm">,</span> <span class="i">$decimal</span></code>
 will
show a decimal number in hexadecimal.  If you have just the
"hex digits" of a hexadecimal number, you can use the <code class="inline"><a class="l_k" href="functions/hex.html">hex()</a></code> function.</p>
<pre class="verbatim"><ol><li>    <a class="l_k" href="functions/print.html">print</a> <span class="n">0x0009</span><span class="cm">,</span> <span class="q">&quot;\n&quot;</span><span class="sc">;</span>    <span class="c"># 9</span></li><li>    <a class="l_k" href="functions/print.html">print</a> <span class="n">0x000a</span><span class="cm">,</span> <span class="q">&quot;\n&quot;</span><span class="sc">;</span>    <span class="c"># 10</span></li><li>    <a class="l_k" href="functions/print.html">print</a> <span class="n">0x000f</span><span class="cm">,</span> <span class="q">&quot;\n&quot;</span><span class="sc">;</span>    <span class="c"># 15</span></li><li>    <a class="l_k" href="functions/print.html">print</a> <span class="n">0x0010</span><span class="cm">,</span> <span class="q">&quot;\n&quot;</span><span class="sc">;</span>    <span class="c"># 16</span></li><li>    <a class="l_k" href="functions/print.html">print</a> <span class="n">0x0011</span><span class="cm">,</span> <span class="q">&quot;\n&quot;</span><span class="sc">;</span>    <span class="c"># 17</span></li><li>    <a class="l_k" href="functions/print.html">print</a> <span class="n">0x0100</span><span class="cm">,</span> <span class="q">&quot;\n&quot;</span><span class="sc">;</span>    <span class="c"># 256</span></li><li></li><li>    <a class="l_k" href="functions/print.html">print</a> <span class="n">0x0041</span><span class="cm">,</span> <span class="q">&quot;\n&quot;</span><span class="sc">;</span>    <span class="c"># 65</span></li><li></li><li>    <a class="l_k" href="functions/printf.html">printf</a> <span class="q">&quot;%x\n&quot;</span><span class="cm">,</span>  <span class="n">65</span><span class="sc">;</span>    <span class="c"># 41</span></li><li>    <a class="l_k" href="functions/printf.html">printf</a> <span class="q">&quot;%#x\n&quot;</span><span class="cm">,</span> <span class="n">65</span><span class="sc">;</span>    <span class="c"># 0x41</span></li><li></li><li>    <a class="l_k" href="functions/print.html">print</a> <a class="l_k" href="functions/hex.html">hex</a><span class="s">(</span><span class="q">&quot;41&quot;</span><span class="s">)</span><span class="cm">,</span> <span class="q">&quot;\n&quot;</span><span class="sc">;</span> <span class="c"># 65</span></li></ol></pre><a name="Further-Resources"></a><h2>Further Resources</h2>
<ul>
<li>
<p>Unicode Consortium</p>
<p><a href="http://www.unicode.org/">http://www.unicode.org/</a></p>
</li>
<li>
<p>Unicode FAQ</p>
<p><a href="http://www.unicode.org/unicode/faq/">http://www.unicode.org/unicode/faq/</a></p>
</li>
<li>
<p>Unicode Glossary</p>
<p><a href="http://www.unicode.org/glossary/">http://www.unicode.org/glossary/</a></p>
</li>
<li>
<p>Unicode Recommended Reading List</p>
<p>The Unicode Consortium has a list of articles and books, some of which
give a much more in depth treatment of Unicode:
<a href="http://unicode.org/resources/readinglist.html">http://unicode.org/resources/readinglist.html</a></p>
</li>
<li>
<p>Unicode Useful Resources</p>
<p><a href="http://www.unicode.org/unicode/onlinedat/resources.html">http://www.unicode.org/unicode/onlinedat/resources.html</a></p>
</li>
<li>
<p>Unicode and Multilingual Support in HTML, Fonts, Web Browsers and Other Applications</p>
<p><a href="http://www.alanwood.net/unicode/">http://www.alanwood.net/unicode/</a></p>
</li>
<li>
<p>UTF-8 and Unicode FAQ for Unix/Linux</p>
<p><a href="http://www.cl.cam.ac.uk/~mgk25/unicode.html">http://www.cl.cam.ac.uk/~mgk25/unicode.html</a></p>
</li>
<li>
<p>Legacy Character Sets</p>
<p><a href="http://www.czyborra.com/">http://www.czyborra.com/</a>
<a href="http://www.eki.ee/letter/">http://www.eki.ee/letter/</a></p>
</li>
<li>
<p>You can explore various information from the Unicode data files using
the <code class="inline"><span class="w">Unicode::UCD</span></code>
 module.</p>
</li>
</ul>
<a name="UNICODE-IN-OLDER-PERLS"></a><h1>UNICODE IN OLDER PERLS</h1>
<p>If you cannot upgrade your Perl to 5.8.0 or later, you can still
do some Unicode processing by using the modules <code class="inline"><span class="w">Unicode::String</span></code>
,
<code class="inline"><span class="w">Unicode::Map8</span></code>
, and <code class="inline"><span class="w">Unicode::Map</span></code>
, available from CPAN.
If you have the GNU recode installed, you can also use the
Perl front-end <code class="inline"><span class="w">Convert::Recode</span></code>
 for character conversions.</p>
<p>The following are fast conversions from ISO 8859-1 (Latin-1) bytes
to UTF-8 bytes and back, the code works even with older Perl 5 versions.</p>
<pre class="verbatim"><ol><li>    <span class="c"># ISO 8859-1 to UTF-8</span></li><li>    <span class="q">s/([\x80-\xFF])/chr(0xC0|ord($1)&gt;&gt;6).chr(0x80|ord($1)&amp;0x3F)/eg</span><span class="sc">;</span></li><li></li><li>    <span class="c"># UTF-8 to ISO 8859-1</span></li><li>    <span class="q">s/([\xC2\xC3])([\x80-\xBF])/chr(ord($1)&lt;&lt;6&amp;0xC0|ord($2)&amp;0x3F)/eg</span><span class="sc">;</span></li></ol></pre><a name="SEE-ALSO"></a><h1>SEE ALSO</h1>
<p><a href="perlunitut.html">perlunitut</a>, <a href="perlunicode.html">perlunicode</a>, <a href="Encode.html">Encode</a>, <a href="open.html">open</a>, <a href="utf8.html">utf8</a>, <a href="bytes.html">bytes</a>,
<a href="perlretut.html">perlretut</a>, <a href="perlrun.html">perlrun</a>, <a href="Unicode/Collate.html">Unicode::Collate</a>, <a href="Unicode/Normalize.html">Unicode::Normalize</a>,
<a href="Unicode/UCD.html">Unicode::UCD</a></p>
<a name="ACKNOWLEDGMENTS"></a><h1>ACKNOWLEDGMENTS</h1>
<p>Thanks to the kind readers of the perl5-porters@perl.org,
perl-unicode@perl.org, linux-utf8@nl.linux.org, and unicore@unicode.org
mailing lists for their valuable feedback.</p>
<a name="AUTHOR%2c-COPYRIGHT%2c-AND-LICENSE"></a><h1>AUTHOR, COPYRIGHT, AND LICENSE</h1>
<p>Copyright 2001-2011 Jarkko Hietaniemi &lt;jhi@iki.fi&gt;.
Now maintained by Perl 5 Porters.</p>
<p>This document may be distributed under the same terms as Perl itself.</p>




  <div id="page_index" class="hud_container">
    <div id="page_index_header" class="hud_header">
      <div id="page_index_close" class="hud_close"><a href="#" onClick="pageIndex.hide();return false;"></a></div>
      <div id="page_index_title" class="hud_title"><span class="hud_span_top">Page index</span></div>
      <div id="page_index_topright" class="hud_topright"></div>
    </div>
    <div id="page_index_content" class="hud_content">
      <ul><li><a href="#NAME">NAME</a><li><a href="#DESCRIPTION">DESCRIPTION</a><ul><li><a href="#Unicode">Unicode</a><li><a href="#Perl's-Unicode-Support">Perl's Unicode Support</a><li><a href="#Perl's-Unicode-Model">Perl's Unicode Model</a><li><a href="#Unicode-and-EBCDIC">Unicode and EBCDIC</a><li><a href="#Creating-Unicode">Creating Unicode</a><li><a href="#Handling-Unicode">Handling Unicode</a><li><a href="#Legacy-Encodings">Legacy Encodings</a><li><a href="#Unicode-I%2fO">Unicode I/O</a><li><a href="#Displaying-Unicode-As-Text">Displaying Unicode As Text</a><li><a href="#Special-Cases">Special Cases</a><li><a href="#Advanced-Topics">Advanced Topics</a><li><a href="#Miscellaneous">Miscellaneous</a><li><a href="#Questions-With-Answers">Questions With Answers</a><li><a href="#Hexadecimal-Notation">Hexadecimal Notation</a><li><a href="#Further-Resources">Further Resources</a></ul><li><a href="#UNICODE-IN-OLDER-PERLS">UNICODE IN OLDER PERLS</a><li><a href="#SEE-ALSO">SEE ALSO</a><li><a href="#ACKNOWLEDGMENTS">ACKNOWLEDGMENTS</a><li><a href="#AUTHOR%2c-COPYRIGHT%2c-AND-LICENSE">AUTHOR, COPYRIGHT, AND LICENSE</a></ul>
    </div>
    <div id="page_index_footer" class="hud_footer">
      <div id="page_index_bottomleft" class="hud_bottomleft"></div>
      <div id="page_index_bottom" class="hud_bottom"><span class="hud_span_bottom"></span></div>
      <div id="page_index_resize" class="hud_resize"></div>
    </div>
  </div>


	    &nbsp;
          </div>
          <div id="content_footer">
          </div>
        </div>
        <div class="clear"></div>
      </div>
      
    <div id="footer">
      <div id="footer_content">
        <div id="footer_strapline">
          perldoc.perl.org - Official documentation for the Perl programming language
        </div>
        <div id="footer_links">
          <div id="address">
            <p class="name">Contact details</p>
            <p class="address">
	      Site maintained by <a href="mailto:jj@jonallen.info">Jon Allen (JJ)</a><br>
	    </p>
            <p class="contact">
              Documentation maintained by the <a href="http://lists.cpan.org/showlist.cgi?name=perl5-porters">Perl 5 Porters</a>
            </p>
          </div>
          <ul class="f1">
            <li>Manual
              <ul class="f2">
                <li><a href="index-overview.html">Overview</a>
                <li><a href="index-tutorials.html">Tutorials</a>
                <li><a href="index-faq.html">FAQs</a>
                <li><a href="index-history.html">Changes</a>
              </ul>
            <li>Reference
              <ul class="f2">
                <li><a href="index-language.html">Language</a>
                <li><a href="index-functions.html">Functions</a>
                <li><a href="perlop.html">Operators</a>
                <li><a href="perlvar.html">Variables</a>
              </ul>
            <li>Modules
              <ul class="f2">
                <li><a href="index-modules-A.html">Modules</a>
                <li><a href="index-pragmas.html">Pragmas</a>
                <li><a href="index-utilities.html">Utilities</a>
              </ul>
            <li>Misc
              <ul class="f2">
                <li><a href="index-licence.html">License</a>
                <li><a href="index-internals.html">Internals</a>
                <li><a href="index-platforms.html">Platforms</a>
              </ul>          </ul>
          <div class="clear"></div>
        </div>
      </div>
      <div id="footer_end">
      </div>
    </div>
      
    </div>
      <script language="JavaScript" type="text/javascript" src="static/exploreperl.js"></script>
      <script language="JavaScript" src="static/combined-20100403.js" type="text/javascript"></script>
<script language="JavaScript" type="text/javascript">
  perldoc.setPath(0);
  perldoc.pageName    = 'perluniintro';
  perldoc.pageAddress = 'perluniintro.html';
  perldoc.contentPage = 1;
  explorePerl.render();
  explorePerl.addEvents('explore_anchor');
</script>
    
  </body>
</html>