1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
|
program string_47
implicit none
type :: string
character(:), allocatable :: s
end type
type(string), allocatable :: tokens(:)
tokens = bpe("abc")
print *, tokens(1)%s, tokens(2)%s, tokens(3)%s
if( tokens(1)%s /= "a" ) error stop
if( tokens(2)%s /= "b" ) error stop
if( tokens(3)%s /= "c" ) error stop
contains
function merge_utf8_pairs(intokens) result(tokens)
type(string), intent(in) :: intokens(:)
type(string), allocatable :: tokens(:)
allocate(tokens(size(intokens)))
tokens = intokens
end function
function bpe(token) result(tokens)
character(*), intent(in) :: token
type(string), allocatable :: tokens(:)
integer :: i
allocate(tokens(len(token)))
do i = 1, len(token)
tokens(i)%s = token(i:i)
end do
tokens = merge_utf8_pairs(tokens)
end function
end program
|
