1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
|
#!/usr/bin/env ruby
require 'unicode/name'
require 'unicode/scripts'
require 'unicode/blocks'
require 'json'
require 'differ'
module Differ
module Format
module Color
class << self
def as_change(change) # monkey patch
as_insert(change) << "\n" << as_delete(change)
end
end
end
end
end
def readable(c)
j = c.to_json
if j.size == 3
j[1...-1]
else
j
end
end
def explain(s)
ret = ''
hist = Hash.new(0)
s.each_char do |c|
hist[c] += 1 unless c.ord.between?(32, 126)
end
hist.delete("\n")
hist.keys.sort.group_by {|c| Unicode::Blocks.block(c)}.each do |block, l|
scripts = Set[]
l.each do |c|
scripts << Unicode::Scripts.scripts(c)
end
ret << "*** #{block}"
ret << " (#{scripts.join})" if scripts.size == 1
ret << "\n"
l.each do |c|
ret << "#{readable(c)}: U+#{"%04X %4d" % [c.ord, hist[c]]
} #{Unicode::Name.correct(c) ||
Unicode::Name.label(c)
}"
ret << " (#{Unicode::Scripts.scripts(c).join(", ")})" if scripts.size != 1
ret << "\n"
end
end
ret
end
s = ARGF.read
es = explain(s)
n = s.unicode_normalize
en = explain(n)
if es == en
puts es
else
puts "*** Warning: some characters are not normalized and are shown in red."
puts " ...showing a normalized variant (NFC) in green."
puts " Lack of normalization may or may not be a problem."
puts " (Characters may appear to be under wrong block heading.)"
puts Differ.diff_by_line(en, es).format_as(:color)
end
|
