1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
|
#! /usr/bin/python -B
# Copyright (C) 2016 and later: Unicode, Inc. and others.
# License & terms of use: http://www.unicode.org/copyright.html
# Copyright (C) 2009-2011, International Business Machines Corporation, Google and Others.
# All rights reserved.
#
# Script to check that ICU source files contain only valid UTF-8 encoded text,
# and that all files except '.txt' files do not contain a Byte Order Mark (BOM).
#
# THIS SCRIPT DOES NOT WORK ON WINDOWS
# It only works correctly on platforms where the native line ending is a plain \n
#
# usage:
# icu-file-utf8-check.py [options]
#
# options:
# -h | --help Print a usage line and exit.
#
# The tool operates recursively on the directory from which it is run.
# Only files from the ICU github repository are checked.
# No changes are made to the repository; only the working copy will be altered.
from __future__ import print_function
import sys
import os
import os.path
import re
import getopt
def runCommand(cmd):
output_file = os.popen(cmd);
output_text = output_file.read();
exit_status = output_file.close();
if exit_status:
print('"', cmd, '" failed. Exiting.', file=sys.stderr)
sys.exit(exit_status)
return output_text
def usage():
print("usage: " + sys.argv[0] + " [-h | --help]")
#
# File check. Check source code files for UTF-8 and all except text files for not containing a BOM
# file_name: name of a text file.
# is_source: Flag, set to True if file is a source code file (.c, .cpp, .h, .java).
#
def check_file(file_name, is_source):
f = open(file_name, 'rb')
bytes = f.read()
f.close()
if is_source:
try:
bytes.decode("UTF-8")
except UnicodeDecodeError:
print("Error: %s is a source code file but contains non-utf-8 bytes." % file_name)
if bytes[0] == 0xef:
if not (file_name.endswith(".txt") or file_name.endswith(".sln")
or file_name.endswith(".targets")
or ".vcxproj" in file_name):
print("Warning: file %s contains a UTF-8 BOM: " % file_name)
return
def main(argv):
try:
opts, args = getopt.getopt(argv, "h", ("help"))
except getopt.GetoptError:
print("unrecognized option: " + argv[0])
usage()
sys.exit(2)
for opt, arg in opts:
if opt in ("-h", "--help"):
usage()
sys.exit()
if args:
print("unexpected command line argument")
usage()
sys.exit()
output = runCommand("git ls-files ");
file_list = output.splitlines()
source_file_re = re.compile(".*((?:\\.c$)|(?:\\.cpp$)|(?:\\.h$)|(?:\\.java$))")
for f in file_list:
if os.path.isdir(f):
print("Skipping dir " + f)
continue
if not os.path.isfile(f):
print("Repository file not in working copy: " + f)
continue;
source_file = source_file_re.match(f)
check_file(f, source_file)
if __name__ == "__main__":
main(sys.argv[1:])
|
