1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
|
#!/bin/sh
TOOLDIR=../../tools/src
TOOL=
FORMAT_TOOL=
COMPARE_TOOL=
TXT2FST=
if [ "$1" = '--python' ]; then
TOOL="python3 ./hfst-lexc.py"
FORMAT_TOOL="python3 ./hfst-format.py"
COMPARE_TOOL="python3 ./hfst-compare.py"
TXT2FST="python3 ./hfst-txt2fst.py"
else
TOOL=$TOOLDIR/hfst-lexc
FORMAT_TOOL=$TOOLDIR/hfst-format
COMPARE_TOOL=$TOOLDIR/hfst-compare
TXT2FST=$TOOLDIR/hfst-txt2fst
for tool in $TOOL $FORMAT_TOOL $COMPARE_TOOL $TXT2FST;
do
if ! test -x $tool ; then
echo "missing hfst-lexc, assuming configured off, skipping"
exit 77
fi
done
fi
LEXCTESTS="basic.cat-dog-bird.lexc basic.colons.lexc basic.comments.lexc
basic.empty-sides.lexc basic.escapes.lexc
basic.infostrings.lexc basic.initial-lexicon-empty.lexc
basic.multichar-symbols.lexc
basic.no-Root.lexc
basic.multi-entry-lines.lexc basic.no-newline-at-end.lexc
basic.punctuation.lexc basic.root-loop.lexc
basic.spurious-lexicon.lexc basic.string-pairs.lexc
basic.two-lexicons.lexc basic.UTF-8.lexc basic.zeros-epsilons.lexc
basic.lowercase-lexicon-end.lexc
basic.multichar-escaped-zero.lexc
basic.almost-reserved-words.lexc
basic.regexps.lexc
hfst.weights.lexc
xre.more-than-twice.lexc
xre.less-than-twice.lexc
xre.automatic-multichar-symbols.lexc xre.basic.lexc
xre.definitions.lexc xre.months.lexc xre.nested-definitions.lexc
xre.numeric-star.lexc xre.sharp.lexc xre.quotations.lexc
xre.star-plus-optional.lexc
no-newline-before-sublexicon.lexc xre.any-variations.lexc"
# basic.lowercase-lexicon-end.lexc fails with -F with python
# xre.any-variations.lexc # - hfst works file, foma's eliminate_flags removes valid paths (hfst-compare -e)
# basic.end.lexc -hfst doesn't parse till end
# xre.any-variations.lexc -foma ?:? problem
# basic.multichar-symbol-with-0.lexc - hfst works fine, foma wrong
# basic.multichar-flag-with-zero.lexc - foma wrong
LEXCXFAIL="xfail.bogus.lexc xfail.ISO-8859-1.lexc xfail.lexicon-semicolon.lexc xfail.sublexicon-defined-more-than-once.lexc"
LEXCWARN="warn.sublexicon-mentioned-but-not-defined.lexc warn.one-sided-flags.lexc"
if test "$srcdir" = ""; then
srcdir="./"
fi
for i in .sfst .ofst .foma ; do
FFLAG=
FNAME=
case $i in
.sfst)
FNAME="sfst";
FFLAG="-f sfst";;
.ofst)
FNAME="openfst-tropical";
FFLAG="-f openfst-tropical";;
.foma)
FNAME="foma";
FFLAG="-f foma";;
*)
FNAME=;
FFLAG=;;
esac
#echo "---- $FNAME --------"
if ! ($FORMAT_TOOL --test-format $FNAME ) ; then
continue;
fi
if test -f cat$i ; then
if ! $TOOL $FFLAG $srcdir/cat.lexc -o test 2> /dev/null; then
echo lexc $FFLAG cat.lexc failed with $?
exit 1
fi
if ! $COMPARE_TOOL -e -s cat$i test ; then
echo "results differ: " "cat"$i" test"
exit 1
fi
rm test
fi
for f in $LEXCTESTS $LEXCWARN ; do
#check non-flag result
if ! $TOOL $FFLAG $srcdir/$f -o test 2> /dev/null; then
echo lexc $FFLAG $f failed with $?
exit 1
fi
RESULT="$f.result.prolog"
# create foma result
# RESULT_GZ="$RESULT.gz"
# echo "read lexc $srcdir/$f \n
# save stack test.foma.gz \n
# quit \n" > tmp-foma-script
# foma -q < tmp-foma-script
# gunzip test.foma.gz
# mv test.foma $RESULT
# rm tmp-foma-script
$TXT2FST --prolog $FFLAG -i $RESULT -o $RESULT.tmp
#echo "comparing file: $f"
if ! $COMPARE_TOOL -e -s $RESULT.tmp test ; then
echo "results differ: $f"
exit 1
fi
rm $RESULT.tmp
rm test
# check flag results
RESULT="$f.flag.result.prolog"
if ! $TOOL -F $FFLAG $srcdir/$f -o test 2> /dev/null; then
echo hfst-lexc -F $FFLAG $f failed with $?
exit 1
fi
$TXT2FST --prolog $FFLAG -i $RESULT -o $RESULT.tmp
#echo "comparing flag file: $f"
if ! $COMPARE_TOOL -e -s $RESULT.tmp test ; then
echo "flag results differ: $f: "$RESULT".tmp != test"
exit 1
fi
rm $RESULT.tmp
rm test
done
for f in $LEXCWARN ; do
if $TOOL --Werror $FFLAG $srcdir/$f -o test 2> /dev/null; then
echo lexc $FFLAG $f passed although --Werror was used
exit 1
fi
done
if ! $TOOL $FFLAG $srcdir/basic.multi-file-1.lexc \
$srcdir/basic.multi-file-2.lexc \
$srcdir/basic.multi-file-3.lexc -o test 2> /dev/null; then
echo hfst-lexc $FFLAG basic.multi-file-{1,2,3}.lexc failed with $?
exit 1
fi
if ! $COMPARE_TOOL -e -s walk_or_dog$i test ; then
exit 1
fi
done
#exit 77
|
