1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
|
#!/bin/sh
if [ "$1" = "--python" ]; then
exit 77
fi
TOOLDIR=../../tools/src
if [ "$srcdir" = "" ]; then
srcdir="./";
fi
# Prerequisites
if ! $TOOLDIR/hfst-lexc -q < $srcdir/tokenize-dog-in.lexc > $srcdir/tokenize-dog-gen.hfst; then
echo lexc dog fail
exit 1
fi
if ! $TOOLDIR/hfst-invert < $srcdir/tokenize-dog-gen.hfst > $srcdir/tokenize-dog.hfst; then
echo invert dog fail
exit 1
fi
if ! $TOOLDIR/hfst-pmatch2fst < $srcdir/tokenize-dog.pmscript > $srcdir/tokenize-dog.pmhfst; then
echo pmatch2fst tokenize-dog fail
exit 1
fi
# basic lookup
if ! echo "test dog be dog catdog" | $TOOLDIR/hfst-tokenize $srcdir/tokenize-dog.pmhfst > test.strings ; then
echo tokenize fail:
cat test.strings
exit 1
fi
if ! diff test.strings $srcdir/tokenize-dog-out.strings ; then
echo diff test.strings $srcdir/tokenize-dog-out.strings
exit 1
fi
# --cg
if ! echo "test dog be dog catdog" | $TOOLDIR/hfst-tokenize --cg $srcdir/tokenize-dog.pmhfst > test.strings ; then
echo tokenize --cg fail:
cat test.strings
exit 1
fi
if ! diff test.strings $srcdir/tokenize-dog-out-cg.strings ; then
echo diff test.strings $srcdir/tokenize-dog-out-cg.strings
exit 1
fi
# --giella-cg
if ! echo "test dog be dog catdog собака" | $TOOLDIR/hfst-tokenize --giella-cg $srcdir/tokenize-dog.pmhfst > test.strings ; then
echo tokenize --giella-cg fail:
cat test.strings
exit 1
fi
if ! diff test.strings $srcdir/tokenize-dog-out-giella-cg.strings ; then
echo diff test.strings $srcdir/tokenize-dog-out-giella-cg.strings
exit 1
fi
# --xerox
if ! echo "test dog be dog catdog" | $TOOLDIR/hfst-tokenize --xerox $srcdir/tokenize-dog.pmhfst > test.strings ; then
echo tokenize --xerox fail:
cat test.strings
exit 1
fi
if ! diff test.strings $srcdir/tokenize-dog-out-xerox.strings ; then
echo diff test.strings $srcdir/tokenize-dog-out-xerox.strings
exit 1
fi
# --giella-cg superblanks
if ! printf 'dog[\\\n<\\\\>]cat !and \ndogs[][\n]' | $TOOLDIR/hfst-tokenize --giella-cg --superblanks $srcdir/tokenize-dog.pmhfst > test.strings ; then
echo tokenize --giella-cg --superblanks superblank fail:
cat test.strings
exit 1
fi
if ! diff test.strings $srcdir/tokenize-dog-out-giella-cg-superblank.strings ; then
echo diff test.strings $srcdir/tokenize-dog-out-giella-cg-superblank.strings
exit 1
fi
rm test.strings tokenize-dog.pmhfst tokenize-dog.hfst tokenize-dog-gen.hfst
exit 0
|
