1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
|
#!/bin/sh
unset LANG LC_CTYPE LC_ALL LC_COLLATE
make
echo
echo "Statistics on input files:"
echo "--------------------------"
echo -n "wolig.dat: "
echo -n `grep " " wolig.dat | grep -vc "^#"`
echo -n " noun lines, "
echo -n `grep " " wolig.dat | grep -vc "^#"`
echo " adjective lines."
echo -n "woo.dat: "
echo -n `grep " " woo.dat | grep -vc "^#"`
echo " verb lines."
echo -n "shemp.dat: "
echo -n `grep " " shemp.dat | grep -vc "^#"`
echo " auto-generated gerunds."
echo -n "misc data lines:" `egrep -hcv "^[-#]|^$" extrawords.hif` "extrawords, "
echo -n `grep -hcv "^[-#]" milot.hif` "milot, "
echo -n `grep -hcv "^[-#]" biza-verbs.hif` "bizaverbs, "
echo `grep -hc "^[-#]" biza-nouns.hif` "bizanouns. "
echo
echo "Unique baseword counts:"
echo "-----------------------"
NN=`grep -h " " wolig.dat shemp.dat | sed "/^#/d;s/ *#.*$//" | sort -u | wc -l`
NN1=`grep -h " " wolig.dat shemp.dat | sed "/^#/d;s/ *#.*$//" | sort -u | grep -vc "$"`
NN2=`sed "s/#.*$//" < wolig.dat | egrep ",(|)" | grep ","| wc -l`
NN3=`grep -h " " wolig.dat shemp.dat | sed "/^#/d;s/ *#.*$//" | sort -u | egrep ",(|||||)" |wc -l`
echo Nouns: $NN "(of" them, $NN3 need plural hints, $NN1 need inflection hints, $NN2 explicit "gender)."
VV=`grep -c -- ---- verbs.hif`
echo Verbs: $VV
AA=`grep " " wolig.dat | grep -v "^#" | sed "s/ *#.*$//" | sort -u | wc -l`
echo Adjectives: $AA
EE=`grep -hv "^[-#]" extrawords.hif milot.hif biza-verbs.hif biza-nouns.hif | sed "s/ *#.*$//" | tr -d - | sort -u | wc -l`
echo Other words: $EE
echo
echo Total number of base words - `expr $NN + $VV + $AA + $EE`
echo
echo "Final word count:"
echo "-----------------"
# we can count words in hebrew.wgz even without compiling wunzip :)
WW=`zcat hebrew.wgz | tr [0-9] '\012' | grep -vc "^$"`
echo Unique words in hebrew.wgz: $WW
echo "Dictionary file sizes (in bytes):"
wc -c hebrew.wgz*
echo "Memory use (spell-checker only):"
gzip -dc hebrew.wgz | ./find_sizes >/dev/null
# NOTE: to find duplicates in wolig.dat:
# grep " " wolig.dat | grep -v "^#"| sed "s/ *#.*$//"|sort |uniq -c | sort -n | less
|
