1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
|
#!/usr/bin/env python3
from __future__ import print_function, unicode_literals, division, absolute_import
import argparse
import sys
from math import log
from collections import defaultdict
def pmi(sentences1, sentences2,discount = 0):
jointcount = len(sentences1 & sentences2) - discount
if jointcount <= 0: return None
return log( jointcount / (len(sentences1) * len(sentences2))), jointcount+discount
def npmi(sentences1, sentences2,discount=0):
jointcount = len(sentences1 & sentences2) - discount
if jointcount <= 0: return None
return log( jointcount / (len(sentences1) * len(sentences2))) / -log(jointcount), jointcount+discount
def main():
parser = argparse.ArgumentParser(description="Simple cooccurence computation", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('-f','--inputtext', type=str,help="Input file (plaintext, tokenised, utf-8, one sentence per line)", action='store',default="",required=True)
parser.add_argument('-s','--sorted', help="Output sorted by co-occurrence score", action='store_true',default=False)
parser.add_argument('-t','--threshold', help="Joined occurrence threshold, do not consider words occuring less than this", type=int, action='store',default=1)
parser.add_argument('-a','--adjacency', help="Compute the adjacency fraction (how many co-occurrence are immediate bigrams)", action='store_true',default=False)
parser.add_argument('-A','--discountadjacency', help="Do not take immediately adjacent fragments (bigrams) into account when computing mutual information (requires -a)", action='store_true',default=False)
parser.add_argument('--pmi',help="Compute pointwise mutual information", action='store_true',default=False)
parser.add_argument('--npmi',help="Compute normalised pointwise mutual information", action='store_true',default=False)
parser.add_argument('--jaccard',help="Compute jaccard similarity coefficient", action='store_true',default=False)
parser.add_argument('--dice',help="Compute dice coefficient", action='store_true',default=False)
args = parser.parse_args()
if not args.pmi and not args.npmi and not args.jaccard and not args.dice:
args.pmi = True
count = defaultdict(int)
cooc = defaultdict(lambda: defaultdict(int))
adjacent = defaultdict(lambda: defaultdict(int))
total = 0
f = open(args.inputtext,'r',encoding='utf-8')
for i, line in enumerate(f):
sentence = i + 1
if sentence % 1000 == 0: print("Indexing @" + str(sentence),file=sys.stderr)
if line:
words = list(enumerate(line.split()))
for pos, word in words:
count[word] += 1
total += 1
for pos2, word2 in words:
if pos2 > pos:
cooc[word][word2] += 1
if args.adjacency and pos2 == pos + len(word.split()):
adjacent[word][word2] += 1
f.close()
l = len(cooc)
output = []
for i, (word, coocdata) in enumerate(cooc.items()):
print("Computing mutual information @" + str(i+1) + "/" + str(l) + ": \"" + word + "\" , co-occurs with " + str(len(coocdata)) + " words",file=sys.stderr)
for word2, jointcount in coocdata.items():
if jointcount> args.threshold:
if args.adjacency and word in adjacent and word2 in adjacent[word]:
adjcount = adjacent[word][word2]
else:
adjcount = 0
if args.discountadjacency:
discount = adjcount
else:
discount = 0
if args.pmi:
score = log( ((jointcount-discount)/total) / ((count[word]/total) * (count[word2]/total)))
elif args.npmi:
score = log( ((jointcount-discount)/total) / ((count[word]/total) * (count[word2]/total))) / -log((jointcount-discount)/total)
elif args.jaccard or args.dice:
score = (jointcount-discount) / (count[word] + count[word2] - (jointcount - discount) )
if args.dice:
score = 2*score / (1+score)
if args.sorted:
outputdata = (word,word2,score, jointcount, adjcount, adjcount / jointcount if args.adjacency else None)
output.append(outputdata)
else:
if args.adjacency:
print(word + "\t" + word2 + "\t" + str(score) + "\t" + str(jointcount) + "\t" + str(adjcount) + "\t" + str(adjcount / jointcount))
else:
print(word + "\t" + word2 + "\t" + str(score) + "\t" + str(jointcount))
if args.sorted:
print("Outputting " + str(len(output)) + " pairs",file=sys.stderr)
if args.adjacency:
print("#WORD\tWORD2\tSCORE\tJOINTCOUNT\tBIGRAMCOUNT\tBIGRAMRATIO")
else:
print("#WORD\tWORD2\tSCORE\tJOINTCOUNT\tBIGRAMCOUNT\tBIGRAMRATIO")
if args.npmi:
sign = 1
else:
sign = -1
for word,word2,score,jointcount,adjcount, adjratio in sorted(output, key=lambda x: sign * x[2]):
if args.adjacency:
print(word + "\t" + word2 + "\t" + str(score) + "\t" + str(jointcount) + "\t" + str(adjcount) + "\t" + str(adjratio) )
else:
print(word + "\t" + word2 + "\t" + str(score) + "\t" + str(jointcount))
if __name__ == '__main__':
main()
|
