#### PATTERN | WORDNET ############################################################################# # -*- coding: utf-8 -*- # Copyright (c) 2010 University of Antwerp, Belgium # Author: Tom De Smedt # License: BSD (see LICENSE.txt for details). # http://www.clips.ua.ac.be/pages/pattern #################################################################################################### # WordNet is a lexical database for English. # It disambiguates word senses, e.g., "tree" in the sense of a plant or in the sense of a graph. # It groups similar word senses into sets of synonyms called synsets, # with a short description and semantic relations to other synsets: # - synonym = a word that is similar in meaning, # - hypernym = a word with a broader meaning, (tree => plant) # - hyponym = a word with a more specific meaning, (tree => oak) # - holonym = a word that is the whole of parts, (tree => forest) # - meronym = a word that is a part of the whole, (tree => trunk) # - antonym = a word that is opposite in meaning. import os import sys import glob from math import log try: MODULE = os.path.dirname(os.path.realpath(__file__)) except: MODULE = "" WORDNET_DICT = "/usr/share/wordnet" # Path to WordNet /dict folder. CORPUS = "" os.environ["WNHOME"] = os.path.join(MODULE, CORPUS) os.environ["WNSEARCHDIR"] = WORDNET_DICT from pywordnet import wordnet as wn from pywordnet import wntools # The bundled version of PyWordNet has custom fixes. # - line 365: check if lexnames exist. # - line 765: check if lexnames exist + use os.path.join(). # - line 674: add HYPONYM and HYPERNYM to the pointer table. # - line 916: implement "x in Dictionary" instead of Dictionary.has_key(x) # - line 804: Dictionary.dataFile now stores a list of (file, size)-tuples. # - line 1134: _dataFilePath() returns a list (i.e., data.noun can be split into data.noun1 + data.noun2). # - line 1186: _lineAt() seeks in second datafile if offset > EOF first datafile. VERSION = "" s = open(os.path.join(WORDNET_DICT, "index.noun")).read(2048) if "WordNet 2.1" in s: VERSION = "2.1" if "WordNet 3.0" in s: VERSION = "3.0" del s #--------------------------------------------------------------------------------------------------- DIACRITICS = { "a": ("á","ä","â","à","å"), "e": ("é","ë","ê","è"), "i": ("í","ï","î","ì"), "o": ("ó","ö","ô","ò","ō","ø"), "u": ("ú","ü","û","ù","ů"), "y": ("ý","ÿ","ý"), "s": ("š",), "c": ("ç","č"), "n": ("ñ",), "z": ("ž",) } def normalize(word): """ Normalizes the word for synsets() or Sentiwordnet[] by removing diacritics (PyWordNet does not take unicode). """ if not isinstance(word, basestring): word = str(word) if not isinstance(word, str): try: word = word.encode("utf-8", "ignore") except: pass for k, v in DIACRITICS.items(): for v in v: word = word.replace(v, k) return word ### SYNSET ######################################################################################### NOUNS, VERBS, ADJECTIVES, ADVERBS = \ wn.N, wn.V, wn.ADJ, wn.ADV NOUN, VERB, ADJECTIVE, ADVERB = \ NN, VB, JJ, RB = \ "NN", "VB", "JJ", "RB" def synsets(word, pos=NOUN): """ Returns a list of Synset objects, one for each word sense. Each word can be understood in different "senses", each of which is part of a set of synonyms (= Synset). """ word, pos = normalize(word), pos.lower() try: if pos.startswith(NOUN.lower()): # "NNS" or "nn" will also pass. w = wn.N[word] elif pos.startswith(VERB.lower()): w = wn.V[word] elif pos.startswith(ADJECTIVE.lower()): w = wn.ADJ[word] elif pos.startswith(ADVERB.lower()): w = wn.ADV[word] else: raise TypeError("part of speech must be NOUN, VERB, ADJECTIVE or ADVERB, not %s" % repr(pos)) return [Synset(s.synset) for i, s in enumerate(w)] except KeyError: return [] return [] class Synset(object): def __init__(self, synset=None, pos=NOUN): """ A set of synonyms that share a common meaning. """ if isinstance(synset, int): synset = wn.getSynset({NN: "n", VB: "v", JJ: "adj", RB: "adv"}[pos], synset) if isinstance(synset, basestring): synset = synsets(synset, pos)[0]._synset self._synset = synset def __iter__(self): for s in self._synset.getSenses(): yield unicode(s.form) def __len__(self): return len(self._synset.getSenses()) def __getitem__(self, i): return unicode(self._synset.getSenses()[i].form) def __eq__(self, synset): return isinstance(synset, Synset) and self.id == synset.id def __ne__(self, synset): return not self.__eq__(synset) def __repr__(self): return "Synset(%s)" % repr(self[0]) @property def id(self): return self._synset.offset @property def pos(self): """ Yields the part-of-speech tag (NOUN, VERB, ADJECTIVE or ADVERB). """ pos = self._synset.pos if pos == "noun": return NOUN if pos == "verb": return VERB if pos == "adjective": return ADJECTIVE if pos == "adverb": return ADVERB part_of_speech = tag = pos @property def synonyms(self): """ Yields a list of word forms (i.e. synonyms), for example: synsets("TV")[0].synonyms => ["television", "telecasting", "TV", "video"] """ return [unicode(s.form) for s in self._synset.getSenses()] senses = synonyms # Backwards compatibility; senses = list of Synsets for a word. @property def gloss(self): """ Yields a descriptive string, for example: synsets("glass")[0].gloss => "a brittle transparent solid with irregular atomic structure". """ return unicode(self._synset.gloss) @property def lexname(self): """ Yields a category, e.g., noun.animal. """ return self._synset.lexname and unicode(self._synset.lexname) or None @property def antonym(self): """ Yields the semantically opposite synset, for example: synsets("death")[0].antonym => Synset("birth"). """ p = self._synset.getPointers(wn.ANTONYM) return len(p) > 0 and Synset(p[0].getTarget()) or None def meronyms(self): """ Yields a list of synsets that are semantic members/parts of this synset, for example: synsets("house")[0].meronyms() => [Synset("library"), Synset("loft"), Synset("porch") ] """ p = self._synset.getPointers(wn.MEMBER_HOLONYM) p+= self._synset.getPointers(wn.PART_HOLONYM) return [Synset(p.getTarget()) for p in p] def holonyms(self): """ Yields a list of synsets of which this synset is a member/part, for example: synsets("tree")[0].holonyms() => Synset("forest"). """ p = self._synset.getPointers(wn.MEMBER_MERONYM) p+= self._synset.getPointers(wn.PART_MERONYM) return [Synset(p.getTarget()) for p in p] def hyponyms(self, recursive=False, depth=None): """ Yields a list of semantically more specific synsets, for example: synsets("train")[0].hyponyms() => [Synset("boat train"), Synset("car train"), Synset("freight train"), Synset("hospital train"), Synset("mail train"), Synset("passenger train"), Synset("streamliner"), Synset("subway train") ] """ p = [Synset(p.getTarget()) for p in self._synset.getPointers(wn.HYPONYM)] if depth is None and recursive is False: return p if depth == 0: return [] if depth is not None: depth -= 1 if depth is None or depth > 0: [p.extend(s.hyponyms(True, depth)) for s in list(p)] return p def hypernyms(self, recursive=False, depth=None): """ Yields a list of semantically broader synsets. """ p = [Synset(p.getTarget()) for p in self._synset.getPointers(wn.HYPERNYM)] if depth is None and recursive is False: return p if depth == 0: return [] if depth is not None: depth -= 1 if depth is None or depth > 0: [p.extend(s.hypernyms(True, depth)) for s in list(p)] return p @property def hypernym(self): """ Yields the synset that is the semantic parent, for example: synsets("train")[0].hypernym => Synset("public transport"). """ p = self._synset.getPointers(wn.HYPERNYM) return len(p) > 0 and Synset(p[0].getTarget()) or None def similar(self): """ Returns a list of similar synsets for adjectives and adverbs, for example: synsets("almigthy",JJ)[0].similar() => Synset("powerful"). """ # ALSO_SEE returns wn.Sense instead of wn.Synset in some cases: s = lambda x: isinstance(x, wn.Sense) and x.synset or x p = [Synset(s(p.getTarget())) for p in self._synset.getPointers(wn.SIMILAR)] p+= [Synset(s(p.getTarget())) for p in self._synset.getPointers(wn.ALSO_SEE)] return p def similarity(self, synset): """ Returns the semantic similarity of the given synsets (0.0-1.0). synsets("cat")[0].similarity(synsets("dog")[0]) => 0.86. synsets("cat")[0].similarity(synsets("box")[0]) => 0.17. """ if self == synset: return 1.0 try: # Lin semantic distance measure. lin = 2.0 * log(lcs(self, synset).ic) / (log(self.ic * synset.ic) or 1) except OverflowError: lin = 0.0 except ValueError: # / log(0) lin = 0.0 return abs(lin) @property def ic(self): return information_content(self) @property def weight(self): return sentiwordnet is not None \ and sentiwordnet.synset(self.id, self.pos)[:2] \ or None def similarity(synset1, synset2): """ Returns the semantic similarity of the given synsets. """ return synset1.similarity(synset2) def ancestor(synset1, synset2): """ Returns the common ancestor of both synsets. For example synsets("cat")[0].ancestor(synsets("dog")[0]) => Synset("carnivore") """ h1, h2 = synset1.hypernyms(recursive=True), synset2.hypernyms(recursive=True) for s in h1: if s in h2: return s least_common_subsumer = lcs = ancestor ### INFORMATION CONTENT ############################################################################ # Information Content (IC) is used to calculate semantic similarity in Synset.similarity(). # Information Content values for each synset are derived from word frequency in a given corpus. # The idea is that less frequent words convey more information. # Semantic similarity depends on the amount of information two concepts (synsets) have in common, # given by the Most Specific Common Abstraction (MSCA), i.e. the shared ancestor in the taxonomy. # http://www.d.umn.edu/~tpederse/Pubs/AAAI04PedersenT.pdf # http://afflatus.ucd.ie/papers/ecai2004b.pdf IC = {} # Switch data file according to WordNet version: IC_CORPUS = os.path.join(MODULE, "resnik-ic" + VERSION[0] + ".txt") IC_MAX = 0 def information_content(synset): """ Returns the IC value for the given Synset (trained on the Brown corpus). """ global IC_MAX if not IC: IC[NOUN] = {} IC[VERB] = {} for s in open(IC_CORPUS).readlines()[1:]: # Skip the header. s = s.split() id, w, pos = ( int(s[0][:-1]), float(s[1]), s[0][-1] == "n" and NOUN or VERB) if len(s) == 3 and s[2] == "ROOT": IC[pos][0] = IC[pos].get(0,0) + w if w != 0: IC[pos][id] = w if w > IC_MAX: IC_MAX = w return IC.get(synset.pos, {}).get(synset.id, 0.0) / IC_MAX ### WORDNET3 TO WORDNET2 ########################################################################### # Map WordNet3 synset id's to WordNet2 synset id's. _map32_pos1 = {NN: "n", VB: "v", JJ: "a", RB: "r"} _map32_pos2 = {"n": NN, "v": VB, "a": JJ, "r": RB} _map32_cache = None def map32(id, pos=NOUN): """ Returns an (id, pos)-tuple with the WordNet2 synset id for the given WordNet3 synset id. Returns None if no id was found. """ global _map32_cache if not _map32_cache: _map32_cache = open(os.path.join(MODULE, "dict", "index.32")).readlines() _map32_cache = (x for x in _map32_cache if x[0] != ";") # comments _map32_cache = dict(x.strip().split(" ") for x in _map32_cache) k = pos in _map32_pos2 and pos or _map32_pos1.get(pos, "x") k+= str(id).lstrip("0") k = _map32_cache.get(k, None) if k is not None: return int(k[1:]), _map32_pos2[k[0]] return None #### SENTIWORDNET ################################################################################## # http://nmis.isti.cnr.it/sebastiani/Publications/LREC06.pdf # http://nmis.isti.cnr.it/sebastiani/Publications/LREC10.pdf sys.path.insert(0, os.path.join(MODULE, "..", "..")) try: from pattern.text import Sentiment except: class Sentiment(object): PLACEHOLDER = True sys.path.pop(0) class SentiWordNet(Sentiment): def __init__(self, path="SentiWordNet*.txt", language="en"): """ A sentiment lexicon with scores from SentiWordNet. The value for each word is a tuple with values for polarity (-1.0-1.0), subjectivity (0.0-1.0) and intensity (0.5-2.0). """ Sentiment.__init__(self, path=path, language=language) def load(self): # Backwards compatibility: look for SentiWordNet*.txt in: # given path, pattern/text/en/ or pattern/text/en/wordnet/ try: f = ( glob.glob(os.path.join(self.path)) + \ glob.glob(os.path.join(MODULE, self.path)) + \ glob.glob(os.path.join(MODULE, "..", self.path)))[0] except IndexError: raise ImportError("can't find SentiWordnet data file") # Map synset id: a-00193480" => (193480, JJ). # Map synset id's to WordNet2 if VERSION == 2: if int(float(VERSION)) == 3: m = lambda id, pos: (int(id.lstrip("0")), _map32_pos2[pos]) if int(float(VERSION)) == 2: m = map32 for s in open(f): if not s.startswith(("#", "\t")): pos, id, p, n, senses, gloss = s.split("\t") w = senses.split() k = m(id, pos) v = (float(p) - float(n), float(p) + float(n) ) # Apply the score to the first synonym in the synset. # Several WordNet3 entries may point to the same WordNet2 entry. if k is not None: k = "%s-%s" % (pos, str(k[0]).zfill(8)) # "a-00193480" if k not in self._synsets or w[0].endswith("#1"): self._synsets[k] = v for w in w: if w.endswith("#1"): dict.__setitem__(self, w[:-2].replace("_", " "), v) # Words are stored without diacritics, # use wordnet.normalize(word). def __getitem__(self, k): return Sentiment.__getitem__(self, normalize(k)) def get(self, k, *args, **kwargs): return Sentiment.get(self, normalize(k), *args, **kwargs) def assessments(self, words=[], negation=True): raise NotImplementedError def __call__(self, s, negation=True): raise NotImplementedError if not hasattr(Sentiment, "PLACEHOLDER"): sentiwordnet = SentiWordNet() else: sentiwordnet = None # Backwards compatibility. # Older code may be using pattern.en.wordnet.sentiment[w], # which yields a (positive, negative, neutral)-tuple. class sentiment(object): def load(self, **kwargs): sentiwordnet.load(**kwargs) def __getitem__(self, w): p, s = sentiwordnet.get(w, (0.0, 0.0)) return p < 0 and (0.0, -p, 1.0-s) or (p, 0.0, 1.0-s) def __contains__(self, w): return w in sentiwordnet sentiment = sentiment() #print sentiwordnet["industry"] # (0.0, 0.0) #print sentiwordnet["horrible"] # (-0.625, 0.625) #print sentiwordnet.synset(synsets("horrible", pos="JJ")[0].id, pos="JJ") #print synsets("horrible", pos="JJ")[0].weight