Index: pattern-5b85d998c30ddc6772b56310713530224466083a/test/test_de.py =================================================================== --- pattern-5b85d998c30ddc6772b56310713530224466083a.orig/test/test_de.py +++ pattern-5b85d998c30ddc6772b56310713530224466083a/test/test_de.py @@ -104,13 +104,9 @@ class TestInflection(unittest.TestCase): for v1, v2 in de.inflect.verbs.inflections.items(): if de.inflect.verbs.find_lemma(v1) == v2: i += 1 - else: - pass n += 1 self.assertTrue(float(i) / n > 0.86) - rate_correct = float(i) / n - self.assertTrue(float(i) / n > 0.86) - print("pattern.de.inflect.verbs.find_lemma() - hit rate {}".format(rate_correct)) + print("pattern.de.inflect.verbs.find_lemma()") def test_find_lexeme(self): # Assert the accuracy of the verb conjugation algorithm. @@ -123,9 +119,8 @@ class TestInflection(unittest.TestCase): if lexeme1[j] == lexeme2[j]: i += 1 n += 1 - rate_correct = float(i) / n self.assertTrue(float(i) / n > 0.86) - print("pattern.de.inflect.verbs.find_lexeme() - hit rate {}".format(rate_correct)) + print("pattern.de.inflect.verbs.find_lexeme()") def test_conjugate(self): # Assert different tenses with different conjugations. @@ -159,22 +154,8 @@ class TestInflection(unittest.TestCase): ("sein", "wäre", (de.PAST, 3, de.SINGULAR, de.SUBJUNCTIVE)), ("sein", "wären", (de.PAST, 1, de.PLURAL, de.SUBJUNCTIVE)), ("sein", "wäret", (de.PAST, 2, de.PLURAL, de.SUBJUNCTIVE)), - ("sein", "wären", (de.PAST, 3, de.PLURAL, de.SUBJUNCTIVE)), - ("vorgehen", "gingst vor", (de.PAST, 2, de.SINGULAR)), # separable prefix, irregular base - ("betreffen", "betroffen", (de.PAST, de.PARTICIPLE)), # inseparable prefix, irregular base - ("umbenennen", "benanntest um", (de.PAST, 2, de.SINGULAR)), # stacked prefixes, irregular base - ("einberufen", 'berief ein', (de.PAST, 3, de.SINGULAR)), # stacked prefixes, irregular base - ('entern', 'geentert', (de.PAST, de.PARTICIPLE)), # looks like prefix ent- - ('zurren', 'zurrt', (de.PRESENT, 3, de.SINGULAR)), # looks like prefix zu- - ('bechern', 'gebechert', (de.PAST, de.PARTICIPLE)), # looks like prefix be- - ('drangsalieren', 'drangsaliert', (de.PAST, de.PARTICIPLE)), # blocking ge-prefixation - ('stapfen', 'gestapft', (de.PAST, de.PARTICIPLE)), # - ('fristen', 'gefristet', (de.PAST, de.PARTICIPLE)), # might be misinterpreted as past form - ('gieren', 'gegiert', (de.PAST, de.PARTICIPLE)), # ends in -ieren but doesn't block ge-prefixation - ('angeln', 'angeltest', (de.PAST, 2, de.SINGULAR)), # looks like prefix an- - ('geifern', 'gegeifert', (de.PAST, de.PARTICIPLE)), # looks like prefix ge- - ): - self.assertEqual(de.conjugate(v1, tense, allow_inflected=False), v2,) + ("sein", "wären", (de.PAST, 3, de.PLURAL, de.SUBJUNCTIVE))): + self.assertEqual(de.conjugate(v1, tense), v2) print("pattern.de.conjugate()") def test_lexeme(self): @@ -192,8 +173,6 @@ class TestInflection(unittest.TestCase): # Assert tense recognition. self.assertTrue((de.PRESENT, 3, de.SG) in de.tenses("ist")) self.assertTrue("2sg" in de.tenses("bist")) - self.assertTrue((de.PAST, 2, de.SINGULAR) in de.tenses('gingst vor')) - self.assertTrue((de.PRESENT, 2, de.SINGULAR, de.SUBJUNCTIVE) in de.tenses('gehest vor')) print("pattern.de.tenses()") #--------------------------------------------------------------------------------------------------- @@ -206,12 +185,9 @@ class TestParser(unittest.TestCase): def test_find_lemmata(self): # Assert lemmata for nouns, adjectives and verbs. - v = de.parser.find_lemmata([['Man', 'PRP'], ['nimmt an', 'VB'], [',', 'PUNC'], ["ich", "PRP"], ["sage", "VB"], ["schöne", "JJ"], ["Dinge", "NNS"]]) + v = de.parser.find_lemmata([["Ich", "PRP"], ["sage", "VB"], ["schöne", "JJ"], ["Dinge", "NNS"]]) self.assertEqual(v, [ - ['Man', 'PRP', 'man'], - ['nimmt an', 'VB', 'annehmen'], - [',', 'PUNC', ','], - ["ich", "PRP", "ich"], + ["Ich", "PRP", "ich"], ["sage", "VB", "sagen"], ["schöne", "JJ", "schön"], ["Dinge", "NNS", "ding"]]) @@ -258,13 +234,13 @@ class TestParser(unittest.TestCase): def test_command_line(self): # Assert parsed output from the command-line (example from the documentation). - p = ["python", "-m", "pattern.de", "-s", "Der grosse Hund.", "-OTCRL"] + p = ["python3", "-m", "pattern.de", "-s", "Der grosse Hund.", "-OTCRL"] p = subprocess.Popen(p, stdout=subprocess.PIPE) p.wait() v = p.stdout.read().decode('utf-8') v = v.strip() self.assertEqual(v, "Der/DT/B-NP/O/O/der grosse/JJ/I-NP/O/O/gross Hund/NN/I-NP/O/O/hund ././O/O/O/.") - print("python -m pattern.de") + print("python3 -m pattern.de") #--------------------------------------------------------------------------------------------------- Index: pattern-5b85d998c30ddc6772b56310713530224466083a/test/test_en.py =================================================================== --- pattern-5b85d998c30ddc6772b56310713530224466083a.orig/test/test_en.py +++ pattern-5b85d998c30ddc6772b56310713530224466083a/test/test_en.py @@ -15,6 +15,9 @@ import unittest import random import subprocess +from random import seed +seed(0) + from pattern import text from pattern import en @@ -59,7 +62,7 @@ class TestInflection(unittest.TestCase): # Assert the accuracy of the pluralization algorithm. from pattern.db import Datasheet i, n = 0, 0 - for sg, pl in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-en-celex.csv")): + for sg, pl in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-en-celex.csv"), encoding='utf8'): if en.inflect.pluralize(sg) == pl: i += 1 n += 1 @@ -70,7 +73,7 @@ class TestInflection(unittest.TestCase): # Assert the accuracy of the singularization algorithm. from pattern.db import Datasheet i, n = 0, 0 - for sg, pl in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-en-celex.csv")): + for sg, pl in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-en-celex.csv"), encoding='utf8'): if en.inflect.singularize(pl) == sg: i += 1 n += 1 @@ -293,7 +296,7 @@ class TestSpelling(unittest.TestCase): # Note: simply training on more text will not improve accuracy. i = j = 0.0 from pattern.db import Datasheet - for correct, wrong in Datasheet.load(os.path.join(PATH, "corpora", "spelling-birkbeck.csv")): + for correct, wrong in Datasheet.load(os.path.join(PATH, "corpora", "spelling-birkbeck.csv"), encoding='utf8'): for w in wrong.split(" "): if en.suggest(w)[0][0] == correct: i += 1 @@ -546,7 +549,7 @@ class TestParser(unittest.TestCase): # 7) Assert the accuracy of the English tagger. i, n = 0, 0 for corpus, a in (("tagged-en-wsj.txt", (0.968, 0.945)), ("tagged-en-oanc.txt", (0.929, 0.932))): - for sentence in open(os.path.join(PATH, "corpora", corpus)).readlines(): + for sentence in open(os.path.join(PATH, "corpora", corpus), encoding='utf8').readlines(): sentence = sentence.strip() s1 = [w.split("/") for w in sentence.split(" ")] s2 = [[w for w, pos in s1]] @@ -606,13 +609,13 @@ class TestParser(unittest.TestCase): def test_command_line(self): # Assert parsed output from the command-line (example from the documentation). - p = ["python", "-m", "pattern.en", "-s", "Nice cat.", "-OTCRL"] + p = ["python3", "-m", "pattern.en", "-s", "Nice cat.", "-OTCRL"] p = subprocess.Popen(p, stdout=subprocess.PIPE) p.wait() v = p.stdout.read().decode('utf-8') v = v.strip() self.assertEqual(v, "Nice/JJ/B-NP/O/O/nice cat/NN/I-NP/O/O/cat ././O/O/O/.") - print("python -m pattern.en") + print("python3 -m pattern.en") #--------------------------------------------------------------------------------------------------- @@ -886,7 +889,7 @@ class TestModality(unittest.TestCase): from pattern.db import Datasheet from pattern.metrics import test sentences = [] - for certain, sentence in Datasheet.load(os.path.join(PATH, "corpora", "uncertainty-conll2010.csv")): + for certain, sentence in Datasheet.load(os.path.join(PATH, "corpora", "uncertainty-conll2010.csv"), encoding='utf8'): sentence = en.parse(sentence, chunks=False, light=True) sentence = en.Sentence(sentence) sentences.append((sentence, int(certain) > 0)) @@ -929,7 +932,7 @@ class TestSentiment(unittest.TestCase): from pattern.db import Datasheet from pattern.metrics import test reviews = [] - for score, review in Datasheet.load(os.path.join(PATH, "corpora", "polarity-en-pang&lee1.csv")): + for score, review in Datasheet.load(os.path.join(PATH, "corpora", "polarity-en-pang&lee1.csv"), encoding='utf8'): reviews.append((review, int(score) > 0)) from time import time t = time() @@ -943,7 +946,7 @@ class TestSentiment(unittest.TestCase): # Given are the scores for Pang & Lee's sentence polarity dataset v1.0: # http://www.cs.cornell.edu/people/pabo/movie-review-data/ reviews = [] - for score, review in Datasheet.load(os.path.join(PATH, "corpora", "polarity-en-pang&lee2.csv")): + for score, review in Datasheet.load(os.path.join(PATH, "corpora", "polarity-en-pang&lee2.csv"), encoding='utf8'): reviews.append((review, int(score) > 0)) A, P, R, F = test(lambda review: en.positive(review), reviews) #print(A, P, R, F) Index: pattern-5b85d998c30ddc6772b56310713530224466083a/test/test_es.py =================================================================== --- pattern-5b85d998c30ddc6772b56310713530224466083a.orig/test/test_es.py +++ pattern-5b85d998c30ddc6772b56310713530224466083a/test/test_es.py @@ -35,7 +35,7 @@ class TestInflection(unittest.TestCase): # Assert the accuracy of the pluralization algorithm. from pattern.db import Datasheet test = {} - for w, lemma, tag, f in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-es-davies.csv")): + for w, lemma, tag, f in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-es-davies.csv"), encoding='utf8'): if tag == "n": test.setdefault(lemma, []).append(w) i, n = 0, 0 @@ -51,7 +51,7 @@ class TestInflection(unittest.TestCase): # Assert the accuracy of the singularization algorithm. from pattern.db import Datasheet test = {} - for w, lemma, tag, f in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-es-davies.csv")): + for w, lemma, tag, f in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-es-davies.csv"), encoding='utf8'): if tag == "n": test.setdefault(lemma, []).append(w) i, n = 0, 0 @@ -80,7 +80,7 @@ class TestInflection(unittest.TestCase): # Assert the accuracy of the predicative algorithm ("horribles" => "horrible"). from pattern.db import Datasheet test = {} - for w, lemma, tag, f in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-es-davies.csv")): + for w, lemma, tag, f in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-es-davies.csv"), encoding='utf8'): if tag == "j": test.setdefault(lemma, []).append(w) i, n = 0, 0 @@ -232,7 +232,7 @@ class TestParser(unittest.TestCase): ) # Assert the accuracy of the Spanish tagger. i, n = 0, 0 - for sentence in open(os.path.join(PATH, "corpora", "tagged-es-wikicorpus.txt")).readlines(): + for sentence in open(os.path.join(PATH, "corpora", "tagged-es-wikicorpus.txt"), encoding='utf8').readlines(): sentence = sentence.strip() s1 = [w.split("/") for w in sentence.split(" ")] s2 = [[w for w, pos in s1]] @@ -254,13 +254,13 @@ class TestParser(unittest.TestCase): def test_command_line(self): # Assert parsed output from the command-line (example from the documentation). - p = ["python", "-m", "pattern.es", "-s", "El gato negro.", "-OTCRL"] + p = ["python3", "-m", "pattern.es", "-s", "El gato negro.", "-OTCRL"] p = subprocess.Popen(p, stdout=subprocess.PIPE) p.wait() v = p.stdout.read().decode('utf-8') v = v.strip() self.assertEqual(v, "El/DT/B-NP/O/O/el gato/NN/I-NP/O/O/gato negro/JJ/I-NP/O/O/negro ././O/O/O/.") - print("python -m pattern.es") + print("python3 -m pattern.es") #--------------------------------------------------------------------------------------------------- Index: pattern-5b85d998c30ddc6772b56310713530224466083a/test/test_fr.py =================================================================== --- pattern-5b85d998c30ddc6772b56310713530224466083a.orig/test/test_fr.py +++ pattern-5b85d998c30ddc6772b56310713530224466083a/test/test_fr.py @@ -35,7 +35,7 @@ class TestInflection(unittest.TestCase): # Assert the accuracy of the predicative algorithm ("belles" => "bea"). from pattern.db import Datasheet i, n = 0, 0 - for pred, attr, tag in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-fr-lexique.csv")): + for pred, attr, tag in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-fr-lexique.csv"), encoding='utf8'): if tag == "a": if fr.predicative(attr) == pred: i += 1 @@ -179,7 +179,7 @@ class TestParser(unittest.TestCase): # Assert the accuracy of the French tagger. f = fr.penntreebank2universal i, n = 0, 0 - for sentence in open(os.path.join(PATH, "corpora", "tagged-fr-wikinews.txt")).readlines(): + for sentence in open(os.path.join(PATH, "corpora", "tagged-fr-wikinews.txt"), encoding='utf8').readlines(): sentence = sentence.strip() s1 = [w.split("/") for w in sentence.split(" ")] s2 = [[w for w, pos in s1]] @@ -201,13 +201,13 @@ class TestParser(unittest.TestCase): def test_command_line(self): # Assert parsed output from the command-line (example from the documentation). - p = ["python", "-m", "pattern.fr", "-s", "Le chat noir.", "-OTCRL"] + p = ["python3", "-m", "pattern.fr", "-s", "Le chat noir.", "-OTCRL"] p = subprocess.Popen(p, stdout=subprocess.PIPE) p.wait() v = p.stdout.read().decode('utf-8') v = v.strip() self.assertEqual(v, "Le/DT/B-NP/O/O/le chat/NN/I-NP/O/O/chat noir/JJ/I-NP/O/O/noir ././O/O/O/.") - print("python -m pattern.fr") + print("python3 -m pattern.fr") #--------------------------------------------------------------------------------------------------- @@ -227,7 +227,7 @@ class TestSentiment(unittest.TestCase): from pattern.db import Datasheet from pattern.metrics import test reviews = [] - for review, score in Datasheet.load(os.path.join(PATH, "corpora", "polarity-fr-amazon.csv")): + for review, score in Datasheet.load(os.path.join(PATH, "corpora", "polarity-fr-amazon.csv"), encoding='utf8'): reviews.append((review, int(score) > 0)) A, P, R, F = test(lambda review: fr.positive(review), reviews) #print(A, P, R, F) Index: pattern-5b85d998c30ddc6772b56310713530224466083a/test/test_it.py =================================================================== --- pattern-5b85d998c30ddc6772b56310713530224466083a.orig/test/test_it.py +++ pattern-5b85d998c30ddc6772b56310713530224466083a/test/test_it.py @@ -61,7 +61,7 @@ class TestInflection(unittest.TestCase): # Assert the accuracy of the gender disambiguation algorithm. from pattern.db import Datasheet i, n = 0, 0 - for pos, sg, pl, mf in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-it-wiktionary.csv")): + for pos, sg, pl, mf in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-it-wiktionary.csv"), encoding='utf8'): g = it.gender(sg) if mf in g and it.PLURAL not in g: i += 1 @@ -76,7 +76,7 @@ class TestInflection(unittest.TestCase): # Assert the accuracy of the pluralization algorithm. from pattern.db import Datasheet i, n = 0, 0 - for pos, sg, pl, mf in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-it-wiktionary.csv")): + for pos, sg, pl, mf in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-it-wiktionary.csv"), encoding='utf8'): if it.pluralize(sg) == pl: i += 1 n += 1 @@ -87,7 +87,7 @@ class TestInflection(unittest.TestCase): # Assert the accuracy of the singularization algorithm. from pattern.db import Datasheet i, n = 0, 0 - for pos, sg, pl, mf in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-it-wiktionary.csv")): + for pos, sg, pl, mf in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-it-wiktionary.csv"), encoding='utf8'): if it.singularize(pl) == sg: i += 1 n += 1 @@ -99,7 +99,7 @@ class TestInflection(unittest.TestCase): from pattern.db import Datasheet i, n = 0, 0 - for pos, sg, pl, mf in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-it-wiktionary.csv")): + for pos, sg, pl, mf in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-it-wiktionary.csv"), encoding='utf8'): if pos != "j": continue if it.predicative(pl) == sg: @@ -240,7 +240,7 @@ class TestParser(unittest.TestCase): ) # Assert the accuracy of the Italian tagger. i, n = 0, 0 - for sentence in open(os.path.join(PATH, "corpora", "tagged-it-wacky.txt")).readlines(): + for sentence in open(os.path.join(PATH, "corpora", "tagged-it-wacky.txt"), encoding='utf8').readlines(): sentence = sentence.strip() s1 = [w.split("/") for w in sentence.split(" ")] s2 = [[w for w, pos in s1]] @@ -267,13 +267,13 @@ class TestParser(unittest.TestCase): def test_command_line(self): # Assert parsed output from the command-line (example from the documentation). - p = ["python", "-m", "pattern.it", "-s", "Il gatto nero.", "-OTCRL"] + p = ["python3", "-m", "pattern.it", "-s", "Il gatto nero.", "-OTCRL"] p = subprocess.Popen(p, stdout=subprocess.PIPE) p.wait() v = p.stdout.read().decode('utf-8') v = v.strip() self.assertEqual(v, "Il/DT/B-NP/O/O/il gatto/NN/I-NP/O/O/gatto nero/JJ/I-NP/O/O/nero ././O/O/O/.") - print("python -m pattern.it") + print("python3 -m pattern.it") #--------------------------------------------------------------------------------------------------- Index: pattern-5b85d998c30ddc6772b56310713530224466083a/test/test_metrics.py =================================================================== --- pattern-5b85d998c30ddc6772b56310713530224466083a.orig/test/test_metrics.py +++ pattern-5b85d998c30ddc6772b56310713530224466083a/test/test_metrics.py @@ -156,7 +156,7 @@ class TestTextMetrics(unittest.TestCase) def test_intertextuality(self): # Evaluate accuracy for plagiarism detection. from pattern.db import Datasheet - data = Datasheet.load(os.path.join(PATH, "corpora", "plagiarism-clough&stevenson.csv")) + data = Datasheet.load(os.path.join(PATH, "corpora", "plagiarism-clough&stevenson.csv"), encoding='utf8') data = [((txt, src), int(plagiarism) > 0) for txt, src, plagiarism in data] def plagiarism(txt, src): Index: pattern-5b85d998c30ddc6772b56310713530224466083a/test/test_nl.py =================================================================== --- pattern-5b85d998c30ddc6772b56310713530224466083a.orig/test/test_nl.py +++ pattern-5b85d998c30ddc6772b56310713530224466083a/test/test_nl.py @@ -37,7 +37,7 @@ class TestInflection(unittest.TestCase): # Assert the accuracy of the pluralization algorithm. from pattern.db import Datasheet i, n = 0, 0 - for pred, attr, sg, pl in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-nl-celex.csv")): + for pred, attr, sg, pl in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-nl-celex.csv"), encoding='utf8'): if nl.pluralize(sg) == pl: i += 1 n += 1 @@ -48,7 +48,7 @@ class TestInflection(unittest.TestCase): # Assert the accuracy of the singularization algorithm. from pattern.db import Datasheet i, n = 0, 0 - for pred, attr, sg, pl in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-nl-celex.csv")): + for pred, attr, sg, pl in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-nl-celex.csv"), encoding='utf8'): if nl.singularize(pl) == sg: i += 1 n += 1 @@ -59,7 +59,7 @@ class TestInflection(unittest.TestCase): # Assert the accuracy of the attributive algorithm ("fel" => "felle"). from pattern.db import Datasheet i, n = 0, 0 - for pred, attr, sg, pl in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-nl-celex.csv")): + for pred, attr, sg, pl in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-nl-celex.csv"), encoding='utf8'): if nl.attributive(pred) == attr: i += 1 n += 1 @@ -70,7 +70,7 @@ class TestInflection(unittest.TestCase): # Assert the accuracy of the predicative algorithm ("felle" => "fel"). from pattern.db import Datasheet i, n = 0, 0 - for pred, attr, sg, pl in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-nl-celex.csv")): + for pred, attr, sg, pl in Datasheet.load(os.path.join(PATH, "corpora", "wordforms-nl-celex.csv"), encoding='utf8'): if nl.predicative(attr) == pred: i += 1 n += 1 @@ -218,7 +218,7 @@ class TestParser(unittest.TestCase): ) # Assert the accuracy of the Dutch tagger. i, n = 0, 0 - for sentence in open(os.path.join(PATH, "corpora", "tagged-nl-twnc.txt")).readlines(): + for sentence in open(os.path.join(PATH, "corpora", "tagged-nl-twnc.txt"), encoding='utf8').readlines(): sentence = sentence.strip() s1 = [w.split("/") for w in sentence.split(" ")] s1 = [nl.wotan2penntreebank(w, tag) for w, tag in s1] @@ -240,13 +240,13 @@ class TestParser(unittest.TestCase): def test_command_line(self): # Assert parsed output from the command-line (example from the documentation). - p = ["python", "-m", "pattern.nl", "-s", "Leuke kat.", "-OTCRL"] + p = ["python3", "-m", "pattern.nl", "-s", "Leuke kat.", "-OTCRL"] p = subprocess.Popen(p, stdout=subprocess.PIPE) p.wait() v = p.stdout.read().decode('utf-8') v = v.strip() self.assertEqual(v, "Leuke/JJ/B-NP/O/O/leuk kat/NN/I-NP/O/O/kat ././O/O/O/.") - print("python -m pattern.nl") + print("python3 -m pattern.nl") #--------------------------------------------------------------------------------------------------- @@ -266,7 +266,7 @@ class TestSentiment(unittest.TestCase): from pattern.db import Datasheet from pattern.metrics import test reviews = [] - for score, review in Datasheet.load(os.path.join(PATH, "corpora", "polarity-nl-bol.com.csv")): + for score, review in Datasheet.load(os.path.join(PATH, "corpora", "polarity-nl-bol.com.csv"), encoding='utf8'): reviews.append((review, int(score) > 0)) A, P, R, F = test(lambda review: nl.positive(review), reviews) #print(A, P, R, F) Index: pattern-5b85d998c30ddc6772b56310713530224466083a/test/test_ru.py =================================================================== --- pattern-5b85d998c30ddc6772b56310713530224466083a.orig/test/test_ru.py +++ pattern-5b85d998c30ddc6772b56310713530224466083a/test/test_ru.py @@ -15,6 +15,9 @@ import unittest import random import subprocess +from random import seed +seed(0) + from pattern import text from pattern import ru @@ -33,7 +36,7 @@ class TestSpelling(unittest.TestCase): def test_spelling(self): i = j = 0.0 from pattern.db import Datasheet - for correct, wrong in Datasheet.load(os.path.join(PATH, "corpora", "spelling-ru.csv")): + for correct, wrong in Datasheet.load(os.path.join(PATH, "corpora", "spelling-ru.csv"), encoding='utf8'): for w in wrong.split(" "): suggested = ru.suggest(w) if suggested[0][0] == correct: Index: pattern-5b85d998c30ddc6772b56310713530224466083a/test/test_vector.py =================================================================== --- pattern-5b85d998c30ddc6772b56310713530224466083a.orig/test/test_vector.py +++ pattern-5b85d998c30ddc6772b56310713530224466083a/test/test_vector.py @@ -37,7 +37,7 @@ def model(top=None): Documents are mostly of a technical nature (developer forum posts). """ documents = [] - for score, message in Datasheet.load(os.path.join(PATH, "corpora", "spam-apache.csv")): + for score, message in Datasheet.load(os.path.join(PATH, "corpora", "spam-apache.csv"), encoding='utf8'): document = vector.Document(message, stemmer="porter", top=top, type=int(score) > 0) documents.append(document) return vector.Model(documents) Index: pattern-5b85d998c30ddc6772b56310713530224466083a/test/test_db.py =================================================================== --- pattern-5b85d998c30ddc6772b56310713530224466083a.orig/test/test_db.py +++ pattern-5b85d998c30ddc6772b56310713530224466083a/test/test_db.py @@ -18,6 +18,9 @@ import codecs import random import unittest +from random import seed +seed(0) + from pattern import db # To test MySQL, you need MySQLdb and a username + password with rights to create a database. Index: pattern-5b85d998c30ddc6772b56310713530224466083a/test/test_search.py =================================================================== --- pattern-5b85d998c30ddc6772b56310713530224466083a.orig/test/test_search.py +++ pattern-5b85d998c30ddc6772b56310713530224466083a/test/test_search.py @@ -14,6 +14,9 @@ import time import re import random +from random import seed +seed(0) + from pattern import search from pattern.en import Sentence, parse