# Test methods with long descriptive names can omit docstrings # pylint: disable=missing-docstring import copy import os import pickle import random import unittest import warnings from unittest.mock import Mock, MagicMock, patch from itertools import chain from math import isnan from threading import Thread from time import sleep, time import numpy as np import scipy.sparse as sp from Orange import data from Orange.data import (filter, Unknown, Table, DiscreteVariable, ContinuousVariable, Domain, StringVariable) from Orange.data.util import SharedComputeValue from Orange.tests import test_dirname from Orange.data.table import _optimize_indices, _select_from_selection, \ _FromTableConversion class TableTestCase(unittest.TestCase): def setUp(self): data.table.dataset_dirs.append(test_dirname()) def tearDown(self): data.table.dataset_dirs.remove(test_dirname()) def test_indexing_class(self): d = data.Table("datasets/test1") self.assertEqual([e.get_class() for e in d], ["t", "t", "f"]) cind = len(d.domain.variables) - 1 self.assertEqual([e[cind] for e in d], ["t", "t", "f"]) self.assertEqual([e["d"] for e in d], ["t", "t", "f"]) cvar = d.domain.class_var self.assertEqual([e[cvar] for e in d], ["t", "t", "f"]) def test_filename(self): dir = data.table.get_sample_datasets_dir() d = data.Table("iris") self.assertEqual(d.__file__, os.path.join(dir, "iris.tab")) d = data.Table("datasets/test2.tab") self.assertTrue(d.__file__.endswith("test2.tab")) # platform dependent def test_indexing(self): with warnings.catch_warnings(): warnings.simplefilter("ignore") d = data.Table("datasets/test2") # regular, discrete varc = d.domain["c"] self.assertEqual(d[0, 1], "0") self.assertEqual(d[0, varc], "0") self.assertEqual(d[0, "c"], "0") self.assertEqual(d[0][1], "0") self.assertEqual(d[0][varc], "0") self.assertEqual(d[0]["c"], "0") self.assertEqual(d[np.int_(0), np.int_(1)], "0") self.assertEqual(d[np.int_(0)][np.int_(1)], "0") # regular, continuous varb = d.domain["b"] self.assertEqual(d[0, 0], 0) self.assertEqual(d[0, varb], 0) self.assertEqual(d[0, "b"], 0) self.assertEqual(d[0][0], 0) self.assertEqual(d[0][varb], 0) self.assertEqual(d[0]["b"], 0) self.assertEqual(d[np.int_(0), np.int_(0)], 0) self.assertEqual(d[np.int_(0)][np.int_(0)], 0) # negative varb = d.domain["b"] self.assertEqual(d[-2, 0], 3.333) self.assertEqual(d[-2, varb], 3.333) self.assertEqual(d[-2, "b"], 3.333) self.assertEqual(d[-2][0], 3.333) self.assertEqual(d[-2][varb], 3.333) self.assertEqual(d[-2]["b"], 3.333) self.assertEqual(d[np.int_(-2), np.int_(0)], 3.333) self.assertEqual(d[np.int_(-2)][np.int_(0)], 3.333) # meta, discrete vara = d.domain["a"] metaa = d.domain.index("a") self.assertEqual(d[0, metaa], "A") self.assertEqual(d[0, vara], "A") self.assertEqual(d[0, "a"], "A") self.assertEqual(d[0][metaa], "A") self.assertEqual(d[0][vara], "A") self.assertEqual(d[0]["a"], "A") self.assertEqual(d[np.int_(0), np.int_(metaa)], "A") self.assertEqual(d[np.int_(0)][np.int_(metaa)], "A") # meta, string vare = d.domain["e"] metae = d.domain.index("e") self.assertEqual(d[0, metae], "i") self.assertEqual(d[0, vare], "i") self.assertEqual(d[0, "e"], "i") self.assertEqual(d[0][metae], "i") self.assertEqual(d[0][vare], "i") self.assertEqual(d[0]["e"], "i") self.assertEqual(d[np.int_(0), np.int_(metae)], "i") self.assertEqual(d[np.int_(0)][np.int_(metae)], "i") def test_indexing_example(self): with warnings.catch_warnings(): warnings.simplefilter("ignore") d = data.Table("datasets/test2") e = d[0] # regular, discrete varc = d.domain["c"] self.assertEqual(e[1], "0") self.assertEqual(e[varc], "0") self.assertEqual(e["c"], "0") self.assertEqual(e[np.int_(1)], "0") # regular, continuous varb = d.domain["b"] self.assertEqual(e[0], 0) self.assertEqual(e[varb], 0) self.assertEqual(e["b"], 0) self.assertEqual(e[np.int_(0)], 0) # meta, discrete vara = d.domain["a"] metaa = d.domain.index("a") self.assertEqual(e[metaa], "A") self.assertEqual(e[vara], "A") self.assertEqual(e["a"], "A") self.assertEqual(e[np.int_(metaa)], "A") # meta, string vare = d.domain["e"] metae = d.domain.index("e") self.assertEqual(e[metae], "i") self.assertEqual(e[vare], "i") self.assertEqual(e["e"], "i") self.assertEqual(e[np.int_(metae)], "i") def test_indexing_assign_value(self): with warnings.catch_warnings(): warnings.simplefilter("ignore") d = data.Table("datasets/test2") # meta vara = d.domain["a"] metaa = d.domain.index("a") self.assertEqual(d[0, "a"], "A") with d.unlocked(): d[0, "a"] = "B" self.assertEqual(d[0, "a"], "B") with d.unlocked(): d[0]["a"] = "A" self.assertEqual(d[0, "a"], "A") with d.unlocked(): d[0, vara] = "B" self.assertEqual(d[0, "a"], "B") with d.unlocked(): d[0][vara] = "A" self.assertEqual(d[0, "a"], "A") with d.unlocked(): d[0, metaa] = "B" self.assertEqual(d[0, "a"], "B") with d.unlocked(): d[0][metaa] = "A" self.assertEqual(d[0, "a"], "A") with d.unlocked(): d[0, np.int_(metaa)] = "B" self.assertEqual(d[0, "a"], "B") with d.unlocked(): d[0][np.int_(metaa)] = "A" self.assertEqual(d[0, "a"], "A") # regular varb = d.domain["b"] self.assertEqual(d[0, "b"], 0) with d.unlocked(): d[0, "b"] = 42 self.assertEqual(d[0, "b"], 42) with d.unlocked(): d[0]["b"] = 0 self.assertEqual(d[0, "b"], 0) with d.unlocked(): d[0, varb] = 42 self.assertEqual(d[0, "b"], 42) with d.unlocked(): d[0][varb] = 0 self.assertEqual(d[0, "b"], 0) with d.unlocked(): d[0, 0] = 42 self.assertEqual(d[0, "b"], 42) with d.unlocked(): d[0][0] = 0 self.assertEqual(d[0, "b"], 0) with d.unlocked(): d[0, np.int_(0)] = 42 self.assertEqual(d[0, "b"], 42) with d.unlocked(): d[0][np.int_(0)] = 0 self.assertEqual(d[0, "b"], 0) def test_indexing_assign_example(self): def almost_equal_list(s, t): for e, f in zip(s, t): self.assertAlmostEqual(e, f) with warnings.catch_warnings(): warnings.simplefilter("ignore") d = data.Table("datasets/test2") self.assertFalse(isnan(d[0, "a"])) with d.unlocked(): d[0] = ["3.14", "1", "f"] almost_equal_list(d[0].values(), [3.14, "1", "f"]) self.assertTrue(isnan(d[0, "a"])) with d.unlocked(): d[0] = [3.15, 1, "t"] almost_equal_list(d[0].values(), [3.15, "0", "t"]) with d.unlocked(): d[np.int_(0)] = [3.15, 2, "f"] almost_equal_list(d[0].values(), [3.15, 2, "f"]) with d.unlocked(), self.assertRaises(ValueError): d[0] = ["3.14", "1"] with d.unlocked(), self.assertRaises(ValueError): d[np.int_(0)] = ["3.14", "1"] ex = data.Instance(d.domain, ["3.16", "1", "f"]) with d.unlocked(): d[0] = ex almost_equal_list(d[0].values(), [3.16, "1", "f"]) ex = data.Instance(d.domain, ["3.16", 2, "t"]) with d.unlocked(): d[np.int_(0)] = ex almost_equal_list(d[0].values(), [3.16, 2, "t"]) ex = data.Instance(d.domain, ["3.16", "1", "f"]) ex["e"] = "mmmapp" with d.unlocked(): d[0] = ex almost_equal_list(d[0].values(), [3.16, "1", "f"]) self.assertEqual(d[0, "e"], "mmmapp") def test_slice(self): with warnings.catch_warnings(): warnings.simplefilter("ignore") d = data.Table("datasets/test2") x = d[:3] self.assertEqual(len(x), 3) self.assertEqual([e[0] for e in x], [0, 1.1, 2.22]) x = d[2:5] self.assertEqual(len(x), 3) self.assertEqual([e[0] for e in x], [2.22, 2.23, 2.24]) x = d[4:1:-1] self.assertEqual(len(x), 3) self.assertEqual([e[0] for e in x], [2.24, 2.23, 2.22]) x = d[-3:] self.assertEqual(len(x), 3) self.assertEqual([e[0] for e in x], [2.26, 3.333, Unknown]) def test_assign_slice_value(self): with warnings.catch_warnings(): warnings.simplefilter("ignore") d = data.Table("datasets/test2") with d.unlocked(): d[2:5, 0] = 42 self.assertEqual([e[0] for e in d], [0, 1.1, 42, 42, 42, 2.25, 2.26, 3.333, Unknown]) with d.unlocked(): d[:3, "b"] = 43 self.assertEqual([e[0] for e in d], [43, 43, 43, 42, 42, 2.25, 2.26, 3.333, None]) with d.unlocked(): d[-2:, d.domain[0]] = 44 self.assertEqual([e[0] for e in d], [43, 43, 43, 42, 42, 2.25, 2.26, 44, 44]) with d.unlocked(): d[2:5, "a"] = "A" self.assertEqual([e["a"] for e in d], list("ABAAACCDE")) def test_multiple_indices(self): with warnings.catch_warnings(): warnings.simplefilter("ignore") d = data.Table("datasets/test2") with self.assertRaises(IndexError): x = d[2, 5, 1] with self.assertRaises(IndexError): x = d[(2, 5, 1)] x = d[[2, 5, 1]] self.assertEqual([e[0] for e in x], [2.22, 2.25, 1.1]) def test_assign_multiple_indices_value(self): with warnings.catch_warnings(): warnings.simplefilter("ignore") d = data.Table("datasets/test2") with d.unlocked(): d[1:4, "b"] = 42 self.assertEqual([e[0] for e in d], [0, 42, 42, 42, 2.24, 2.25, 2.26, 3.333, None]) with d.unlocked(): d[range(5, 2, -1), "b"] = None self.assertEqual([e[d.domain[0]] for e in d], [0, 42, 42, None, "?", "", 2.26, 3.333, None]) def test_set_multiple_indices_example(self): with warnings.catch_warnings(): warnings.simplefilter("ignore") d = data.Table("datasets/test2") vals = [e[0] for e in d] with d.unlocked(): d[[1, 2, 5]] = [42, None, None] vals[1] = vals[2] = vals[5] = 42 self.assertEqual([e[0] for e in d], vals) def test_bool(self): d = data.Table("iris") self.assertTrue(d) d = data.Table("datasets/test3") self.assertFalse(d) d = data.Table("iris") self.assertTrue(d) def test_checksum(self): d = data.Table("zoo") with d.unlocked(): d[42, 3] = 0 crc1 = d.checksum(False) with d.unlocked(): d[42, 3] = 1 crc2 = d.checksum(False) self.assertNotEqual(crc1, crc2) with d.unlocked(): d[42, 3] = 0 crc3 = d.checksum(False) self.assertEqual(crc1, crc3) _ = d[42, "name"] with d.unlocked(): d[42, "name"] = "non-animal" crc4 = d.checksum(False) self.assertEqual(crc1, crc4) crc4 = d.checksum(True) crc5 = d.checksum(1) crc6 = d.checksum(False) self.assertNotEqual(crc1, crc4) self.assertNotEqual(crc1, crc5) self.assertEqual(crc1, crc6) def test_total_weight(self): d = data.Table("zoo") self.assertEqual(d.total_weight(), len(d)) with d.unlocked(): d.set_weights(0) d[0].weight = 0.1 d[10].weight = 0.2 d[-1].weight = 0.3 self.assertAlmostEqual(d.total_weight(), 0.6) def test_has_missing(self): d = data.Table("zoo") self.assertFalse(d.has_missing()) self.assertFalse(d.has_missing_class()) with d.unlocked(): d[10, 3] = "?" self.assertTrue(d.has_missing()) self.assertFalse(d.has_missing_class()) with d.unlocked(): d[10].set_class("?") self.assertTrue(d.has_missing()) self.assertTrue(d.has_missing_class()) with d.unlocked(): d = data.Table("datasets/test3") self.assertFalse(d.has_missing()) self.assertFalse(d.has_missing_class()) def test_shuffle(self): d = data.Table("zoo") crc = d.checksum() names = set(str(x["name"]) for x in d) ids = d.ids with d.unlocked_reference(): d.shuffle() self.assertNotEqual(crc, d.checksum()) self.assertSetEqual(names, set(str(x["name"]) for x in d)) self.assertTrue(np.any(ids - d.ids != 0)) crc2 = d.checksum() x = d[2:10] crcx = x.checksum() with d.unlocked_reference(): d.shuffle() self.assertNotEqual(crc2, d.checksum()) self.assertEqual(crcx, x.checksum()) crc2 = d.checksum() with x.unlocked_reference(): x.shuffle() self.assertNotEqual(crcx, x.checksum()) self.assertEqual(crc2, d.checksum()) self.assertLess(set(x.ids), set(ids)) @staticmethod def not_less_ex(ex1, ex2): for v1, v2 in zip(ex1, ex2): if v1 != v2: return v1 < v2 return True @staticmethod def sorted(d): for i in range(1, len(d)): if not TableTestCase.not_less_ex(d[i - 1], d[i]): return False return True @staticmethod def not_less_ex_ord(ex1, ex2, ord): for a in ord: if ex1[a] != ex2[a]: return ex1[a] < ex2[a] return True @staticmethod def sorted_ord(d, ord): for i in range(1, len(d)): if not TableTestCase.not_less_ex_ord(d[i - 1], d[i], ord): return False return True def test_copy(self): t = data.Table.from_numpy( None, np.zeros((5, 3)), np.arange(5), np.zeros((5, 3))) copy = t.copy() self.assertTrue(np.all(t.X == copy.X)) self.assertTrue(np.all(t.Y == copy.Y)) self.assertTrue(np.all(t.metas == copy.metas)) with copy.unlocked(): copy[0] = [1, 1, 1, 1] self.assertFalse(np.all(t.X == copy.X)) self.assertFalse(np.all(t.Y == copy.Y)) self.assertFalse(np.all(t.metas == copy.metas)) def test_copy_sparse(self): t = data.Table('iris').to_sparse() copy = t.copy() self.assertEqual((t.X != copy.X).nnz, 0) # sparse matrices match by content np.testing.assert_equal(t.Y, copy.Y) np.testing.assert_equal(t.metas, copy.metas) self.assertNotEqual(id(t.X), id(copy.X)) self.assertNotEqual(id(t._Y), id(copy._Y)) self.assertNotEqual(id(t.metas), id(copy.metas)) # ensure that copied sparse arrays do not share data # and that both are unlockable with t.unlocked(): t.X[0, 0] = 42 self.assertEqual(copy.X[0, 0], 5.1) with copy.unlocked(): copy.X[0, 0] = 43 self.assertEqual(t.X[0, 0], 42) def test_concatenate(self): d1 = data.Domain( [data.ContinuousVariable(n) for n in "abc"], data.DiscreteVariable("y", values="ABC"), [data.StringVariable(n) for n in ["m1", "m2"]], ) x1 = np.arange(6).reshape(2, 3) y1 = np.array([0, 1]) m1 = np.array([["foo", "bar"], ["baz", "qux"]]) w1 = np.random.random((2,)) t1 = data.Table.from_numpy(d1, x1, y1, m1, w1) t1.ids = ids1 = np.array([100, 101]) t1.attributes = {"a": 42, "c": 43} x2 = np.arange(6, 15).reshape(3, 3) y2 = np.array([1, 2, 0]) m2 = np.array([["a", "b"], ["c", "d"], ["e", "f"]]) w2 = np.random.random((3,)) t2 = data.Table.from_numpy(d1, x2, y2, m2, w2) t2.ids = ids2 = np.array([102, 103, 104]) t2.name = "t2" t2.attributes = {"a": 44, "b": 45} x3 = np.arange(15, 27).reshape(4, 3) y3 = np.array([2, 1, 1, 0]) m3 = np.array([["g", "h"], ["i", "j"], ["k", "l"], ["m", "n"]]) w3 = np.random.random((4,)) t3 = data.Table.from_numpy(d1, x3, y3, m3, w3) t3.ids = ids3 = np.array([102, 103, 104, 105]) t3.name = "t3" t1b = data.Table.concatenate((t1,)) self.assertEqual(t1b.domain, t1.domain) np.testing.assert_almost_equal(t1b.X, x1) np.testing.assert_almost_equal(t1b.Y, y1) self.assertEqual(list(t1b.metas.flatten()), list(m1.flatten())) np.testing.assert_almost_equal(t1b.W, w1) np.testing.assert_almost_equal(t1b.ids, ids1) self.assertEqual(t1b.name, t1.name) self.assertEqual(t1b.attributes, {"a": 42, "c": 43}) t12 = data.Table.concatenate((t1, t2)) self.assertEqual(t12.domain, t1.domain) np.testing.assert_almost_equal(t12.X, np.vstack((x1, x2))) np.testing.assert_almost_equal(t12.Y, np.hstack((y1, y2))) self.assertEqual(list(t12.metas.flatten()), list(np.vstack((m1, m2)).flatten())) np.testing.assert_almost_equal(t12.W, np.hstack((w1, w2))) np.testing.assert_almost_equal(t12.ids, np.hstack((ids1, ids2))) self.assertEqual(t12.name, "t2") self.assertEqual(t12.attributes, {"a": 42, "c": 43, "b": 45}) t123 = data.Table.concatenate((t1, t2, t3)) self.assertEqual(t123.domain, t1.domain) np.testing.assert_almost_equal(t123.X, np.vstack((x1, x2, x3))) np.testing.assert_almost_equal(t123.Y, np.hstack((y1, y2, y3))) self.assertEqual(list(t123.metas.flatten()), list(np.vstack((m1, m2, m3)).flatten())) np.testing.assert_almost_equal(t123.W, np.hstack((w1, w2, w3))) np.testing.assert_almost_equal(t123.ids, np.hstack((ids1, ids2, ids3))) self.assertEqual(t123.name, "t2") self.assertEqual(t123.attributes, {"a": 42, "c": 43, "b": 45}) with t2.unlocked(t2.Y): t2.Y = np.atleast_2d(t2.Y).T t12 = data.Table.concatenate((t1, t2)) self.assertEqual(t12.domain, t1.domain) np.testing.assert_almost_equal(t12.X, np.vstack((x1, x2))) np.testing.assert_almost_equal(t12.Y, np.hstack((y1, y2))) self.assertEqual(list(t12.metas.flatten()), list(np.vstack((m1, m2)).flatten())) np.testing.assert_almost_equal(t12.W, np.hstack((w1, w2))) np.testing.assert_almost_equal(t12.ids, np.hstack((ids1, ids2))) self.assertEqual(t12.name, "t2") self.assertEqual(t12.attributes, {"a": 42, "c": 43, "b": 45}) def test_concatenate_exceptions(self): zoo = data.Table("zoo") iris = data.Table("iris") self.assertRaises(ValueError, data.Table.concatenate, []) self.assertRaises(ValueError, data.Table.concatenate, [zoo, iris]) def test_concatenate_sparse(self): iris = Table("iris") with iris.unlocked(): iris.X = sp.csc_matrix(iris.X) new = Table.concatenate([iris, iris]) self.assertEqual(len(new), 300) self.assertTrue(sp.issparse(new.X), "Concatenated X is not sparse.") self.assertFalse(sp.issparse(new.Y), "Concatenated Y is not dense.") self.assertFalse(sp.issparse(new.metas), "Concatenated metas is not dense.") self.assertEqual(len(new.ids), 300) def test_pickle(self): d = data.Table("zoo") s = pickle.dumps(d) d2 = pickle.loads(s) self.assertEqual(d[0], d2[0]) self.assertEqual(d.checksum(include_metas=False), d2.checksum(include_metas=False)) d = data.Table("iris") s = pickle.dumps(d) d2 = pickle.loads(s) self.assertEqual(d[0], d2[0]) self.assertEqual(d.checksum(include_metas=False), d2.checksum(include_metas=False)) def test_pickle_setstate(self): d = data.Table("zoo") s = pickle.dumps(d) with patch("Orange.data.Table.__setstate__", Mock()) as mock: pickle.loads(s) state = mock.call_args[0][0] for k in ["X", "_Y", "metas"]: self.assertIn(k, state) self.assertEqual(state[k].ndim, 2) self.assertIn("W", state) for k in ["_X", "Y", "_metas", "_W"]: self.assertNotIn(k, state) def test_translate_through_slice(self): d = data.Table("iris") dom = data.Domain(["petal length", "sepal length", "iris"], source=d.domain) d_ref = d[:10, dom.variables] self.assertEqual(d_ref.domain.class_var, d.domain.class_var) self.assertEqual(d_ref[0, "petal length"], d[0, "petal length"]) self.assertEqual(d_ref[0, "sepal length"], d[0, "sepal length"]) self.assertEqual(d_ref.X.shape, (10, 2)) self.assertEqual(d_ref.Y.shape, (10,)) def test_saveTab(self): d = data.Table("iris")[:3] d.save("test-save.tab") try: d2 = data.Table("test-save.tab") for e1, e2 in zip(d, d2): self.assertEqual(e1, e2) finally: os.remove("test-save.tab") os.remove("test-save.tab.metadata") dom = data.Domain([data.ContinuousVariable("a")]) d = data.Table.from_list(dom, [[i] for i in range(3)]) d.save("test-save.tab") try: d2 = data.Table("test-save.tab") self.assertEqual(len(d.domain.attributes), 1) self.assertEqual(d.domain.class_var, None) for i in range(3): self.assertEqual(d2[i], [i]) finally: os.remove("test-save.tab") dom = data.Domain([data.ContinuousVariable("a")], None) d = data.Table.from_list(dom, [[i] for i in range(3)]) d.save("test-save.tab") try: d2 = data.Table("test-save.tab") self.assertEqual(len(d.domain.attributes), 1) for i in range(3): self.assertEqual(d2[i], [i]) finally: os.remove("test-save.tab") d = data.Table("zoo") d.save("test-zoo.tab") dd = data.Table("test-zoo") try: self.assertTupleEqual(d.domain.metas, dd.domain.metas, msg="Meta attributes don't match.") self.assertTupleEqual(d.domain.variables, dd.domain.variables, msg="Attributes don't match.") np.testing.assert_almost_equal(d.W, dd.W, err_msg="Weights don't match.") for i in range(10): for j in d.domain.variables: self.assertEqual(d[i][j], dd[i][j]) finally: os.remove("test-zoo.tab") os.remove("test-zoo.tab.metadata") d = data.Table("zoo") with d.unlocked(): d.set_weights(range(len(d))) d.save("test-zoo-weights.tab") dd = data.Table("test-zoo-weights") try: self.assertTupleEqual(d.domain.metas, dd.domain.metas, msg="Meta attributes don't match.") self.assertTupleEqual(d.domain.variables, dd.domain.variables, msg="Attributes don't match.") np.testing.assert_almost_equal(d.W, dd.W, err_msg="Weights don't match.") for i in range(10): for j in d.domain.variables: self.assertEqual(d[i][j], dd[i][j]) finally: os.remove("test-zoo-weights.tab") os.remove("test-zoo-weights.tab.metadata") def test_save_pickle(self): table = data.Table("iris") try: table.save("iris.pickle") table2 = data.Table.from_file("iris.pickle") np.testing.assert_almost_equal(table.X, table2.X) np.testing.assert_almost_equal(table.Y, table2.Y) self.assertEqual(table.domain[0], table2.domain[0]) finally: os.remove("iris.pickle") def test_read_pickle_ids(self): table = data.Table("iris") try: table.save("iris.pickle") table1 = data.Table.from_file("iris.pickle") table2 = data.Table.from_file("iris.pickle") self.assertEqual(len(set(table1.ids) | set(table2.ids)), 300) finally: os.remove("iris.pickle") def test_from_numpy(self): a = np.arange(20, dtype="d").reshape((4, 5)).copy() m = np.arange(4, dtype="d").reshape((4, 1)).copy() a[:, -1] = [0, 0, 0, 1] dom = data.Domain([data.ContinuousVariable(x) for x in "abcd"], data.DiscreteVariable("e", values=("no", "yes")), metas=[data.ContinuousVariable(x) for x in "f"]) table = data.Table(dom, a, metas=m) with table.unlocked(): for i in range(4): self.assertEqual(table[i].get_class(), "no" if i < 3 else "yes") for j in range(5): self.assertEqual(a[i, j], table[i, j]) with table.unlocked(), self.assertRaises(IndexError): table[0, -6] = 5 def test_filter_is_defined(self): d = data.Table("iris") with d.unlocked(): d[1, 4] = Unknown self.assertTrue(isnan(d[1, 4])) with d.unlocked(): d[140, 0] = Unknown e = filter.IsDefined()(d) self.assertEqual(len(e), len(d) - 2) self.assertEqual(e[0], d[0]) self.assertEqual(e[1], d[2]) self.assertEqual(e[147], d[149]) self.assertTrue(d.has_missing()) self.assertFalse(e.has_missing()) def test_filter_has_class(self): d = data.Table("iris") with d.unlocked(): d[1, 4] = Unknown self.assertTrue(isnan(d[1, 4])) with d.unlocked(): d[140, 0] = Unknown e = filter.HasClass()(d) self.assertEqual(len(e), len(d) - 1) self.assertEqual(e[0], d[0]) self.assertEqual(e[1], d[2]) self.assertEqual(e[148], d[149]) self.assertTrue(d.has_missing()) self.assertTrue(e.has_missing()) self.assertFalse(e.has_missing_class()) def test_filter_random(self): d = data.Table("iris") e = filter.Random(50)(d) self.assertEqual(len(e), 50) e = filter.Random(50, negate=True)(d) self.assertEqual(len(e), 100) for _ in range(5): e = filter.Random(0.2)(d) self.assertEqual(len(e), 30) bc = np.bincount(np.array(e.Y[:], dtype=int)) if min(bc) > 7: break else: self.fail("Filter returns too uneven distributions") def test_filter_values_nested(self): d = data.Table("iris") f1 = filter.FilterContinuous(d.columns.sepal_length, filter.FilterContinuous.Between, min=4.5, max=5.0) f2 = filter.FilterContinuous(d.columns.sepal_width, filter.FilterContinuous.Between, min=3.1, max=3.4) f3 = filter.FilterDiscrete(d.columns.iris, [0, 1]) f = filter.Values([filter.Values([f1, f2], conjunction=False), f3]) self.assertEqual(41, len(f(d))) def test_filter_string_works_for_numeric_columns(self): var = StringVariable("s") data = Table.from_list(Domain([], metas=[var]), [[x] for x in range(21)]) # 1, 2, 3, ..., 18, 19, 20 fs = filter.FilterString filters = [ ((fs.Greater, "5"), dict(rows=4)), # 6, 7, 8, 9 ((fs.Between, "15", "2"), dict(rows=6)), # 15, 16, 17, 18, 19, 2 ((fs.Contains, "2"), dict(rows=3)), # 2, 12, 20 ] for args, expected in filters: f = fs(var, *args) filtered_data = filter.Values([f])(data) self.assertEqual(len(filtered_data), expected["rows"], "{} returned wrong number of rows".format(args)) def test_filter_value_continuous(self): d = data.Table("iris") col = d.X[:, 2] v = d.columns f = filter.FilterContinuous(v.petal_length, filter.FilterContinuous.Between, min=4.5, max=5.1) x = filter.Values([f])(d) self.assertTrue(np.all(x.X[:, 2] >= 4.5)) self.assertTrue(np.all(x.X[:, 2] <= 5.1)) self.assertEqual(sum((col >= 4.5) * (col <= 5.1)), len(x)) f.ref = 5.1 f.oper = filter.FilterContinuous.Equal x = filter.Values([f])(d) self.assertTrue(np.all(x.X[:, 2] == 5.1)) self.assertEqual(sum(col == 5.1), len(x)) f.oper = filter.FilterContinuous.NotEqual x = filter.Values([f])(d) self.assertTrue(np.all(x.X[:, 2] != 5.1)) self.assertEqual(sum(col != 5.1), len(x)) f.oper = filter.FilterContinuous.Less x = filter.Values([f])(d) self.assertTrue(np.all(x.X[:, 2] < 5.1)) self.assertEqual(sum(col < 5.1), len(x)) f.oper = filter.FilterContinuous.LessEqual x = filter.Values([f])(d) self.assertTrue(np.all(x.X[:, 2] <= 5.1)) self.assertEqual(sum(col <= 5.1), len(x)) f.oper = filter.FilterContinuous.Greater x = filter.Values([f])(d) self.assertTrue(np.all(x.X[:, 2] > 5.1)) self.assertEqual(sum(col > 5.1), len(x)) f.oper = filter.FilterContinuous.GreaterEqual x = filter.Values([f])(d) self.assertTrue(np.all(x.X[:, 2] >= 5.1)) self.assertEqual(sum(col >= 5.1), len(x)) f.oper = filter.FilterContinuous.Outside f.ref, f.max = 4.5, 5.1 x = filter.Values([f])(d) for e in x: self.assertTrue(e[2] < 4.5 or e[2] > 5.1) self.assertEqual(sum((col < 4.5) + (col > 5.1)), len(x)) f.oper = filter.FilterContinuous.IsDefined f.ref = f.max = None x = filter.Values([f])(d) self.assertEqual(len(x), len(d)) with d.unlocked(): d[:30, v.petal_length] = Unknown x = filter.Values([f])(d) self.assertEqual(len(x), len(d) - 30) def test_filter_value_continuous_args(self): d = data.Table("iris") col = d.X[:, 2] v = d.columns f = filter.FilterContinuous(v.petal_length, filter.FilterContinuous.Equal, ref=5.1) x = filter.Values([f])(d) self.assertTrue(np.all(x.X[:, 2] == 5.1)) self.assertEqual(sum(col == 5.1), len(x)) f = filter.FilterContinuous(2, filter.FilterContinuous.Equal, ref=5.1) self.assertTrue(np.all(x.X[:, 2] == 5.1)) self.assertEqual(sum(col == 5.1), len(x)) f = filter.FilterContinuous("petal length", filter.FilterContinuous.Equal, ref=5.1) self.assertTrue(np.all(x.X[:, 2] == 5.1)) self.assertEqual(sum(col == 5.1), len(x)) f = filter.FilterContinuous("sepal length", filter.FilterContinuous.Equal, ref=5.1) f.column = 2 self.assertTrue(np.all(x.X[:, 2] == 5.1)) self.assertEqual(sum(col == 5.1), len(x)) f = filter.FilterContinuous("sepal length", filter.FilterContinuous.Equal, ref=5.1) f.column = v.petal_length self.assertTrue(np.all(x.X[:, 2] == 5.1)) self.assertEqual(sum(col == 5.1), len(x)) f = filter.FilterContinuous(v.petal_length, filter.FilterContinuous.Equal, ref=18) f.ref = 5.1 x = filter.Values([f])(d) self.assertTrue(np.all(x.X[:, 2] == 5.1)) self.assertEqual(sum(col == 5.1), len(x)) f = filter.FilterContinuous(v.petal_length, filter.FilterContinuous.Equal, ref=18) f.ref = 5.1 x = filter.Values([f])(d) self.assertTrue(np.all(x.X[:, 2] == 5.1)) self.assertEqual(sum(col == 5.1), len(x)) def test_valueFilter_discrete(self): d = data.Table("zoo") f = filter.FilterDiscrete(d.domain.class_var, values=[2, 3, 4]) for e in filter.Values([f])(d): self.assertTrue(e.get_class() in [2, 3, 4]) f.values = ["mammal"] for e in filter.Values([f])(d): self.assertEqual(e.get_class(), "mammal") f = filter.FilterDiscrete(d.domain.class_var, values=[2, "mammal"]) for e in filter.Values([f])(d): self.assertTrue(e.get_class() in [2, "mammal"]) f = filter.FilterDiscrete(d.domain.class_var, values=[2, "martian"]) self.assertRaises(ValueError, d._filter_values, f) f = filter.FilterDiscrete(d.domain.class_var, values=(2, data.Table)) self.assertRaises(TypeError, d._filter_values, f) v = d.columns f = filter.FilterDiscrete(v.hair, values=None) self.assertEqual(len(filter.Values([f])(d)), len(d)) with d.unlocked(): d[:5, v.hair] = Unknown self.assertEqual(len(filter.Values([f])(d)), len(d) - 5) def test_valueFilter_string_is_defined(self): d = data.Table("datasets/test9.tab") f = filter.FilterString(-5, filter.FilterString.IsDefined) x = filter.Values([f])(d) self.assertEqual(len(x), 7) def test_valueFilter_discrete_meta_is_defined(self): d = data.Table("datasets/test9.tab") f = filter.FilterDiscrete(-4, None) x = filter.Values([f])(d) self.assertEqual(len(x), 8) def test_valueFilter_string_case_sens(self): d = data.Table("zoo") col = d[:, "name"].metas[:, 0] f = filter.FilterString("name", filter.FilterString.Equal, "girl") x = filter.Values([f])(d) self.assertEqual(len(x), 1) self.assertEqual(x[0, "name"], "girl") self.assertTrue(np.all(x.metas == "girl")) f.oper = f.NotEqual x = filter.Values([f])(d) self.assertEqual(len(x), len(d) - 1) self.assertTrue(np.all(x[:, "name"] != "girl")) f.oper = f.Less x = filter.Values([f])(d) self.assertEqual(len(x), sum(col < "girl")) self.assertTrue(np.all(x.metas < "girl")) f.oper = f.LessEqual x = filter.Values([f])(d) self.assertEqual(len(x), sum(col <= "girl")) self.assertTrue(np.all(x.metas <= "girl")) f.oper = f.Greater x = filter.Values([f])(d) self.assertEqual(len(x), sum(col > "girl")) self.assertTrue(np.all(x.metas > "girl")) f.oper = f.GreaterEqual x = filter.Values([f])(d) self.assertEqual(len(x), sum(col >= "girl")) self.assertTrue(np.all(x.metas >= "girl")) f.oper = f.Between f.max = "lion" x = filter.Values([f])(d) self.assertEqual(len(x), sum((col >= "girl") * (col <= "lion"))) self.assertTrue(np.all(x.metas >= "girl")) self.assertTrue(np.all(x.metas <= "lion")) f.oper = f.Outside x = filter.Values([f])(d) self.assertEqual(len(x), sum(col < "girl") + sum(col > "lion")) self.assertTrue(np.all((x.metas < "girl") + (x.metas > "lion"))) f.oper = f.Contains f.ref = "ea" x = filter.Values([f])(d) for e in x: self.assertTrue("ea" in e["name"]) self.assertEqual(len(x), len([e for e in col if "ea" in e])) f.oper = f.StartsWith f.ref = "sea" x = filter.Values([f])(d) for e in x: self.assertTrue(str(e["name"]).startswith("sea")) self.assertEqual(len(x), len([e for e in col if e.startswith("sea")])) f.oper = f.EndsWith f.ref = "ion" x = filter.Values([f])(d) for e in x: self.assertTrue(str(e["name"]).endswith("ion")) self.assertEqual(len(x), len([e for e in col if e.endswith("ion")])) def test_valueFilter_string_case_insens(self): d = data.Table("zoo") with d.unlocked(): d[d[:, "name"].metas[:, 0] == "girl", "name"] = "GIrl" col = d[:, "name"].metas[:, 0] f = filter.FilterString("name", filter.FilterString.Equal, "giRL") f.case_sensitive = False x = filter.Values([f])(d) self.assertEqual(len(x), 1) self.assertEqual(x[0, "name"], "GIrl") self.assertTrue(np.all(x.metas == "GIrl")) f.oper = f.NotEqual x = filter.Values([f])(d) self.assertEqual(len(x), len(d) - 1) self.assertTrue(np.all(x[:, "name"] != "GIrl")) f.oper = f.Less f.ref = "CHiCKEN" x = filter.Values([f])(d) self.assertEqual(len(x), sum(col < "chicken") - 1) # girl! self.assertTrue(np.all(x.metas < "chicken")) f.oper = f.LessEqual x = filter.Values([f])(d) self.assertEqual(len(x), sum(col <= "chicken") - 1) self.assertTrue(np.all(x.metas <= "chicken")) f.oper = f.Greater x = filter.Values([f])(d) self.assertEqual(len(x), sum(col > "chicken") + 1) for e in x: self.assertGreater(str(e["name"]).lower(), "chicken") f.oper = f.GreaterEqual x = filter.Values([f])(d) self.assertEqual(len(x), sum(col >= "chicken") + 1) for e in x: self.assertGreaterEqual(str(e["name"]).lower(), "chicken") f.oper = f.Between f.max = "liOn" x = filter.Values([f])(d) self.assertEqual(len(x), sum((col >= "chicken") * (col <= "lion")) + 1) for e in x: self.assertTrue("chicken" <= str(e["name"]).lower() <= "lion") f.oper = f.Outside x = filter.Values([f])(d) self.assertEqual(len(x), sum(col < "chicken") + sum(col > "lion") - 1) self.assertTrue(np.all((x.metas < "chicken") + (x.metas > "lion"))) f.oper = f.Contains f.ref = "iR" x = filter.Values([f])(d) for e in x: self.assertTrue("ir" in str(e["name"]).lower()) self.assertEqual(len(x), len([e for e in col if "ir" in e]) + 1) f.oper = f.StartsWith f.ref = "GI" x = filter.Values([f])(d) for e in x: self.assertTrue(str(e["name"]).lower().startswith("gi")) self.assertEqual(len(x), len([e for e in col if e.lower().startswith("gi")])) f.oper = f.EndsWith f.ref = "ion" x = filter.Values([f])(d) for e in x: self.assertTrue(str(e["name"]).endswith("ion")) self.assertTrue(str(e["name"]).endswith("ion")) self.assertEqual(len(x), len([e for e in col if e.endswith("ion")])) def test_valueFilter_regex(self): d = data.Table("zoo") f = filter.FilterRegex(d.domain['name'], '^c...$') x = filter.Values([f])(d) self.assertEqual(len(x), 7) def test_valueFilter_stringList(self): data = Table("zoo") var = data.domain["name"] fs = filter.FilterStringList filters = [ ((["swan", "tuna", "wasp"], True), dict(rows=3)), ((["swan", "tuna", "wasp"], False), dict(rows=3)), ((["WoRm", "TOad", "vOLe"], True), dict(rows=0)), ((["WoRm", "TOad", "vOLe"], False), dict(rows=3)), ] for args, expected in filters: f = fs(var, *args) filtered_data = filter.Values([f])(data) self.assertEqual(len(filtered_data), expected["rows"], "{} returned wrong number of rows".format(args)) def test_table_dtypes(self): table = data.Table("iris") metas = np.hstack((table.metas, table.Y.reshape(len(table), 1))) attributes_metas = table.domain.metas + table.domain.class_vars domain_metas = data.Domain(table.domain.attributes[:2], table.domain.attributes[2:], attributes_metas) table_metas = data.Table(domain_metas, table.X[:, : 2], table.X[:, 2:], metas) new_table = data.Table.from_table(data.Domain(table_metas.domain.metas), table_metas) self.assertEqual(new_table.X.dtype, np.float64) new_table = data.Table.from_table(data.Domain((), table_metas.domain.metas), table_metas) self.assertEqual(new_table.Y.dtype, np.float64) new_table = data.Table.from_table(data.Domain((), (), table_metas.domain.metas), table_metas) self.assertEqual(new_table.metas.dtype, np.float64) def test_attributes(self): table = data.Table("iris") table.attributes = {1: "test"} table2 = table[:4] self.assertEqual(table2.attributes[1], "test") table2.attributes[1] = "modified" self.assertEqual(table.attributes[1], "test") self.assertEqual(table2.attributes[1], "modified") # TODO Test conjunctions and disjunctions of conditions def test_is_sparse(self): table = data.Table("iris") self.assertFalse(table.is_sparse()) with table.unlocked(): table.X = sp.csr_matrix(table.X) self.assertTrue(table.is_sparse()) def test_repr_sparse_with_one_row(self): table = data.Table("iris")[:1] with table.unlocked_reference(): table.X = sp.csr_matrix(table.X) r = repr(table) # make sure repr does not crash self.assertEqual(r.replace("\n", ""), "[[sepal length=5.1, sepal width=3.5, " "petal length=1.4, petal width=0.2 | Iris-setosa]]") def test_inf(self): a = np.array([[2, 0, 0, 0], [0, np.nan, np.nan, 1], [0, 0, np.inf, 0]]) with self.assertWarns(Warning): tab = data.Table.from_numpy(None, a) self.assertEqual(tab.get_nan_frequency_attribute(), 3/12) def test_str(self): iris = Table("iris") # instance self.assertEqual("[5.1, 3.5, 1.4, 0.2 | Iris-setosa]", str(iris[0])) # table table_str = str(iris) lines = table_str.split('\n') self.assertEqual(150, len(lines)) self.assertEqual("[[5.1, 3.5, 1.4, 0.2 | Iris-setosa],", lines[0]) self.assertEqual(" [5.9, 3.0, 5.1, 1.8 | Iris-virginica]]", lines[-1]) def test_str_sparse(self): iris = Table("iris") with iris.unlocked_reference(): iris.X = sp.csr_matrix(iris.X) # instance s0 = "[sepal length=5.1, sepal width=3.5, " \ "petal length=1.4, petal width=0.2 | Iris-setosa]" self.assertEqual(s0, str(iris[0])) # table table_str = str(iris) lines = table_str.split('\n') self.assertEqual(150, len(lines)) self.assertEqual("[" + s0 + ",", lines[0]) slast = "[sepal length=5.9, sepal width=3.0, " \ "petal length=5.1, petal width=1.8 | Iris-virginica]" self.assertEqual(" " + slast + "]", lines[-1]) def column_sizes(table): return (len(table.domain.attributes), len(table.domain.class_vars), len(table.domain.metas)) class TableTests(unittest.TestCase): attributes = ["Feature %i" % i for i in range(10)] class_vars = ["Class %i" % i for i in range(1)] metas = ["Meta %i" % i for i in range(5)] nrows = 10 row_indices = (1, 5, 7, 9) @classmethod def setUpClass(cls): cls.saved_max_rows_at_once = _FromTableConversion.max_rows_at_once _FromTableConversion.max_rows_at_once = cls.nrows - 1 @classmethod def tearDownClass(cls): _FromTableConversion.max_rows_at_once = cls.saved_max_rows_at_once def setUp(self): self.data = np.random.random((self.nrows, len(self.attributes))) self.class_data = np.random.random((self.nrows, len(self.class_vars))) if len(self.class_vars) == 1: self.class_data = self.class_data.flatten() self.meta_data = np.random.randint(0, 5, (self.nrows, len(self.metas)) ).astype(object) self.weight_data = np.random.random((self.nrows, 1)) def mock_domain(self, with_classes=False, with_metas=False): attributes = self.attributes class_var = self.class_vars[0] if with_classes else None class_vars = self.class_vars if with_classes else [] metas = self.metas if with_metas else [] variables = attributes + class_vars return MagicMock(data.Domain, attributes=attributes, class_var=class_var, class_vars=class_vars, metas=metas, variables=variables) def create_domain(self, attributes=(), classes=(), metas=()): attr_vars = [data.ContinuousVariable(name=a) if isinstance(a, str) else a for a in attributes] class_vars = [data.ContinuousVariable(name=c) if isinstance(c, str) else c for c in classes] meta_vars = [data.DiscreteVariable(name=m, values=map(str, range(5))) if isinstance(m, str) else m for m in metas] domain = data.Domain(attr_vars, class_vars, meta_vars) return domain class CreateEmptyTable(TableTests): def test_calling_new_with_no_parameters_constructs_a_new_instance(self): table = data.Table() self.assertIsInstance(table, data.Table) def test_table_has_file(self): table = data.Table() self.assertIsNone(table.__file__) class CreateTableWithFilename(TableTests): filename = "data.tab" @patch("os.path.exists", Mock(return_value=True)) def test_read_data_calls_reader(self): table_mock = Mock(data.Table) reader_instance = Mock(read=Mock(return_value=table_mock)) reader_mock = Mock(return_value=reader_instance) with patch.dict(data.io.FileFormat.readers, {'.xlsx': reader_mock}): table = data.Table.from_file("test.xlsx") reader_mock.assert_called_with("test.xlsx") reader_instance.read.assert_called_with() self.assertEqual(table, table_mock) @patch("os.path.exists", Mock(return_value=False)) def test_raises_error_if_file_does_not_exist(self): with self.assertRaises(IOError): data.Table.from_file(self.filename) @patch("os.path.exists", Mock(return_value=True)) def test_raises_error_if_file_has_unknown_extension(self): with self.assertRaises(IOError): data.Table.from_file("file.invalid_extension") @patch("Orange.data.table.Table.from_file") def test_calling_new_with_string_argument_calls_read_data(self, read_data): data.Table(self.filename) read_data.assert_called_with(self.filename) @patch("Orange.data.table.Table.from_file") def test_calling_new_with_keyword_argument_filename_calls_read_data( self, read_data): data.Table(self.filename) read_data.assert_called_with(self.filename) class CreateTableWithUrl(TableTests): def test_url_no_scheme(self): class SkipRest(Exception): pass mock_urlopen = Mock(side_effect=SkipRest()) url = 'www.foo.bar/xx.csv' with patch('Orange.data.io.UrlReader.urlopen', mock_urlopen): try: Table.from_url(url) except SkipRest: pass mock_urlopen.assert_called_once_with('http://' + url) class _MockUrlOpen(MagicMock): headers = {'content-disposition': 'attachment; filename="Something-FormResponses.tsv"; ' 'filename*=UTF-8''Something%20%28Responses%29.tsv'} def __enter__(self): return self def __exit__(self, *args, **kwargs): pass def read(self): return b'''\ a\tb\tc 1\t2\t3 2\t3\t4''' urlopen = _MockUrlOpen() @patch('Orange.data.io.urlopen', urlopen) def test_trimmed_urls(self): for url in ('https://docs.google.com/spreadsheets/d/ABCD/edit', 'https://www.dropbox.com/s/ABCD/filename.csv'): self._MockUrlOpen.url = url d = data.Table(url) request = self.urlopen.call_args[0][0] self.assertNotEqual(url, request.full_url) self.assertIn('Mozilla/5.0', request.headers.get('User-agent', '')) self.assertEqual(len(d), 2) self.assertEqual(d.name, 'Something-FormResponses') class CreateTableWithDomain(TableTests): def test_creates_an_empty_table_with_given_domain(self): domain = self.mock_domain() table = data.Table.from_domain(domain) self.assertEqual(table.domain, domain) def test_creates_zero_filled_rows_in_X_if_domain_contains_attributes(self): domain = self.mock_domain() table = data.Table.from_domain(domain, self.nrows) self.assertEqual(table.X.shape, (self.nrows, len(domain.attributes))) self.assertFalse(table.X.any()) def test_creates_zero_filled_rows_in_Y_if_domain_contains_class_vars(self): domain = self.mock_domain(with_classes=True) table = data.Table.from_domain(domain, self.nrows) if len(domain.class_vars) != 1: self.assertEqual(table.Y.shape, (self.nrows, len(domain.class_vars))) else: self.assertEqual(table.Y.shape, (self.nrows,)) self.assertFalse(table.Y.any()) def test_creates_zero_filled_rows_in_metas_if_domain_contains_metas(self): domain = self.mock_domain(with_metas=True) table = data.Table.from_domain(domain, self.nrows) self.assertEqual(table.metas.shape, (self.nrows, len(domain.metas))) self.assertFalse(table.metas.any()) def test_creates_weights_if_weights_are_true(self): domain = self.mock_domain() table = data.Table.from_domain(domain, self.nrows, True) self.assertEqual(table.W.shape, (self.nrows, )) def test_does_not_create_weights_if_weights_are_false(self): domain = self.mock_domain() table = data.Table.from_domain(domain, self.nrows, False) self.assertEqual(table.W.shape, (self.nrows, 0)) @patch("Orange.data.table.Table.from_domain") def test_calling_new_with_domain_calls_new_from_domain( self, new_from_domain): domain = self.mock_domain() data.Table.from_domain(domain) new_from_domain.assert_called_with(domain) class CreateTableWithData(TableTests): def test_creates_a_table_with_given_X(self): # from numpy table = data.Table.from_numpy(None, np.array(self.data)) self.assertIsInstance(table.domain, data.Domain) np.testing.assert_almost_equal(table.X, self.data) # from list table = data.Table.from_numpy(None, list(self.data)) self.assertIsInstance(table.domain, data.Domain) np.testing.assert_almost_equal(table.X, self.data) # from tuple table = data.Table.from_numpy(None, tuple(self.data)) self.assertIsInstance(table.domain, data.Domain) np.testing.assert_almost_equal(table.X, self.data) def test_creates_a_table_from_domain_and_list(self): domain = data.Domain([data.DiscreteVariable(name="a", values="mf"), data.ContinuousVariable(name="b")], data.DiscreteVariable(name="y", values="abc")) table = data.Table.from_list(domain, [[0, 1, 2], [1, 2, "?"], ["m", 3, "a"], ["?", "?", "c"]]) self.assertIs(table.domain, domain) np.testing.assert_almost_equal( table.X, np.array([[0, 1], [1, 2], [0, 3], [np.nan, np.nan]])) np.testing.assert_almost_equal(table.Y, np.array([2, np.nan, 0, 2])) def test_creates_a_table_from_domain_and_list_and_weights(self): domain = data.Domain([data.DiscreteVariable(name="a", values="mf"), data.ContinuousVariable(name="b")], data.DiscreteVariable(name="y", values="abc")) table = data.Table.from_list(domain, [[0, 1, 2], [1, 2, "?"], ["m", 3, "a"], ["?", "?", "c"]], [1, 2, 3, 4]) self.assertIs(table.domain, domain) np.testing.assert_almost_equal( table.X, np.array([[0, 1], [1, 2], [0, 3], [np.nan, np.nan]])) np.testing.assert_almost_equal(table.Y, np.array([2, np.nan, 0, 2])) np.testing.assert_almost_equal(table.W, np.array([1, 2, 3, 4])) def test_creates_a_table_from_domain_and_list_and_metas(self): metas = [data.DiscreteVariable("Meta 1", values="XYZ"), data.ContinuousVariable("Meta 2"), data.StringVariable("Meta 3")] domain = data.Domain([data.DiscreteVariable(name="a", values="mf"), data.ContinuousVariable(name="b")], data.DiscreteVariable(name="y", values="abc"), metas=metas) table = data.Table.from_list(domain, [[0, 1, 2, "X", 2, "bb"], [1, 2, "?", "Y", 1, "aa"], ["m", 3, "a", "Z", 3, "bb"], ["?", "?", "c", "X", 1, "aa"]]) self.assertIs(table.domain, domain) np.testing.assert_almost_equal( table.X, np.array([[0, 1], [1, 2], [0, 3], [np.nan, np.nan]])) np.testing.assert_almost_equal(table.Y, np.array([2, np.nan, 0, 2])) np.testing.assert_array_equal(table.metas, np.array([[0, 2., "bb"], [1, 1., "aa"], [2, 3., "bb"], [0, 1., "aa"]], dtype=object)) def test_creates_a_table_from_list_of_instances(self): table = data.Table('iris') new_table = data.Table.from_list(table.domain, [d for d in table]) self.assertIs(table.domain, new_table.domain) np.testing.assert_almost_equal(table.X, new_table.X) np.testing.assert_almost_equal(table.Y, new_table.Y) np.testing.assert_almost_equal(table.W, new_table.W) self.assertEqual(table.domain, new_table.domain) np.testing.assert_array_equal(table.metas, new_table.metas) def test_creates_a_table_from_list_of_instances_with_metas(self): table = data.Table('zoo') new_table = data.Table.from_list(table.domain, [d for d in table]) self.assertIs(table.domain, new_table.domain) np.testing.assert_almost_equal(table.X, new_table.X) np.testing.assert_almost_equal(table.Y, new_table.Y) np.testing.assert_almost_equal(table.W, new_table.W) self.assertEqual(table.domain, new_table.domain) np.testing.assert_array_equal(table.metas, new_table.metas) def test_creates_a_table_with_domain_and_given_X(self): domain = self.mock_domain() table = data.Table(domain, self.data) self.assertIsInstance(table.domain, data.Domain) self.assertEqual(table.domain, domain) np.testing.assert_almost_equal(table.X, self.data) def test_creates_a_table_with_given_X_and_Y(self): table = data.Table.from_numpy(None, self.data, self.class_data) self.assertIsInstance(table.domain, data.Domain) np.testing.assert_almost_equal(table.X, self.data) np.testing.assert_almost_equal(table.Y, self.class_data) def test_creates_a_table_with_given_X_Y_and_metas(self): table = data.Table.from_numpy( None, self.data, self.class_data, self.meta_data) self.assertIsInstance(table.domain, data.Domain) np.testing.assert_almost_equal(table.X, self.data) np.testing.assert_almost_equal(table.Y, self.class_data) np.testing.assert_almost_equal(table.metas, self.meta_data) def test_creates_a_discrete_class_if_Y_has_few_distinct_values(self): Y = np.array([float(np.random.randint(0, 2)) for i in self.data]) table = data.Table.from_numpy(None, self.data, Y, self.meta_data) np.testing.assert_almost_equal(table.Y, Y) self.assertIsInstance(table.domain.class_vars[0], data.DiscreteVariable) self.assertEqual(table.domain.class_vars[0].values, ("v1", "v2")) def test_creates_a_table_with_given_domain(self): domain = self.mock_domain() table = data.Table.from_numpy(domain, self.data) self.assertEqual(table.domain, domain) def test_sets_Y_if_given(self): domain = self.mock_domain(with_classes=True) table = data.Table.from_numpy(domain, self.data, self.class_data) np.testing.assert_almost_equal(table.Y, self.class_data) def test_sets_metas_if_given(self): domain = self.mock_domain(with_metas=True) table = data.Table.from_numpy(domain, self.data, metas=self.meta_data) np.testing.assert_almost_equal(table.metas, self.meta_data) def test_sets_weights_if_given(self): domain = self.mock_domain() table = data.Table.from_numpy(domain, self.data, W=self.weight_data) np.testing.assert_equal(table.W.shape, (len(self.data), )) np.testing.assert_almost_equal(table.W.flatten(), self.weight_data.flatten()) def test_splits_X_and_Y_if_given_in_same_array(self): joined_data = np.column_stack((self.data, self.class_data)) domain = self.mock_domain(with_classes=True) table = data.Table.from_numpy(domain, joined_data) np.testing.assert_almost_equal(table.X, self.data) np.testing.assert_almost_equal(table.Y, self.class_data) def test_initializes_Y_metas_and_W_if_not_given(self): domain = self.mock_domain() table = data.Table.from_numpy(domain, self.data) self.assertEqual(table.Y.shape, (self.nrows, len(domain.class_vars))) self.assertEqual(table.metas.shape, (self.nrows, len(domain.metas))) self.assertEqual(table.W.shape, (self.nrows, 0)) def test_raises_error_if_columns_in_domain_and_data_do_not_match(self): domain = self.mock_domain(with_classes=True, with_metas=True) ones = np.zeros((self.nrows, 1)) with self.assertRaises(ValueError): data_ = np.hstack((self.data, ones)) data.Table.from_numpy(domain, data_, self.class_data, self.meta_data) with self.assertRaises(ValueError): classes_ = np.hstack((self.class_data, ones)) data.Table.from_numpy(domain, self.data, classes_, self.meta_data) with self.assertRaises(ValueError): metas_ = np.hstack((self.meta_data, ones)) data.Table.from_numpy(domain, self.data, self.class_data, metas_) def test_raises_error_if_lengths_of_data_do_not_match(self): domain = self.mock_domain(with_classes=True, with_metas=True) with self.assertRaises(ValueError): data_ = np.vstack((self.data, np.zeros((1, len(self.attributes))))) data.Table(domain, data_, self.class_data, self.meta_data) with self.assertRaises(ValueError): class_data_ = np.vstack((self.class_data, np.zeros((1, len(self.class_vars))))) data.Table(domain, self.data, class_data_, self.meta_data) with self.assertRaises(ValueError): meta_data_ = np.vstack((self.meta_data, np.zeros((1, len(self.metas))))) data.Table(domain, self.data, self.class_data, meta_data_) @patch("Orange.data.table.Table.from_numpy") def test_calling_new_with_domain_and_numpy_arrays_calls_new_from_numpy( self, new_from_numpy): domain = self.mock_domain() data.Table(domain, self.data) new_from_numpy.assert_called_with(domain, self.data) domain = self.mock_domain(with_classes=True) data.Table(domain, self.data, self.class_data) new_from_numpy.assert_called_with(domain, self.data, self.class_data) domain = self.mock_domain(with_classes=True, with_metas=True) data.Table(domain, self.data, self.class_data, self.meta_data) new_from_numpy.assert_called_with( domain, self.data, self.class_data, self.meta_data) data.Table(domain, self.data, self.class_data, self.meta_data, self.weight_data) new_from_numpy.assert_called_with(domain, self.data, self.class_data, self.meta_data, self.weight_data) def test_from_numpy_reconstructable(self): def assert_equal(t1, t2): np.testing.assert_array_equal(t1.X, t2.X) np.testing.assert_array_equal(t1.Y, t2.Y) np.testing.assert_array_equal(t1.metas, t2.metas) np.testing.assert_array_equal(t1.W, t2.W) nullcol = np.empty((self.nrows, 0)) domain = self.create_domain(self.attributes) table = data.Table(domain, self.data) table_1 = data.Table.from_numpy( domain, table.X, table.Y, table.metas, table.W) assert_equal(table, table_1) domain = self.create_domain(classes=self.class_vars) table = data.Table(domain, nullcol, self.class_data) table_1 = data.Table.from_numpy( domain, table.X, table.Y, table.metas, table.W) assert_equal(table, table_1) domain = self.create_domain(metas=self.metas) table = data.Table(domain, nullcol, nullcol, self.meta_data) table_1 = data.Table.from_numpy( domain, table.X, table.Y, table.metas, table.W) assert_equal(table, table_1) class CreateTableWithDomainAndTable(TableTests): interesting_slices = [ slice(0, 0), # [0:0] - empty slice slice(1), # [:1] - only first element slice(1, None), # [1:] - all but first slice(-1, None), # [-1:] - only last element slice(-1), # [:-1] - all but last slice(None), # [:] - all elements slice(None, None, 2), # [::2] - even elements slice(None, None, -1), # [::-1]- all elements reversed slice(9, 5, -10), # slice a big negative stride and thus 1 element ] row_indices = [1, 5, 6, 7] def setUp(self): super().setUp() self.domain = self.create_domain( self.attributes, self.class_vars, self.metas) self.table = data.Table( self.domain, self.data, self.class_data, self.meta_data) def test_creates_table_with_given_domain(self): new_table = data.Table.from_table(self.table.domain, self.table) self.assertIsInstance(new_table, data.Table) self.assertIsNot(self.table, new_table) self.assertEqual(new_table.domain, self.domain) def test_transform(self): class MyTableClass(data.Table): pass table = MyTableClass.from_table(self.table.domain, self.table) domain = table.domain attr = ContinuousVariable("x") new_domain = data.Domain(list(domain.attributes) + [attr], None) new_table = table.transform(new_domain) self.assertIsInstance(new_table, MyTableClass) self.assertIsNot(table, new_table) self.assertIs(new_table.domain, new_domain) def test_transform_same_domain(self): iris = data.Table("iris") new_domain = copy.copy(iris.domain) new_data = iris.transform(new_domain) self.assertIs(new_data.domain, new_domain) def test_can_copy_table(self): new_table = data.Table.from_table(self.domain, self.table) self.assert_table_with_filter_matches(new_table, self.table) def test_can_filter_rows_with_list(self): for indices in ([0], [1, 5, 6, 7]): new_table = data.Table.from_table( self.domain, self.table, row_indices=indices) self.assert_table_with_filter_matches( new_table, self.table, rows=indices) def test_can_filter_row_with_slice_from_table_rows(self): # calling from_table with the same domain will forward to from_table_rows # and thus _FromTableConversion.convert should not be called with patch.object(_FromTableConversion, "convert") as convert: for slice_ in self.interesting_slices: new_table = data.Table.from_table( self.domain, self.table, row_indices=slice_) self.assert_table_with_filter_matches( new_table, self.table, rows=slice_) convert.assert_not_called() def test_can_filter_row_with_slice_from_table(self): # a utility class needed for mocking of the convert method in this case class MockedConversion(_FromTableConversion): objects = [] def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.convert = Mock(wraps=self.convert) self.objects.append(self) # calling from_table with a domain copy will use indexing in from_table and # _FromTableConversion.convert with patch("Orange.data.table._FromTableConversion", MockedConversion): for slice_ in self.interesting_slices: new_table = data.Table.from_table( self.domain.copy(), self.table, row_indices=slice_) self.assert_table_with_filter_matches( new_table, self.table, rows=slice_) self.assertEqual(len(MockedConversion.objects), 1) MockedConversion.objects.clear() def test_can_use_attributes_as_new_columns(self): a, _, _ = column_sizes(self.table) order = np.random.permutation(a) new_attributes = [self.domain.attributes[i] for i in order] new_domain = self.create_domain( new_attributes[:2], new_attributes[2:4], new_attributes[4:]) new_table = data.Table.from_table(new_domain, self.table) self.assert_table_with_filter_matches( new_table, self.table, xcols=order[:2], ycols=order[2:4], mcols=order[4:]) def test_can_use_class_vars_as_new_columns(self): a, _, _ = column_sizes(self.table) order = np.random.permutation(range(a)) cvs = [self.domain.attributes[i] for i in order[:2]] metas = [self.domain.attributes[i] for i in order[2:]] new_domain = self.create_domain(self.domain.class_vars, cvs, metas) new_table = data.Table.from_table(new_domain, self.table) self.assert_table_with_filter_matches( new_table, self.table, xcols=[a], ycols=order[:2], mcols=order[2:]) def test_can_use_metas_as_new_columns(self): _, _, m = column_sizes(self.table) order = np.random.permutation(range(-m, 0)) new_metas = [self.domain.metas[::-1][i] for i in order] new_domain = self.create_domain(new_metas[0:2], [new_metas[2]], new_metas[3:5]) new_table = data.Table.from_table(new_domain, self.table) self.assert_table_with_filter_matches( new_table, self.table, xcols=order[0:2], ycols=order[2], mcols=order[3:5]) def test_can_use_combination_of_all_as_new_columns(self): a, c, m = column_sizes(self.table) order = (list(range(a+c)) + list(range(-m+1, 0))) random. shuffle(order) vars_ = list(self.domain.variables) + list(self.domain.metas[::-1]) atrs = [vars_[order[i]] for i in range(a)] cv = [vars_[order[i]] for i in range(a, a+c)] metas = [vars_[order[i]] for i in range(a+c, a+c+m-1)] new_domain = self.create_domain(atrs, cv, metas) new_table = data.Table.from_table(new_domain, self.table) self.assert_table_with_filter_matches( new_table, self.table, xcols=order[:a], ycols=order[a:a+c], mcols=order[a+c:]) def test_creates_table_with_given_domain_and_row_filter(self): a, c, m = column_sizes(self.table) order = (list(range(a+c)) + list(range(-m+1, 0))) random.shuffle(order) vars_ = list(self.domain.variables) + list(self.domain.metas[::-1]) atrs = [vars_[order[i]] for i in range(a)] cv = [vars_[order[i]] for i in range(a, a+c)] metas = [vars_[order[i]] for i in range(a+c, a+c+m-1)] new_domain = self.create_domain(atrs, cv, metas) new_table = data.Table.from_table(new_domain, self.table, [0]) self.assert_table_with_filter_matches( new_table, self.table[:1], xcols=order[:a], ycols=order[a:a+c], mcols=order[a+c:]) new_table = data.Table.from_table(new_domain, self.table, [2, 1, 0]) self.assert_table_with_filter_matches( new_table, self.table[2::-1], xcols=order[:a], ycols=order[a:a+c], mcols=order[a+c:]) new_table = data.Table.from_table(new_domain, self.table, []) self.assert_table_with_filter_matches( new_table, self.table[:0], xcols=order[:a], ycols=order[a:a+c], mcols=order[a+c:]) def test_from_table_with_boolean_row_filter(self): a, c, m = column_sizes(self.table) domain = self.table.domain sel = [False]*len(self.table) sel[2] = True with patch.object(Table, "from_table_rows", wraps=Table.from_table_rows) \ as from_table_rows: new_table = Table.from_table(self.table.domain, self.table, row_indices=sel) from_table_rows.assert_called() self.assert_table_with_filter_matches( new_table, self.table[2:3]) new_domain1 = Domain(domain.attributes[:1], domain.class_vars[:1], domain.metas[:1]) with patch.object(Table, "from_table_rows", wraps=Table.from_table_rows) \ as from_table_rows: new_table = Table.from_table(new_domain1, self.table, row_indices=sel) from_table_rows.assert_not_called() self.assert_table_with_filter_matches( new_table, self.table[2:3], xcols=[0], ycols=[a], mcols=[a+c+m-1]) new_domain2 = Domain(domain.attributes[:1] + (ContinuousVariable("new"),), domain.class_vars[:1], domain.metas[:1]) new_table = Table.from_table(new_domain2, self.table, row_indices=sel) self.assert_table_with_filter_matches( new_table.transform(new_domain1), self.table[2:3], xcols=[0], ycols=[a], mcols=[a+c+m-1]) def test_from_table_sparse_move_some_to_empty_metas(self): iris = data.Table("iris").to_sparse() new_domain = data.domain.Domain( iris.domain.attributes[:2], iris.domain.class_vars, iris.domain.attributes[2:], source=iris.domain) new_iris = iris.transform(new_domain) self.assertTrue(sp.issparse(new_iris.X)) self.assertTrue(sp.issparse(new_iris.metas)) self.assertEqual(new_iris.X.shape, (len(iris), 2)) self.assertEqual(new_iris.metas.shape, (len(iris), 2)) # move back back_iris = new_iris.transform(iris.domain) self.assertEqual(back_iris.domain, iris.domain) self.assertTrue(sp.issparse(back_iris.X)) self.assertFalse(sp.issparse(back_iris.metas)) self.assertEqual(back_iris.X.shape, iris.X.shape) self.assertEqual(back_iris.metas.shape, iris.metas.shape) def test_from_table_sparse_move_all_to_empty_metas(self): iris = data.Table("iris").to_sparse() new_domain = data.domain.Domain( [], iris.domain.class_vars, iris.domain.attributes, source=iris.domain) new_iris = iris.transform(new_domain) self.assertFalse(sp.issparse(new_iris.X)) self.assertTrue(sp.issparse(new_iris.metas)) self.assertEqual(new_iris.X.shape, (len(iris), 0)) self.assertEqual(new_iris.metas.shape, (len(iris), 4)) # move back back_iris = new_iris.transform(iris.domain) self.assertEqual(back_iris.domain, iris.domain) self.assertTrue(sp.issparse(back_iris.X)) self.assertFalse(sp.issparse(back_iris.metas)) self.assertEqual(back_iris.X.shape, iris.X.shape) self.assertEqual(back_iris.metas.shape, iris.metas.shape) def test_from_table_sparse_move_to_nonempty_metas(self): brown = data.Table("brown-selected").to_sparse() n_attr = len(brown.domain.attributes) n_metas = len(brown.domain.metas) new_domain = data.domain.Domain( brown.domain.attributes[:-10], brown.domain.class_vars, brown.domain.attributes[-10:] + brown.domain.metas, source=brown.domain) new_brown = data.Table.from_table(new_domain, brown) self.assertTrue(sp.issparse(new_brown.X)) self.assertFalse(sp.issparse(new_brown.metas)) self.assertEqual(new_brown.X.shape, (len(new_brown), n_attr-10)) self.assertEqual(new_brown.metas.shape, (len(new_brown), n_metas+10)) # move back back_brown = data.Table.from_table(brown.domain, new_brown) self.assertEqual(brown.domain, back_brown.domain) self.assertTrue(sp.issparse(back_brown.X)) self.assertFalse(sp.issparse(back_brown.metas)) self.assertEqual(back_brown.X.shape, brown.X.shape) self.assertEqual(back_brown.metas.shape, brown.metas.shape) def test_from_table_partwise(self): def sum_x(d): return d.X.sum(axis=1) sum_x = Mock(wraps=sum_x) def sum_y(d): return d.Y.sum(axis=1) sum_y = Mock(wraps=sum_y) def sum_metas(d): return d.metas.sum(axis=1) sum_metas = Mock(wraps=sum_metas) sum_x_var = ContinuousVariable("sum_x", compute_value=sum_x) sum_y_var = ContinuousVariable("sum_y", compute_value=sum_y) sum_metas_var = ContinuousVariable("sum_metas", compute_value=sum_metas) target_domain = Domain([sum_x_var], [sum_y_var], [sum_metas_var]) def assure_sum(): np.testing.assert_equal(orig.X.sum(axis=1), transformed.X[:,0]) np.testing.assert_equal(orig.Y.sum(axis=1), transformed.Y) np.testing.assert_equal(orig.metas.sum(axis=1), transformed.metas[:,0]) def long_table(rows): avars = [ContinuousVariable(n) for n in "abcdef"] vals = np.random.RandomState(0).random((rows, 6)) domain = Domain(avars[:2], avars[2:4], avars[4:]) return Table.from_numpy(domain, X=vals[:, :2], Y=vals[:, 2:4], metas=vals[:, 4:]) max_rows = _FromTableConversion.max_rows_at_once # domain conversion fits into a single part orig = long_table(max_rows) transformed = Table.from_table(target_domain, orig) assure_sum() sum_x.assert_called_once() sum_y.assert_called_once() sum_metas.assert_called_once() sum_x.reset_mock() sum_y.reset_mock() sum_metas.reset_mock() # domain conversion does not fit a single part orig = long_table(max_rows + 1) transformed = Table.from_table(target_domain, orig) assure_sum() self.assertEqual(sum_x.call_count, 2) self.assertEqual(sum_y.call_count, 2) self.assertEqual(sum_metas.call_count, 2) self.assertEqual(len(sum_x.call_args_list[0][0][0]), max_rows) self.assertEqual(len(sum_x.call_args_list[1][0][0]), 1) self.assertEqual(len(sum_y.call_args_list[0][0][0]), max_rows) self.assertEqual(len(sum_y.call_args_list[1][0][0]), 1) self.assertEqual(len(sum_metas.call_args_list[0][0][0]), max_rows) self.assertEqual(len(sum_metas.call_args_list[1][0][0]), 1) def test_from_table_shared_compute_value(self): iris = data.Table("iris").to_sparse() d1 = Domain( [ ContinuousVariable( name=at.name, compute_value=PreprocessSharedComputeValue( i, None, PreprocessShared(iris.domain, None) ) ) for i, at in enumerate(iris.domain.attributes) ] ) new_table = Table.from_table(d1, iris) np.testing.assert_array_equal(new_table.X, iris.X.todense() * 2) new_table = Table.from_table(d1, iris, row_indices=[0, 1, 2]) np.testing.assert_array_equal(new_table.X, iris.X.todense()[:3] * 2) def assert_table_with_filter_matches( self, new_table, old_table, rows=..., xcols=..., ycols=..., mcols=...): a, c, m = column_sizes(old_table) xcols = slice(a) if xcols is Ellipsis else xcols ycols = slice(a, a + c) if ycols is Ellipsis else ycols mcols = slice(None, -m - 1, -1) if mcols is Ellipsis else mcols # Indexing used by convert_domain uses positive indices for variables # and classes (classes come after attributes) and negative indices for # meta features. This is equivalent to ordinary indexing in a magic # table below. magic = np.hstack((old_table.X, old_table.Y[:, None], old_table.metas[:, ::-1])) np.testing.assert_almost_equal(new_table.X, magic[rows, xcols]) Y = magic[rows, ycols] if len(Y.shape) == 2 and Y.shape[1] == 1: Y = Y.flatten() np.testing.assert_almost_equal(new_table.Y, Y) np.testing.assert_almost_equal(new_table.metas, magic[rows, mcols]) np.testing.assert_almost_equal(new_table.W, old_table.W[rows]) def test_attributes_copied(self): """Table created from table attributes dict copied""" self.table.attributes = {"A": "Test", "B": []} # from_table new_table = self.table.from_table(self.table.domain, self.table) self.assertDictEqual(new_table.attributes, {"A": "Test", "B": []}) new_table.attributes["A"] = "Changed" new_table.attributes["B"].append(1) self.assertDictEqual(new_table.attributes, {"A": "Changed", "B": [1]}) # attributes dict of old table not be changed since new dist is a copy self.assertDictEqual(self.table.attributes, {"A": "Test", "B": []}) # from_table_rows new_table = self.table.from_table_rows(self.table, [1, 2]) self.assertDictEqual(new_table.attributes, {"A": "Test", "B": []}) new_table.attributes["A"] = "Changed" new_table.attributes["B"].append(1) self.assertDictEqual(new_table.attributes, {"A": "Changed", "B": [1]}) # attributes dict of old table not be changed since new dist is a copy self.assertDictEqual(self.table.attributes, {"A": "Test", "B": []}) def test_attributes_copied_once(self): A = Mock() A.__deepcopy__ = Mock() self.table.attributes = {"A": A} # a single direct transformation self.table.from_table(self.table.domain, self.table) self.assertEqual(1, A.__deepcopy__.call_count) A.__deepcopy__.reset_mock() # hierarchy of transformations ndom = Domain([a.copy(compute_value=lambda x: x.transform(Domain([a]))) for a in self.table.domain.attributes]) self.table.from_table(ndom, self.table) self.assertEqual(1, A.__deepcopy__.call_count) # HISTORIC: before only the outermost transformation deepcopied the # attributes, here were 23 calls to __deepcopy__ instead of 1 def isspecial(s): return isinstance(s, slice) or s is Ellipsis def split_columns(indices, t): a, c, m = column_sizes(t) if indices is ...: return slice(a), slice(c), slice(m) elif isinstance(indices, slice): return indices, slice(0, 0), slice(0, 0) elif not isinstance(indices, list) and not isinstance(indices, tuple): indices = [indices] return ( [t.domain.index(x) for x in indices if 0 <= t.domain.index(x) < a] or slice(0, 0), [t.domain.index(x) - a for x in indices if t.domain.index(x) >= a] or slice(0, 0), [-t.domain.index(x) - 1 for x in indices if t.domain.index(x) < 0] or slice(0, 0)) def getname(variable): return variable.name class TableIndexingTests(TableTests): def setUp(self): super().setUp() d = self.domain = \ self.create_domain(self.attributes, self.class_vars, self.metas) t = self.table = \ data.Table(self.domain, self.data, self.class_data, self.meta_data) self.magic_table = \ np.column_stack((self.table.X, self.table.Y, self.table.metas[:, ::-1])) self.rows = [0, -1] self.multiple_rows = [slice(0, 0), ..., slice(1, -1, -1)] a, c, m = column_sizes(t) columns = [0, a - 1, a, a + c - 1, -1, -m] self.columns = chain(columns, map(lambda x: d[x], columns), map(lambda x: d[x].name, columns)) self.multiple_columns = chain( self.multiple_rows, [d.attributes, d.class_vars, d.metas, [0, a, -1]], [self.attributes, self.class_vars, self.metas], [self.attributes + self.class_vars + self.metas]) # TODO: indexing with [[0,1], [0,1]] produces weird results # TODO: what should be the results of table[1, :] def test_can_select_a_single_value(self): for r in self.rows: for c in self.columns: value = self.table[r, c] self.assertAlmostEqual( value, self.magic_table[r, self.domain.index(c)]) value = self.table[r][c] self.assertAlmostEqual( value, self.magic_table[r, self.domain.index(c)]) def test_can_select_a_single_row(self): for r in self.rows: row = self.table[r] new_row = np.hstack( (self.data[r, :], self.class_data[r, None])) np.testing.assert_almost_equal( np.array(list(row)), new_row) def test_can_select_a_subset_of_rows_and_columns(self): for r in self.rows: for c in self.multiple_columns: table = self.table[r, c] attr, cls, metas = split_columns(c, self.table) X = self.table.X[[r], attr] if X.ndim == 1: X = X.reshape(-1, len(table.domain.attributes)) np.testing.assert_almost_equal(table.X, X) Y = self.table.Y[:, None][[r], cls] if len(Y.shape) == 1 or Y.shape[1] == 1: Y = Y.flatten() np.testing.assert_almost_equal(table.Y, Y) metas_ = self.table.metas[[r], metas] if metas_.ndim == 1: metas_ = metas_.reshape(-1, len(table.domain.metas)) np.testing.assert_almost_equal(table.metas, metas_) for r in self.multiple_rows: for c in chain(self.columns, self.multiple_rows): table = self.table[r, c] attr, cls, metas = split_columns(c, self.table) np.testing.assert_almost_equal(table.X, self.table.X[r, attr]) Y = self.table.Y[:, None][r, cls] if len(Y.shape) > 1 and Y.shape[1] == 1: Y = Y.flatten() np.testing.assert_almost_equal(table.Y, Y) np.testing.assert_almost_equal(table.metas, self.table.metas[r, metas]) def test_optimize_indices(self): # ordinary conversion self.assertEqual(_optimize_indices([1, 2, 3], 4), slice(1, 4, 1)) self.assertEqual(_optimize_indices([], 1), []) self.assertEqual(_optimize_indices([0, 2], 3), slice(0, 4, 2)) # not slices np.testing.assert_equal(_optimize_indices([1, 2, 4], 5), [1, 2, 4]) np.testing.assert_equal(_optimize_indices((1, 2, 4), 5), [1, 2, 4]) # internally convert boolean arrays into indices np.testing.assert_equal(_optimize_indices([False, False, False, False], 4), []) np.testing.assert_equal(_optimize_indices([True, False, True, True], 4), [0, 2, 3]) np.testing.assert_equal(_optimize_indices([True, False, True], 3), slice(0, 4, 2)) with self.assertRaises(IndexError): _optimize_indices([True, False, True], 2) with self.assertRaises(IndexError): _optimize_indices([True, False, True], 4) # do not convert if step is negative np.testing.assert_equal(_optimize_indices([4, 2, 0], 5), [4, 2, 0]) # try range np.testing.assert_equal(_optimize_indices([3, 4, 5], 5), [3, 4, 5]) # out of range self.assertEqual(_optimize_indices((3, 4, 5), 6), slice(3, 6, 1)) # negative elements np.testing.assert_equal(_optimize_indices([-1, 0, 1], 5), [-1, 0, 1]) # single element self.assertEqual(_optimize_indices([1], 2), slice(1, 2, 1)) self.assertEqual(_optimize_indices([-2], 5), slice(-2, -3, -1)) def test_select_from_selection(self): fn = _select_from_selection self.assertEqual(fn(slice(10), slice(11), 10), slice(0, 10, 1)) self.assertEqual(fn(slice(10), slice(None, 10, 2), 10), slice(0, 10, 2)) self.assertEqual(fn(slice(None, 10, 2), slice(None, 10, 2), 10), slice(0, 10, 4)) self.assertEqual(fn(slice(None, None, -1), slice(0, 9, None), 10), slice(9, 0, -1)) # [9, 8, 7, 6, 5, 4, 3, 2, 1] self.assertEqual(fn(slice(None, None, -1), slice(9, 10, None), 10), slice(0, None, -1)) # [0] self.assertEqual(fn(slice(None, 10, 2), slice(None, None, -1), 10), slice(8, None, -2)) self.assertEqual(fn(slice(None, 10, 2), slice(None, None, -2), 10), slice(8, None, -4)) class TableElementAssignmentTest(TableTests): def setUp(self): super().setUp() self.domain = \ self.create_domain(self.attributes, self.class_vars, self.metas) self.table = \ data.Table(self.domain, self.data, self.class_data, self.meta_data) def test_can_assign_values(self): with self.table.unlocked(): self.table[0, 0] = 42. self.assertAlmostEqual(self.table.X[0, 0], 42.) def test_can_assign_values_to_classes(self): a, _, _ = column_sizes(self.table) with self.table.unlocked(): self.table[0, a] = 42. self.assertAlmostEqual(self.table.Y[0], 42.) def test_can_assign_values_to_metas(self): with self.table.unlocked(): self.table[0, -1] = 42. self.assertAlmostEqual(self.table.metas[0, 0], 42.) def test_can_assign_rows_to_rows(self): with self.table.unlocked(): self.table[0] = self.table[1] np.testing.assert_almost_equal( self.table.X[0], self.table.X[1]) np.testing.assert_almost_equal( self.table.Y[0], self.table.Y[1]) np.testing.assert_almost_equal( self.table.metas[0], self.table.metas[1]) def test_can_assign_lists(self): a, _, _ = column_sizes(self.table) new_example = [float(i) for i in range(len(self.attributes + self.class_vars))] with self.table.unlocked(): self.table[0] = new_example np.testing.assert_almost_equal( self.table.X[0], np.array(new_example[:a])) np.testing.assert_almost_equal( self.table.Y[0], np.array(new_example[a:])) def test_can_assign_np_array(self): a, _, _ = column_sizes(self.table) new_example = \ np.array([float(i) for i in range(len(self.attributes + self.class_vars))]) with self.table.unlocked(): self.table[0] = new_example np.testing.assert_almost_equal(self.table.X[0], new_example[:a]) np.testing.assert_almost_equal(self.table.Y[0], new_example[a:]) class InterfaceTest(unittest.TestCase): """Basic tests each implementation of Table should pass.""" features = ( data.ContinuousVariable(name="Continuous Feature 1"), data.ContinuousVariable(name="Continuous Feature 2"), data.DiscreteVariable(name="Discrete Feature 1", values=("0", "1")), data.DiscreteVariable(name="Discrete Feature 2", values=("value1", "value2")), ) class_vars = ( data.ContinuousVariable(name="Continuous Class"), data.DiscreteVariable(name="Discrete Class", values=("m", "f")) ) feature_data = ( (1, 0, 0, 0), (0, 1, 0, 0), (0, 0, 1, 0), (0, 0, 0, 1), ) class_data = ( (1, 0), (0, 1), (1, 0), (0, 1) ) data = tuple(a + c for a, c in zip(feature_data, class_data)) nrows = 4 def setUp(self): self.domain = data.Domain(attributes=self.features, class_vars=self.class_vars) self.table = data.Table.from_numpy( self.domain, np.array(self.feature_data), np.array(self.class_data), ) def test_len(self): self.assertEqual(len(self.table), self.nrows) def test_row_len(self): for i in range(self.nrows): self.assertEqual(len(self.table[i]), len(self.data[i])) def test_iteration(self): for row, expected_data in zip(self.table, self.data): self.assertEqual(tuple(row), expected_data) def test_row_indexing(self): for i in range(self.nrows): self.assertEqual(tuple(self.table[i]), self.data[i]) def test_row_slicing(self): t = self.table[1:] self.assertEqual(len(t), self.nrows - 1) def test_value_indexing(self): for i in range(self.nrows): for j in range(len(self.table[i])): self.assertEqual(self.table[i, j], self.data[i][j]) def test_row_assignment(self): new_value = 2. with self.table.unlocked(): for i in range(self.nrows): new_row = [new_value] * len(self.data[i]) self.table[i] = np.array(new_row) self.assertEqual(list(self.table[i]), new_row) def test_value_assignment(self): new_value = 0. with self.table.unlocked(): for i in range(self.nrows): for j in range(len(self.table[i])): self.table[i, j] = new_value self.assertEqual(self.table[i, j], new_value) class TestTableStats(TableTests): def test_get_nan_frequency(self): metas = [DiscreteVariable("x", values=tuple("abc")), StringVariable("s")] meta_data = np.array([list(range(self.nrows)), ["x"] * self.nrows]).T domain = self.create_domain(self.attributes, self.class_vars) domain = Domain(domain.attributes, domain.class_vars, metas) table = data.Table(domain, self.data, self.class_data, meta_data) def test_counts(at, cl, me): x, y, metas = table.X, table.Y, table.metas for _ in range(2): self.assertEqual(table.get_nan_count_attribute(), at) self.assertEqual(table.get_nan_count_class(), cl) self.assertEqual(table.get_nan_count_metas(), me) self.assertEqual(table.get_nan_frequency_attribute(), at / np.prod(x.shape)) self.assertEqual(table.get_nan_frequency_class(), cl / np.prod(y.shape)) self.assertEqual(table.get_nan_frequency_metas(), me / np.prod(metas.shape)) with table.unlocked(): table.X = sp.csr_matrix(x) table.Y = sp.csr_matrix(y) with table.unlocked(): table.X, table.Y = x, y test_counts(0, 0, 0) with table.unlocked(): table.X[1, 2] = table.X[4, 5] = np.nan test_counts(2, 0, 0) with table.unlocked(): table.Y[3:6] = np.nan test_counts(2, 3, 0) with table.unlocked(): table.X[1, 2] = table.X[4, 5] = 0 test_counts(0, 3, 0) with table.unlocked(): table.metas[1, 0] = table.metas[3, 0] = np.nan test_counts(0, 3, 2) with table.unlocked(): table.metas[5, 1] = "" test_counts(0, 3, 3) def test_get_nan_frequency_empty_table(self): domain = self.create_domain(self.attributes, self.class_vars) table = data.Table.from_domain(domain) self.assertEqual(table.get_nan_count_attribute(), 0) self.assertEqual(table.get_nan_count_class(), 0) self.assertEqual(table.get_nan_count_metas(), 0) self.assertEqual(table.get_nan_frequency_attribute(), 0) self.assertEqual(table.get_nan_frequency_class(), 0) self.assertEqual(table.get_nan_frequency_metas(), 0) class TestRowInstance(unittest.TestCase): def test_assignment(self): table = data.Table("zoo") inst = table[2] self.assertIsInstance(inst, data.RowInstance) with table.unlocked(): inst[1] = 0 self.assertEqual(table[2, 1], 0) with table.unlocked(): inst[1] = 1 self.assertEqual(table[2, 1], 1) with table.unlocked(): inst.set_class("mammal") self.assertEqual(table[2, len(table.domain.attributes)], "mammal") with table.unlocked(): inst.set_class("fish") self.assertEqual(table[2, len(table.domain.attributes)], "fish") with table.unlocked(): inst[-1] = "Foo" self.assertEqual(table[2, -1], "Foo") def test_iteration_with_assignment(self): table = data.Table("iris") with table.unlocked(): for i, row in enumerate(table): row[0] = i np.testing.assert_array_equal(table.X[:, 0], np.arange(len(table))) def test_sparse_assignment(self): X = np.eye(4) Y = X[2].copy() table = data.Table.from_numpy(None, X, Y) row = table[1] self.assertFalse(sp.issparse(row.sparse_x)) self.assertEqual(row[0], 0) self.assertEqual(row[1], 1) with table.unlocked(): table.X = sp.csr_matrix(table.X) table.Y = sp.csr_matrix(table.Y) sparse_row = table[1] self.assertTrue(sp.issparse(sparse_row.sparse_x)) self.assertEqual(sparse_row[0], 0) self.assertEqual(sparse_row[1], 1) with table.unlocked(): sparse_row[1] = 0 self.assertEqual(sparse_row[1], 0) self.assertEqual(table.X[1, 1], 0) self.assertEqual(table[2][4], 1) with table.unlocked(): table[2][4] = 0 self.assertEqual(table[2][4], 0) class TestTableTranspose(unittest.TestCase): def test_transpose_no_class(self): attrs = [ContinuousVariable("c1"), ContinuousVariable("c2")] table = Table(Domain(attrs), np.arange(8).reshape((4, 2))) att = [ContinuousVariable("Feature 1"), ContinuousVariable("Feature 2"), ContinuousVariable("Feature 3"), ContinuousVariable("Feature 4")] domain = Domain(att, metas=[StringVariable("Feature name")]) result = Table(domain, np.arange(8).reshape((4, 2)).T, metas=np.array(["c1", "c2"])[:, None]) # transpose and compare self._compare_tables(result, Table.transpose(table)) # transpose of transpose is original t = Table.transpose(Table.transpose(table), "Feature name") self._compare_tables(table, t) # original should not change self.assertDictEqual(table.domain.attributes[0].attributes, {}) def test_transpose_discrete_class(self): attrs = [ContinuousVariable("c1"), ContinuousVariable("c2")] domain = Domain(attrs, [DiscreteVariable("cls", values=("a", "b"))]) table = Table(domain, np.arange(8).reshape((4, 2)), np.array([1, 1, 0, 0])) att = [ContinuousVariable("Feature 1"), ContinuousVariable("Feature 2"), ContinuousVariable("Feature 3"), ContinuousVariable("Feature 4")] att[0].attributes = {"cls": "b"} att[1].attributes = {"cls": "b"} att[2].attributes = {"cls": "a"} att[3].attributes = {"cls": "a"} domain = Domain(att, metas=[StringVariable("Feature name")]) result = Table(domain, np.arange(8).reshape((4, 2)).T, metas=np.array(["c1", "c2"])[:, None]) # transpose and compare self._compare_tables(result, Table.transpose(table)) # transpose of transpose is original t = Table.transpose(Table.transpose(table), "Feature name") self._compare_tables(table, t) # original should not change self.assertDictEqual(table.domain.attributes[0].attributes, {}) def test_transpose_continuous_class(self): attrs = [ContinuousVariable("c1"), ContinuousVariable("c2")] domain = Domain(attrs, [ContinuousVariable("cls")]) table = Table(domain, np.arange(8).reshape((4, 2)), np.arange(4, 0, -1)) att = [ContinuousVariable("Feature 1"), ContinuousVariable("Feature 2"), ContinuousVariable("Feature 3"), ContinuousVariable("Feature 4")] att[0].attributes = {"cls": "4"} att[1].attributes = {"cls": "3"} att[2].attributes = {"cls": "2"} att[3].attributes = {"cls": "1"} domain = Domain(att, metas=[StringVariable("Feature name")]) result = Table(domain, np.arange(8).reshape((4, 2)).T, metas=np.array(["c1", "c2"])[:, None]) # transpose and compare self._compare_tables(result, Table.transpose(table)) # transpose of transpose t = Table.transpose(Table.transpose(table), "Feature name") self._compare_tables(table, t) # original should not change self.assertDictEqual(table.domain.attributes[0].attributes, {}) def test_transpose_missing_class(self): attrs = [ContinuousVariable("c1"), ContinuousVariable("c2")] domain = Domain(attrs, [ContinuousVariable("cls")]) table = Table(domain, np.arange(8).reshape((4, 2)), np.array([np.nan, 3, 2, 1])) att = [ContinuousVariable("Feature 1"), ContinuousVariable("Feature 2"), ContinuousVariable("Feature 3"), ContinuousVariable("Feature 4")] att[1].attributes = {"cls": "3"} att[2].attributes = {"cls": "2"} att[3].attributes = {"cls": "1"} domain = Domain(att, metas=[StringVariable("Feature name")]) result = Table(domain, np.arange(8).reshape((4, 2)).T, metas=np.array(["c1", "c2"])[:, None]) # transpose and compare self._compare_tables(result, Table.transpose(table)) # transpose of transpose t = Table.transpose(Table.transpose(table), "Feature name") self._compare_tables(table, t) # original should not change self.assertDictEqual(table.domain.attributes[0].attributes, {}) def test_transpose_multiple_class(self): attrs = [ContinuousVariable("c1"), ContinuousVariable("c2")] class_vars = [ContinuousVariable("cls1"), ContinuousVariable("cls2")] domain = Domain(attrs, class_vars) table = Table(domain, np.arange(8).reshape((4, 2)), np.arange(8).reshape((4, 2))) att = [ContinuousVariable("Feature 1"), ContinuousVariable("Feature 2"), ContinuousVariable("Feature 3"), ContinuousVariable("Feature 4")] att[0].attributes = {"cls1": "0", "cls2": "1"} att[1].attributes = {"cls1": "2", "cls2": "3"} att[2].attributes = {"cls1": "4", "cls2": "5"} att[3].attributes = {"cls1": "6", "cls2": "7"} domain = Domain(att, metas=[StringVariable("Feature name")]) result = Table(domain, np.arange(8).reshape((4, 2)).T, metas=np.array(["c1", "c2"])[:, None]) # transpose and compare self._compare_tables(result, Table.transpose(table)) # transpose of transpose t = Table.transpose(Table.transpose(table), "Feature name") self._compare_tables(table, t) # original should not change self.assertDictEqual(table.domain.attributes[0].attributes, {}) def test_transpose_metas(self): attrs = [ContinuousVariable("c1"), ContinuousVariable("c2")] metas = [StringVariable("m1")] domain = Domain(attrs, metas=metas) X = np.arange(8).reshape((4, 2)) M = np.array(["aa", "bb", "cc", "dd"])[:, None] table = Table(domain, X, metas=M) att = [ContinuousVariable("Feature 1"), ContinuousVariable("Feature 2"), ContinuousVariable("Feature 3"), ContinuousVariable("Feature 4")] att[0].attributes = {"m1": "aa"} att[1].attributes = {"m1": "bb"} att[2].attributes = {"m1": "cc"} att[3].attributes = {"m1": "dd"} domain = Domain(att, metas=[StringVariable("Feature name")]) result = Table(domain, np.arange(8).reshape((4, 2)).T, metas=np.array(["c1", "c2"])[:, None]) # transpose and compare self._compare_tables(result, Table.transpose(table)) # transpose of transpose t = Table.transpose(Table.transpose(table), "Feature name") self._compare_tables(table, t) # original should not change self.assertDictEqual(table.domain.attributes[0].attributes, {}) def test_transpose_discrete_metas(self): attrs = [ContinuousVariable("c1"), ContinuousVariable("c2")] metas = [DiscreteVariable("m1", values=("aa", "bb"))] domain = Domain(attrs, metas=metas) X = np.arange(8).reshape((4, 2)) M = np.array([0, 1, 0, 1])[:, None] table = Table(domain, X, metas=M) att = [ContinuousVariable("Feature 1"), ContinuousVariable("Feature 2"), ContinuousVariable("Feature 3"), ContinuousVariable("Feature 4")] att[0].attributes = {"m1": "aa"} att[1].attributes = {"m1": "bb"} att[2].attributes = {"m1": "aa"} att[3].attributes = {"m1": "bb"} domain = Domain(att, metas=[StringVariable("Feature name")]) result = Table(domain, np.arange(8).reshape((4, 2)).T, metas=np.array(["c1", "c2"])[:, None]) # transpose and compare self._compare_tables(result, Table.transpose(table)) # transpose of transpose is original t = Table.transpose(Table.transpose(table), "Feature name") self._compare_tables(table, t) # original should not change self.assertDictEqual(table.domain.attributes[0].attributes, {}) def test_transpose_continuous_metas(self): attrs = [ContinuousVariable("c1"), ContinuousVariable("c2")] metas = [ContinuousVariable("m1")] domain = Domain(attrs, metas=metas) X = np.arange(8).reshape((4, 2)) M = np.array([0.0, 1.0, 0.0, 1.0])[:, None] table = Table(domain, X, metas=M) att = [ContinuousVariable("Feature 1"), ContinuousVariable("Feature 2"), ContinuousVariable("Feature 3"), ContinuousVariable("Feature 4")] att[0].attributes = {"m1": "0"} att[1].attributes = {"m1": "1"} att[2].attributes = {"m1": "0"} att[3].attributes = {"m1": "1"} domain = Domain(att, metas=[StringVariable("Feature name")]) result = Table(domain, np.arange(8).reshape((4, 2)).T, metas=np.array(["c1", "c2"])[:, None]) # transpose and compare self._compare_tables(result, Table.transpose(table)) # transpose of transpose is original t = Table.transpose(Table.transpose(table), "Feature name") self._compare_tables(table, t) # original should not change self.assertDictEqual(table.domain.attributes[0].attributes, {}) def test_transpose_missing_metas(self): attrs = [ContinuousVariable("c1"), ContinuousVariable("c2")] metas = [StringVariable("m1")] domain = Domain(attrs, metas=metas) X = np.arange(8).reshape((4, 2)) M = np.array(["aa", "bb", "", "dd"])[:, None] table = Table(domain, X, metas=M) att = [ContinuousVariable("Feature 1"), ContinuousVariable("Feature 2"), ContinuousVariable("Feature 3"), ContinuousVariable("Feature 4")] att[0].attributes = {"m1": "aa"} att[1].attributes = {"m1": "bb"} att[3].attributes = {"m1": "dd"} domain = Domain(att, metas=[StringVariable("Feature name")]) result = Table(domain, np.arange(8).reshape((4, 2)).T, metas=np.array(["c1", "c2"])[:, None]) # transpose and compare self._compare_tables(result, Table.transpose(table)) # transpose of transpose t = Table.transpose(Table.transpose(table), "Feature name") self._compare_tables(table, t) # original should not change self.assertDictEqual(table.domain.attributes[0].attributes, {}) def test_transpose_multiple_metas(self): attrs = [ContinuousVariable("c1"), ContinuousVariable("c2")] metas = [StringVariable("m1"), StringVariable("m2")] domain = Domain(attrs, metas=metas) X = np.arange(8).reshape((4, 2)) M = np.array([["aa", "aaa"], ["bb", "bbb"], ["cc", "ccc"], ["dd", "ddd"]]) table = Table(domain, X, metas=M) att = [ContinuousVariable("Feature 1"), ContinuousVariable("Feature 2"), ContinuousVariable("Feature 3"), ContinuousVariable("Feature 4")] att[0].attributes = {"m1": "aa", "m2": "aaa"} att[1].attributes = {"m1": "bb", "m2": "bbb"} att[2].attributes = {"m1": "cc", "m2": "ccc"} att[3].attributes = {"m1": "dd", "m2": "ddd"} domain = Domain(att, metas=[StringVariable("Feature name")]) result = Table(domain, np.arange(8).reshape((4, 2)).T, metas=np.array(["c1", "c2"])[:, None]) # transpose and compare self._compare_tables(result, Table.transpose(table)) # transpose of transpose t = Table.transpose(Table.transpose(table), "Feature name") self._compare_tables(table, t) # original should not change self.assertDictEqual(table.domain.attributes[0].attributes, {}) def test_transpose_class_and_metas(self): attrs = [ContinuousVariable("c1"), ContinuousVariable("c2")] metas = [StringVariable("m1"), StringVariable("m2")] domain = Domain(attrs, [ContinuousVariable("cls")], metas) M = np.array([["aa", "aaa"], ["bb", "bbb"], ["cc", "ccc"], ["dd", "ddd"]]) table = Table(domain, np.arange(8).reshape((4, 2)), np.arange(1, 5), M) att = [ContinuousVariable("Feature 1"), ContinuousVariable("Feature 2"), ContinuousVariable("Feature 3"), ContinuousVariable("Feature 4")] att[0].attributes = {"cls": "1", "m1": "aa", "m2": "aaa"} att[1].attributes = {"cls": "2", "m1": "bb", "m2": "bbb"} att[2].attributes = {"cls": "3", "m1": "cc", "m2": "ccc"} att[3].attributes = {"cls": "4", "m1": "dd", "m2": "ddd"} domain = Domain(att, metas=[StringVariable("Feature name")]) result = Table(domain, np.arange(8).reshape((4, 2)).T, metas=np.array(["c1", "c2"])[:, None]) # transpose and compare self._compare_tables(result, Table.transpose(table)) # transpose of transpose t = Table.transpose(Table.transpose(table), "Feature name") self._compare_tables(table, t) # original should not change self.assertDictEqual(table.domain.attributes[0].attributes, {}) def test_transpose_attributes_of_attributes_discrete(self): attrs = [ContinuousVariable("c1"), ContinuousVariable("c2")] attrs[0].attributes = {"attr1": "a", "attr2": "aa"} attrs[1].attributes = {"attr1": "b", "attr2": "bb"} domain = Domain(attrs) table = Table(domain, np.arange(8).reshape((4, 2))) att = [ContinuousVariable("Feature 1"), ContinuousVariable("Feature 2"), ContinuousVariable("Feature 3"), ContinuousVariable("Feature 4")] metas = [StringVariable("Feature name"), DiscreteVariable("attr1", values=("a", "b")), DiscreteVariable("attr2", values=("aa", "bb"))] domain = Domain(att, metas=metas) M = np.array([["c1", 0.0, 0.0], ["c2", 1.0, 1.0]], dtype=object) result = Table(domain, np.arange(8).reshape((4, 2)).T, metas=M) # transpose and compare self._compare_tables(result, Table.transpose(table)) # transpose of transpose t = Table.transpose(Table.transpose(table), "Feature name") self._compare_tables(table, t) # original should not change self.assertDictEqual(table.domain.attributes[0].attributes, {"attr1": "a", "attr2": "aa"}) def test_transpose_attributes_of_attributes_continuous(self): attrs = [ContinuousVariable("c1"), ContinuousVariable("c2")] attrs[0].attributes = {"attr1": "1.1", "attr2": "1.3"} attrs[1].attributes = {"attr1": "2.2", "attr2": "2.3"} domain = Domain(attrs) table = Table(domain, np.arange(8).reshape((4, 2))) att = [ContinuousVariable("Feature 1"), ContinuousVariable("Feature 2"), ContinuousVariable("Feature 3"), ContinuousVariable("Feature 4")] metas = [StringVariable("Feature name"), ContinuousVariable("attr1"), ContinuousVariable("attr2")] domain = Domain(att, metas=metas) result = Table(domain, np.arange(8).reshape((4, 2)).T, metas=np.array([["c1", 1.1, 1.3], ["c2", 2.2, 2.3]], dtype=object)) # transpose and compare self._compare_tables(result, Table.transpose(table)) # transpose of transpose t = Table.transpose(Table.transpose(table), "Feature name") self._compare_tables(table, t) # original should not change self.assertDictEqual(table.domain.attributes[0].attributes, {"attr1": "1.1", "attr2": "1.3"}) def test_transpose_attributes_of_attributes_missings(self): attrs = [ContinuousVariable("c1"), ContinuousVariable("c2")] attrs[0].attributes = {"attr1": "a", "attr2": "aa"} attrs[1].attributes = {"attr1": "b"} domain = Domain(attrs) table = Table(domain, np.arange(8).reshape((4, 2))) att = [ContinuousVariable("Feature 1"), ContinuousVariable("Feature 2"), ContinuousVariable("Feature 3"), ContinuousVariable("Feature 4")] metas = [StringVariable("Feature name"), DiscreteVariable("attr1", values=("a", "b")), DiscreteVariable("attr2", values=("aa",))] domain = Domain(att, metas=metas) M = np.array([["c1", 0.0, 0.0], ["c2", 1.0, np.nan]], dtype=object) result = Table(domain, np.arange(8).reshape((4, 2)).T, metas=M) # transpose and compare self._compare_tables(result, Table.transpose(table)) # transpose of transpose t = Table.transpose(Table.transpose(table), "Feature name") self._compare_tables(table, t) # original should not change self.assertDictEqual(table.domain.attributes[0].attributes, {"attr1": "a", "attr2": "aa"}) def test_transpose_class_metas_attributes(self): attrs = [ContinuousVariable("c1"), ContinuousVariable("c2")] attrs[0].attributes = {"attr1": "a1", "attr2": "aa1"} attrs[1].attributes = {"attr1": "b1", "attr2": "bb1"} metas = [StringVariable("m1"), StringVariable("m2")] domain = Domain(attrs, [ContinuousVariable("cls")], metas) M = np.array([["aa", "aaa"], ["bb", "bbb"], ["cc", "ccc"], ["dd", "ddd"]]) table = Table(domain, np.arange(8).reshape((4, 2)), np.arange(1, 5), M) att = [ContinuousVariable("Feature 1"), ContinuousVariable("Feature 2"), ContinuousVariable("Feature 3"), ContinuousVariable("Feature 4")] att[0].attributes = {"cls": "1", "m1": "aa", "m2": "aaa"} att[1].attributes = {"cls": "2", "m1": "bb", "m2": "bbb"} att[2].attributes = {"cls": "3", "m1": "cc", "m2": "ccc"} att[3].attributes = {"cls": "4", "m1": "dd", "m2": "ddd"} metas = [StringVariable("Feature name"), DiscreteVariable("attr1", values=("a1", "b1")), DiscreteVariable("attr2", values=("aa1", "bb1"))] domain = Domain(att, metas=metas) M = np.array([["c1", 0.0, 0.0], ["c2", 1.0, 1.0]], dtype=object) result = Table(domain, np.arange(8).reshape((4, 2)).T, metas=M) # transpose and compare self._compare_tables(result, Table.transpose(table)) # transpose of transpose t = Table.transpose(Table.transpose(table), "Feature name") self._compare_tables(table, t) # original should not change self.assertDictEqual(table.domain.attributes[0].attributes, {"attr1": "a1", "attr2": "aa1"}) def test_transpose_duplicate_feature_names(self): table = Table("iris") domain = table.domain attrs, metas = domain.attributes[:3], domain.attributes[3:] table = table.transform(Domain(attrs, domain.class_vars, metas)) transposed = Table.transpose(table, domain.attributes[3].name) names = [f.name for f in transposed.domain.attributes] self.assertEqual(len(names), len(set(names))) def test_transpose(self): zoo = Table("zoo") t1 = Table.transpose(zoo) t2 = Table.transpose(t1, "Feature name") t3 = Table.transpose(t2) self._compare_tables(zoo, t2) self._compare_tables(t1, t3) def test_transpose_callback(self): zoo = Table("zoo") cb = Mock() Table.transpose(zoo, progress_callback=cb) cb.assert_called() def test_transpose_no_class_remove_inst(self): attrs = [ContinuousVariable("c1"), ContinuousVariable("c2")] table = Table(Domain(attrs), np.arange(8).reshape((4, 2))) att = [ContinuousVariable("1"), ContinuousVariable("3"), ContinuousVariable("5"), ContinuousVariable("7")] domain = Domain(att, metas=[StringVariable("Feature name")]) result = Table(domain, np.array([[0, 2, 4, 6]]), metas=np.array([["c1"]])) # transpose and compare transposed = Table.transpose(table, "c2", remove_redundant_inst=True) self._compare_tables(result, transposed) # transpose of transpose is not equal to the original Table.transpose(transposed, "Feature name") Table.transpose(transposed, "3", remove_redundant_inst=True) # original should not change self.assertDictEqual(table.domain.attributes[0].attributes, {}) def test_transpose_discrete_class_remove_inst(self): attrs = [ContinuousVariable("c1"), ContinuousVariable("c2")] domain = Domain(attrs, [DiscreteVariable("cls", values=("a", "b"))]) table = Table(domain, np.arange(8).reshape((4, 2)), np.array([1, 1, 0, 0])) att = [ContinuousVariable("1"), ContinuousVariable("3"), ContinuousVariable("5"), ContinuousVariable("7")] att[0].attributes = {"cls": "b"} att[1].attributes = {"cls": "b"} att[2].attributes = {"cls": "a"} att[3].attributes = {"cls": "a"} domain = Domain(att, metas=[StringVariable("Feature name")]) result = Table(domain, np.array([[0, 2, 4, 6]]), metas=np.array([["c1"]])) # transpose and compare transposed = Table.transpose(table, "c2", remove_redundant_inst=True) self._compare_tables(result, transposed) # transpose of transpose is not equal to the original Table.transpose(transposed, "Feature name") Table.transpose(transposed, "3", remove_redundant_inst=True) # original should not change self.assertDictEqual(table.domain.attributes[0].attributes, {}) def test_transpose_continuous_class_remove_inst(self): attrs = [ContinuousVariable("c1"), ContinuousVariable("c2")] domain = Domain(attrs, [ContinuousVariable("cls")]) table = Table(domain, np.arange(8).reshape((4, 2)), np.arange(4, 0, -1)) att = [ContinuousVariable("1"), ContinuousVariable("3"), ContinuousVariable("5"), ContinuousVariable("7")] att[0].attributes = {"cls": "4"} att[1].attributes = {"cls": "3"} att[2].attributes = {"cls": "2"} att[3].attributes = {"cls": "1"} domain = Domain(att, metas=[StringVariable("Feature name")]) result = Table(domain, np.array([[0, 2, 4, 6]]), metas=np.array([["c1"]])) # transpose and compare transposed = Table.transpose(table, "c2", remove_redundant_inst=True) self._compare_tables(result, transposed) # transpose of transpose is not equal to the original Table.transpose(transposed, "Feature name") Table.transpose(transposed, "3", remove_redundant_inst=True) # original should not change self.assertDictEqual(table.domain.attributes[0].attributes, {}) def test_transpose_missing_class_remove_inst(self): attrs = [ContinuousVariable("c1"), ContinuousVariable("c2")] domain = Domain(attrs, [ContinuousVariable("cls")]) table = Table(domain, np.arange(8).reshape((4, 2)), np.array([np.nan, 3, 2, 1])) att = [ContinuousVariable("1"), ContinuousVariable("3"), ContinuousVariable("5"), ContinuousVariable("7")] att[1].attributes = {"cls": "3"} att[2].attributes = {"cls": "2"} att[3].attributes = {"cls": "1"} domain = Domain(att, metas=[StringVariable("Feature name")]) result = Table(domain, np.array([[0, 2, 4, 6]]), metas=np.array([["c1"]])) # transpose and compare transposed = Table.transpose(table, "c2", remove_redundant_inst=True) self._compare_tables(result, transposed) # transpose of transpose is not equal to the original Table.transpose(transposed, "Feature name") Table.transpose(transposed, "3", remove_redundant_inst=True) # original should not change self.assertDictEqual(table.domain.attributes[0].attributes, {}) def test_transpose_multiple_class_remove_inst(self): attrs = [ContinuousVariable("c1"), ContinuousVariable("c2")] class_vars = [ContinuousVariable("cls1"), ContinuousVariable("cls2")] domain = Domain(attrs, class_vars) table = Table(domain, np.arange(8).reshape((4, 2)), np.arange(8).reshape((4, 2))) att = [ContinuousVariable("1"), ContinuousVariable("3"), ContinuousVariable("5"), ContinuousVariable("7")] att[0].attributes = {"cls1": "0", "cls2": "1"} att[1].attributes = {"cls1": "2", "cls2": "3"} att[2].attributes = {"cls1": "4", "cls2": "5"} att[3].attributes = {"cls1": "6", "cls2": "7"} domain = Domain(att, metas=[StringVariable("Feature name")]) result = Table(domain, np.array([[0, 2, 4, 6]]), metas=np.array([["c1"]])) # transpose and compare transposed = Table.transpose(table, "c2", remove_redundant_inst=True) self._compare_tables(result, transposed) # transpose of transpose is not equal to the original Table.transpose(transposed, "Feature name") Table.transpose(transposed, "3", remove_redundant_inst=True) # original should not change self.assertDictEqual(table.domain.attributes[0].attributes, {}) def test_transpose_metas_remove_inst(self): attrs = [ContinuousVariable("c1"), ContinuousVariable("c2")] metas = [StringVariable("m1")] domain = Domain(attrs, metas=metas) X = np.arange(8).reshape((4, 2)) M = np.array(["aa", "bb", "cc", "dd"])[:, None] table = Table(domain, X, metas=M) att = [ContinuousVariable("1"), ContinuousVariable("3"), ContinuousVariable("5"), ContinuousVariable("7")] att[0].attributes = {"m1": "aa"} att[1].attributes = {"m1": "bb"} att[2].attributes = {"m1": "cc"} att[3].attributes = {"m1": "dd"} domain = Domain(att, metas=[StringVariable("Feature name")]) result = Table(domain, np.array([[0, 2, 4, 6]]), metas=np.array([["c1"]])) # transpose and compare transposed = Table.transpose(table, "c2", remove_redundant_inst=True) self._compare_tables(result, transposed) # transpose of transpose is not equal to the original Table.transpose(transposed, "Feature name") Table.transpose(transposed, "3", remove_redundant_inst=True) # original should not change self.assertDictEqual(table.domain.attributes[0].attributes, {}) def test_transpose_discrete_metas_remove_inst(self): attrs = [ContinuousVariable("c1"), ContinuousVariable("c2")] metas = [DiscreteVariable("m1", values=("aa", "bb"))] domain = Domain(attrs, metas=metas) X = np.arange(8).reshape((4, 2)) M = np.array([0, 1, 0, 1])[:, None] table = Table(domain, X, metas=M) att = [ContinuousVariable("1"), ContinuousVariable("3"), ContinuousVariable("5"), ContinuousVariable("7")] att[0].attributes = {"m1": "aa"} att[1].attributes = {"m1": "bb"} att[2].attributes = {"m1": "aa"} att[3].attributes = {"m1": "bb"} domain = Domain(att, metas=[StringVariable("Feature name")]) result = Table(domain, np.array([[0, 2, 4, 6]]), metas=np.array([["c1"]])) # transpose and compare transposed = Table.transpose(table, "c2", remove_redundant_inst=True) self._compare_tables(result, transposed) # transpose of transpose is not equal to the original Table.transpose(transposed, "Feature name") Table.transpose(transposed, "3", remove_redundant_inst=True) # original should not change self.assertDictEqual(table.domain.attributes[0].attributes, {}) def test_transpose_continuous_metas_remove_inst(self): attrs = [ContinuousVariable("c1"), ContinuousVariable("c2")] metas = [ContinuousVariable("m1")] domain = Domain(attrs, metas=metas) X = np.arange(8).reshape((4, 2)) M = np.array([0.0, 1.0, 0.0, 1.0])[:, None] table = Table(domain, X, metas=M) att = [ContinuousVariable("1"), ContinuousVariable("3"), ContinuousVariable("5"), ContinuousVariable("7")] att[0].attributes = {"m1": "0"} att[1].attributes = {"m1": "1"} att[2].attributes = {"m1": "0"} att[3].attributes = {"m1": "1"} domain = Domain(att, metas=[StringVariable("Feature name")]) result = Table(domain, np.array([[0, 2, 4, 6]]), metas=np.array([["c1"]])) # transpose and compare transposed = Table.transpose(table, "c2", remove_redundant_inst=True) self._compare_tables(result, transposed) # transpose of transpose is not equal to the original Table.transpose(transposed, "Feature name") Table.transpose(transposed, "3", remove_redundant_inst=True) # original should not change self.assertDictEqual(table.domain.attributes[0].attributes, {}) def test_transpose_missing_metas_remove_inst(self): attrs = [ContinuousVariable("c1"), ContinuousVariable("c2")] metas = [StringVariable("m1")] domain = Domain(attrs, metas=metas) X = np.arange(8).reshape((4, 2)) M = np.array(["aa", "bb", "", "dd"])[:, None] table = Table(domain, X, metas=M) att = [ContinuousVariable("1"), ContinuousVariable("3"), ContinuousVariable("5"), ContinuousVariable("7")] att[0].attributes = {"m1": "aa"} att[1].attributes = {"m1": "bb"} att[3].attributes = {"m1": "dd"} domain = Domain(att, metas=[StringVariable("Feature name")]) result = Table(domain, np.array([[0, 2, 4, 6]]), metas=np.array([["c1"]])) # transpose and compare transposed = Table.transpose(table, "c2", remove_redundant_inst=True) self._compare_tables(result, transposed) # transpose of transpose is not equal to the original Table.transpose(transposed, "Feature name") Table.transpose(transposed, "3", remove_redundant_inst=True) # original should not change self.assertDictEqual(table.domain.attributes[0].attributes, {}) def test_transpose_multiple_metas_remove_inst(self): attrs = [ContinuousVariable("c1"), ContinuousVariable("c2")] metas = [StringVariable("m1"), StringVariable("m2")] domain = Domain(attrs, metas=metas) X = np.arange(8).reshape((4, 2)) M = np.array([["aa", "aaa"], ["bb", "bbb"], ["cc", "ccc"], ["dd", "ddd"]]) table = Table(domain, X, metas=M) att = [ContinuousVariable("1"), ContinuousVariable("3"), ContinuousVariable("5"), ContinuousVariable("7")] att[0].attributes = {"m1": "aa", "m2": "aaa"} att[1].attributes = {"m1": "bb", "m2": "bbb"} att[2].attributes = {"m1": "cc", "m2": "ccc"} att[3].attributes = {"m1": "dd", "m2": "ddd"} domain = Domain(att, metas=[StringVariable("Feature name")]) result = Table(domain, np.array([[0, 2, 4, 6]]), metas=np.array([["c1"]])) # transpose and compare transposed = Table.transpose(table, "c2", remove_redundant_inst=True) self._compare_tables(result, transposed) # transpose of transpose is not equal to the original Table.transpose(transposed, "Feature name") Table.transpose(transposed, "3", remove_redundant_inst=True) # original should not change self.assertDictEqual(table.domain.attributes[0].attributes, {}) def test_transpose_class_and_metas_remove_inst(self): attrs = [ContinuousVariable("c1"), ContinuousVariable("c2")] metas = [StringVariable("m1"), StringVariable("m2")] domain = Domain(attrs, [ContinuousVariable("cls")], metas) M = np.array([["aa", "aaa"], ["bb", "bbb"], ["cc", "ccc"], ["dd", "ddd"]]) table = Table(domain, np.arange(8).reshape((4, 2)), np.arange(1, 5), M) att = [ContinuousVariable("1"), ContinuousVariable("3"), ContinuousVariable("5"), ContinuousVariable("7")] att[0].attributes = {"cls": "1", "m1": "aa", "m2": "aaa"} att[1].attributes = {"cls": "2", "m1": "bb", "m2": "bbb"} att[2].attributes = {"cls": "3", "m1": "cc", "m2": "ccc"} att[3].attributes = {"cls": "4", "m1": "dd", "m2": "ddd"} domain = Domain(att, metas=[StringVariable("Feature name")]) result = Table(domain, np.array([[0, 2, 4, 6]]), metas=np.array([["c1"]])) # transpose and compare transposed = Table.transpose(table, "c2", remove_redundant_inst=True) self._compare_tables(result, transposed) # transpose of transpose is not equal to the original Table.transpose(transposed, "Feature name") Table.transpose(transposed, "3", remove_redundant_inst=True) # original should not change self.assertDictEqual(table.domain.attributes[0].attributes, {}) def test_transpose_attributes_of_attributes_discrete_remove_inst(self): attrs = [ContinuousVariable("c1"), ContinuousVariable("c2")] attrs[0].attributes = {"attr1": "a", "attr2": "aa"} attrs[1].attributes = {"attr1": "b", "attr2": "bb"} domain = Domain(attrs) table = Table(domain, np.arange(8).reshape((4, 2))) att = [ContinuousVariable("1"), ContinuousVariable("3"), ContinuousVariable("5"), ContinuousVariable("7")] metas = [StringVariable("Feature name"), DiscreteVariable("attr1", values=("a", "b")), DiscreteVariable("attr2", values=("aa", "bb"))] domain = Domain(att, metas=metas) result = Table(domain, np.array([[0, 2, 4, 6]]), metas=np.array([["c1", 0.0, 0.0]], dtype=object)) # transpose and compare transposed = Table.transpose(table, "c2", remove_redundant_inst=True) self._compare_tables(result, transposed) # transpose of transpose is not equal to the original Table.transpose(transposed, "Feature name") Table.transpose(transposed, "3", remove_redundant_inst=True) # original should not change self.assertDictEqual(table.domain.attributes[0].attributes, {"attr1": "a", "attr2": "aa"}) def test_transpose_attributes_of_attributes_continuous_remove_inst(self): attrs = [ContinuousVariable("c1"), ContinuousVariable("c2")] attrs[0].attributes = {"attr1": "1.1", "attr2": "1.3"} attrs[1].attributes = {"attr1": "2.2", "attr2": "2.3"} domain = Domain(attrs) table = Table(domain, np.arange(8).reshape((4, 2))) att = [ContinuousVariable("1"), ContinuousVariable("3"), ContinuousVariable("5"), ContinuousVariable("7")] metas = [StringVariable("Feature name"), ContinuousVariable("attr1"), ContinuousVariable("attr2")] domain = Domain(att, metas=metas) result = Table(domain, np.array([[0, 2, 4, 6]]), metas=np.array([["c1", 1.1, 1.3]], dtype=object)) # transpose and compare transposed = Table.transpose(table, "c2", remove_redundant_inst=True) self._compare_tables(result, transposed) # transpose of transpose is not equal to the original Table.transpose(transposed, "Feature name") Table.transpose(transposed, "3", remove_redundant_inst=True) # original should not change self.assertDictEqual(table.domain.attributes[0].attributes, {"attr1": "1.1", "attr2": "1.3"}) def test_transpose_attributes_of_attributes_missings_remove_inst(self): attrs = [ContinuousVariable("c1"), ContinuousVariable("c2")] attrs[0].attributes = {"attr1": "a", "attr2": "aa"} attrs[1].attributes = {"attr1": "b"} domain = Domain(attrs) table = Table(domain, np.arange(8).reshape((4, 2))) att = [ContinuousVariable("0"), ContinuousVariable("2"), ContinuousVariable("4"), ContinuousVariable("6")] metas = [StringVariable("Feature name"), DiscreteVariable("attr1", values=("b",))] domain = Domain(att, metas=metas) result = Table(domain, np.array([[1, 3, 5, 7]]), metas=np.array([["c2", 0.0]], dtype=object)) # transpose and compare transposed = Table.transpose(table, "c1", remove_redundant_inst=True) self._compare_tables(result, transposed) # transpose of transpose is not equal to the original Table.transpose(transposed, "Feature name") Table.transpose(transposed, "2", remove_redundant_inst=True) # original should not change self.assertDictEqual(table.domain.attributes[0].attributes, {"attr1": "a", "attr2": "aa"}) def test_transpose_class_metas_attributes_remove_inst(self): attrs = [ContinuousVariable("c1"), ContinuousVariable("c2")] attrs[0].attributes = {"attr1": "a1", "attr2": "aa1"} attrs[1].attributes = {"attr1": "b1", "attr2": "bb1"} metas = [StringVariable("m1"), StringVariable("m2")] domain = Domain(attrs, [ContinuousVariable("cls")], metas) M = np.array([["aa", "aaa"], ["bb", "bbb"], ["cc", "ccc"], ["dd", "ddd"]]) table = Table(domain, np.arange(8).reshape((4, 2)), np.arange(1, 5), M) att = [ContinuousVariable("1"), ContinuousVariable("3"), ContinuousVariable("5"), ContinuousVariable("7")] att[0].attributes = {"cls": "1", "m1": "aa", "m2": "aaa"} att[1].attributes = {"cls": "2", "m1": "bb", "m2": "bbb"} att[2].attributes = {"cls": "3", "m1": "cc", "m2": "ccc"} att[3].attributes = {"cls": "4", "m1": "dd", "m2": "ddd"} metas = [StringVariable("Feature name"), DiscreteVariable("attr1", values=("a1", "b1")), DiscreteVariable("attr2", values=("aa1", "bb1"))] domain = Domain(att, metas=metas) result = Table(domain, np.array([[0, 2, 4, 6]]), metas=np.array([["c1", 0.0, 0.0]], dtype=object)) # transpose and compare transposed = Table.transpose(table, "c2", remove_redundant_inst=True) self._compare_tables(result, transposed) # transpose of transpose is not equal to the original Table.transpose(transposed, "Feature name") Table.transpose(transposed, "3", remove_redundant_inst=True) # original should not change self.assertDictEqual(table.domain.attributes[0].attributes, {"attr1": "a1", "attr2": "aa1"}) def test_transpose_name(self): table = Table("iris") transposed = Table.transpose(table) self.assertEqual(table.name, transposed.name) def _compare_tables(self, table1, table2): self.assertEqual(table1.n_rows, table2.n_rows) np.testing.assert_array_equal(table1.X, table2.X) np.testing.assert_array_equal(table1.Y, table2.Y) np.testing.assert_array_equal(table1.metas.astype(str), table2.metas.astype(str)) np.testing.assert_array_equal(table1.W, table2.W) self.assertEqual([(type(x), x.name, x.attributes) for x in table1.domain.attributes], [(type(x), x.name, x.attributes) for x in table2.domain.attributes]) self.assertEqual([(type(x), x.name, x.attributes) for x in table1.domain.class_vars], [(type(x), x.name, x.attributes) for x in table2.domain.class_vars]) self.assertEqual([(type(x), x.name, x.attributes) for x in table1.domain.metas], [(type(x), x.name, x.attributes) for x in table2.domain.metas]) class SparseCV: def __call__(self, data): return sp.csr_matrix((len(data), 1)) class TestTableSparseDense(unittest.TestCase): def setUp(self): self.iris = Table('iris') def test_sparse_dense_transformation(self): iris = Table('iris') iris_sparse = iris.to_sparse(sparse_attributes=True) self.assertTrue(sp.issparse(iris_sparse.X)) self.assertFalse(sp.issparse(iris_sparse.Y)) self.assertFalse(sp.issparse(iris_sparse.metas)) iris_sparse = iris.to_sparse(sparse_attributes=True, sparse_class=True) self.assertTrue(sp.issparse(iris_sparse.X)) self.assertFalse(sp.issparse(iris_sparse.Y)) self.assertFalse(sp.issparse(iris_sparse.metas)) dense_iris = iris_sparse.to_dense() self.assertFalse(sp.issparse(dense_iris.X)) self.assertFalse(sp.issparse(dense_iris.Y)) self.assertFalse(sp.issparse(dense_iris.metas)) def test_from_table_add_one_sparse_column(self): # add one sparse feature, should remain dense domain = self.iris.domain.copy() domain.attributes += ( ContinuousVariable('S1', compute_value=SparseCV(), sparse=True), ) d = self.iris.transform(domain) self.assertFalse(sp.issparse(d.X)) # try with indices that are not Ellipsis d = Table.from_table(domain, self.iris, row_indices=[0, 1, 2]) np.testing.assert_array_equal( d.X, [[5.1, 3.5, 1.4, 0.2, 0], [4.9, 3.0, 1.4, 0.2, 0], [4.7, 3.2, 1.3, 0.2, 0]] ) self.assertFalse(sp.issparse(d.X)) def test_from_table_add_lots_of_sparse_columns(self): n_attrs = len(self.iris.domain.attributes) # add 2*n_attrs+1 sparse feature, should became sparse domain = self.iris.domain.copy() domain.attributes += tuple( ContinuousVariable('S' + str(i), compute_value=SparseCV(), sparse=True) for i in range(2*n_attrs + 1) ) d = self.iris.transform(domain) self.assertTrue(sp.issparse(d.X)) def test_from_table_replace_attrs_with_sparse(self): # replace attrs with a sparse feature, should became sparse domain = self.iris.domain.copy() domain.attributes = ( ContinuousVariable('S1', compute_value=SparseCV(), sparse=True), ) d = self.iris.transform(domain) self.assertTrue(sp.issparse(d.X)) def test_from_table_sparse_metas(self): # replace metas with a sparse feature, should became sparse domain = self.iris.domain.copy() domain._metas = ( ContinuousVariable('S1', compute_value=SparseCV(), sparse=True), ) d = self.iris.transform(domain) self.assertTrue(sp.issparse(d.metas)) def test_from_table_sparse_metas_with_strings(self): # replace metas with text and 100 sparse features, should be dense domain = self.iris.domain.copy() domain._metas = (StringVariable('text'),) + tuple( ContinuousVariable('S' + str(i), compute_value=SparseCV(), sparse=True) for i in range(100) ) d = self.iris.transform(domain) self.assertFalse(sp.issparse(d.metas)) class ConcurrencyTests(unittest.TestCase): def test_from_table_non_blocking(self): iris = Table("iris")[:10] def slow_compute_value(d): sleep(0.1) return d.X[:, 0] ndom = Domain([ContinuousVariable("a", compute_value=slow_compute_value)]) def run_from_table(): Table.from_table(ndom, iris) start = time() threads = [] for _ in range(10): thread = Thread(target=run_from_table) thread.start() threads.append(thread) for t in threads: t.join() # if from_table was blocking these threads would need at least 0.1*10s duration = time() - start self.assertLess(duration, 0.5) class PreprocessComputeValue: def __init__(self, domain, callback): self.domain = domain self.callback = callback def __call__(self, data_): if self.callback: self.callback(data_) transformed = data_.transform(self.domain) return transformed.X[:, 0] * 2 class PreprocessShared: def __init__(self, domain, callback): self.domain = domain self.callback = callback def __call__(self, data_): if self.callback: self.callback(data_) transformed = data_.transform(self.domain) return transformed.X * 2 class PreprocessSharedComputeValue(SharedComputeValue): def __init__(self, col, callback, shared): super().__init__(compute_shared=shared) self.col = col self.callback = callback # pylint: disable=arguments-differ def compute(self, data_, shared_data): if self.callback: self.callback(data_) return shared_data[:, self.col] def preprocess_domain_single(domain, callback): """ Preprocess domain with single-source compute values. """ return Domain([ ContinuousVariable(name=at.name, compute_value=PreprocessComputeValue(Domain([at]), callback)) for at in domain.attributes]) def preprocess_domain_shared(domain, callback, callback_shared): """ Preprocess domain with shared compute values. """ shared = PreprocessShared(domain, callback_shared) return Domain([ ContinuousVariable(name=at.name, compute_value=PreprocessSharedComputeValue(i, callback, shared)) for i, at in enumerate(domain.attributes)]) def preprocess_domain_single_stupid(domain, callback): """ Preprocess domain with single-source compute values with stupid implementation: before applying it, instead of transforming just one column into the input domain, do a needless transform of the whole domain. """ return Domain([ ContinuousVariable(name=at.name, compute_value=PreprocessComputeValue(domain, callback)) for at in domain.attributes]) class EfficientTransformTests(unittest.TestCase): def setUp(self): self.iris = Table("iris")[:10] def test_simple(self): call_cv = Mock() d1 = preprocess_domain_single(self.iris.domain, call_cv) t = self.iris.transform(d1) self.assertEqual(4, call_cv.call_count) np.testing.assert_equal(t.X, self.iris.X * 2) def test_shared(self): call_cv = Mock() call_shared = Mock() d1 = preprocess_domain_shared(self.iris.domain, call_cv, call_shared) t = self.iris.transform(d1) self.assertEqual(4, call_cv.call_count) self.assertEqual(1, call_shared.call_count) np.testing.assert_equal(t.X, self.iris.X * 2) def test_simple_simple_shared(self): call_cv = Mock() d1 = preprocess_domain_single(self.iris.domain, call_cv) d2 = preprocess_domain_single(d1, call_cv) call_shared = Mock() d3 = preprocess_domain_shared(d2, call_cv, call_shared) t = self.iris.transform(d3) self.assertEqual(1, call_shared.call_count) self.assertEqual(12, call_cv.call_count) np.testing.assert_equal(t.X, self.iris.X * 2**3) def test_simple_simple_shared_simple(self): call_cv = Mock() d1 = preprocess_domain_single(self.iris.domain, call_cv) d2 = preprocess_domain_single(d1, call_cv) call_shared = Mock() d3 = preprocess_domain_shared(d2, call_cv, call_shared) d4 = preprocess_domain_single(d3, call_cv) t = self.iris.transform(d4) self.assertEqual(1, call_shared.call_count) self.assertEqual(16, call_cv.call_count) np.testing.assert_equal(t.X, self.iris.X * 2**4) def test_simple_simple_shared_simple_shared_simple(self): call_cv = Mock() d1 = preprocess_domain_single(self.iris.domain, call_cv) d2 = preprocess_domain_single(d1, call_cv) call_shared = Mock() d3 = preprocess_domain_shared(d2, call_cv, call_shared) d4 = preprocess_domain_single(d3, call_cv) d5 = preprocess_domain_shared(d4, call_cv, call_shared) d6 = preprocess_domain_single(d5, call_cv) t = self.iris.transform(d6) self.assertEqual(2, call_shared.call_count) self.assertEqual(24, call_cv.call_count) np.testing.assert_equal(t.X, self.iris.X * 2**6) def test_simple_simple_stupid(self): call_cv = Mock() d1 = preprocess_domain_single_stupid(self.iris.domain, call_cv) d2 = preprocess_domain_single_stupid(d1, call_cv) t = self.iris.transform(d2) self.assertEqual(8, call_cv.call_count) np.testing.assert_equal(t.X[:, 0], self.iris.X[:, 0] * 4) if __name__ == "__main__": unittest.main()