#!/usr/bin/env python #file test_parse.py __author__ = "Rob Knight" __copyright__ = "Copyright 2011, The QIIME Project" __credits__ = ["Rob Knight", "Justin Kuczynski", "Greg Caporaso",\ "Cathy Lozupone", "Jens Reeder", "Daniel McDonald"] #remember to add yourself __license__ = "GPL" __version__ = "1.4.0" __maintainer__ = "Greg Caporaso" __email__ = "gregcaporaso@gmail.com" __status__ = "Release" from numpy import array, nan from StringIO import StringIO from cogent.util.unit_test import TestCase,main from cogent.util.misc import remove_files from qiime.util import get_tmp_filename from qiime.parse import (group_by_field, group_by_fields, parse_distmat, parse_rarefaction_record, parse_rarefaction, parse_coords, parse_otu_table, make_envs_dict, fields_to_dict, parse_rarefaction_fname, parse_qiime_parameters, parse_qiime_config_files, parse_bootstrap_support, parse_sample_mapping, parse_distmat_to_dict, sample_mapping_to_otu_table, parse_taxonomy, parse_mapping_file, parse_metadata_state_descriptions, parse_rarefaction_data, parse_illumina_line, parse_qual_score, parse_qual_scores, QiimeParseError, parse_newick,parse_trflp,parse_taxa_summary_table, parse_prefs_file, parse_mapping_file_to_dict, mapping_file_to_dict, MinimalQualParser, parse_denoiser_mapping) class TopLevelTests(TestCase): """Tests of top-level functions""" def setUp(self): """define some top-level data""" self.l19_data = array([ [7,1,0,0,0,0,0,0,0], [4,2,0,0,0,1,0,0,0], [2,4,0,0,0,1,0,0,0], [1,7,0,0,0,0,0,0,0], [0,8,0,0,0,0,0,0,0], [0,7,1,0,0,0,0,0,0], [0,4,2,0,0,0,2,0,0], [0,2,4,0,0,0,1,0,0], [0,1,7,0,0,0,0,0,0], [0,0,8,0,0,0,0,0,0], [0,0,7,1,0,0,0,0,0], [0,0,4,2,0,0,0,3,0], [0,0,2,4,0,0,0,1,0], [0,0,1,7,0,0,0,0,0], [0,0,0,8,0,0,0,0,0], [0,0,0,7,1,0,0,0,0], [0,0,0,4,2,0,0,0,4], [0,0,0,2,4,0,0,0,1], [0,0,0,1,7,0,0,0,0] ]) self.l19_sample_names = ['sam1', 'sam2', 'sam3', 'sam4', 'sam5','sam6',\ 'sam7', 'sam8', 'sam9', 'sam_middle', 'sam11', 'sam12', 'sam13', \ 'sam14', 'sam15', 'sam16', 'sam17', 'sam18', 'sam19'] self.l19_taxon_names = ['tax1', 'tax2', 'tax3', 'tax4', 'endbigtaxon',\ 'tax6', 'tax7', 'tax8', 'tax9'] self.SampleMapping = ["OTU1\tsample1\t3", "OTU1\tsample3\t2", \ "OTU2\tsample1\t1", "OTU2\tsample2\t2"] self.SampleMapping2 = ["OTU1\tsample1", "OTU1\tsample3", \ "OTU2\tsample1", "OTU2\tsample2"] self.legacy_otu_table1 = legacy_otu_table1 self.otu_table1 = otu_table1 self.expected_lineages1 = expected_lineages1 self.taxa_summary1 = taxa_summary1 self.taxa_summary1_expected = taxa_summary1_expected self.otu_table1_floats=otu_table1_floats self.files_to_remove = [] self.denoiser_mapping1 = denoiser_mapping1.split('\n') def tearDown(self): remove_files(self.files_to_remove) def test_parse_taxa_summary_table(self): """ parse_taxa_summary_table functions as expected """ actual = parse_taxa_summary_table(self.taxa_summary1.split('\n')) self.assertEqual(actual[0],self.taxa_summary1_expected[0]) self.assertEqual(actual[1],self.taxa_summary1_expected[1]) self.assertEqual(actual[2],self.taxa_summary1_expected[2]) self.assertEqual(actual,self.taxa_summary1_expected) def test_parse_newick(self): """parse_newick correctly matches escaped tip names to otu ids """ # confirm that it works without escaped names t1 = ('((((tax7:0.1,tax3:0.2):.98,tax8:.3, tax4:.3):.4,' '((tax1:0.3, tax6:.09):0.43,tax2:0.4):0.5):.2,' '(tax9:0.3, endbigtaxon:.08));') expected1 = ['tax7','tax3','tax8','tax4','tax1', 'tax6','tax2','tax9','endbigtaxon'] self.assertEqual(set(parse_newick(t1).getTipNames()),set(expected1)) self.assertEqual(set([tip.Name for tip in parse_newick(t1).tips()]), set(expected1)) # throw some screwed up names in t2 = ('((((tax7:0.1,tax3:0.2):.98,tax8:.3, \'tax4\':.3):.4,' "(('ta_______ x1':0.3, tax6:.09):0.43,tax2:0.4):0.5):.2," '(tax9:0.3, endbigtaxon:.08));') expected2 = ['tax7','tax3','tax8','tax4','ta_______ x1', 'tax6','tax2','tax9','endbigtaxon'] self.assertEqual(set(parse_newick(t2).getTipNames()),set(expected2)) self.assertEqual(set([tip.Name for tip in parse_newick(t2).tips()]), set(expected2)) def test_parse_mapping_file(self): """parse_mapping_file functions as expected""" s1 = ['#sample\ta\tb', '#comment line to skip',\ 'x \t y \t z ', ' ', '#more skip', 'i\tj\tk'] exp = ([['x','y','z'],['i','j','k']],\ ['sample','a','b'],\ ['comment line to skip','more skip']) obs = parse_mapping_file(s1) self.assertEqual(obs, exp) # We don't currently support this, but we should soon... # # check that first non-comment, non-blank line is used as # # header # s1 = ['sample\ta\tb', '#comment line to skip',\ # 'x \t y \t z ', ' ', '#more skip', 'i\tj\tk'] # exp = ([['x','y','z'],['i','j','k']],\ # ['sample','a','b'],\ # ['comment line to skip','more skip']) # obs = parse_mapping_file(s1) # self.assertEqual(obs, exp) #check that we strip double quotes by default s2 = ['#sample\ta\tb', '#comment line to skip',\ '"x "\t" y "\t z ', ' ', '"#more skip"', 'i\t"j"\tk'] obs = parse_mapping_file(s2) self.assertEqual(obs, exp) def test_mapping_file_to_dict(self): """parse_mapping_file functions as expected""" s1 = ['#sample\ta\tb', '#comment line to skip',\ 'x \t y \t z ', ' ', '#more skip', 'i\tj\tk'] exp = ([['x','y','z'],['i','j','k']],\ ['sample','a','b'],\ ['comment line to skip','more skip']) mapres = parse_mapping_file(s1) # map_data, header, comments mapdict = mapping_file_to_dict(*mapres[:2]) expdict = {'x':{'a':'y','b':'z'}, 'i':{'a':'j','b':'k'}} self.assertEqual(mapdict, expdict) def test_parse_mapping_file_to_dict(self): """parse_mapping_file functions as expected""" s1 = ['#sample\ta\tb', '#comment line to skip',\ 'x \t y \t z ', ' ', '#more skip', 'i\tj\tk'] exp = ([['x','y','z'],['i','j','k']],\ ['sample','a','b'],\ ['comment line to skip','more skip']) mapdict, comments = parse_mapping_file_to_dict(s1) expdict = {'x':{'a':'y','b':'z'}, 'i':{'a':'j','b':'k'}} self.assertEqual(mapdict, expdict) self.assertEqual(comments, ['comment line to skip','more skip']) def test_parse_mapping_file_handles_filepath(self): """ parse_mapping_file handles being passed a mapping filepath """ fp = get_tmp_filename(prefix='test_parse_mapping_file', suffix='.txt') self.files_to_remove.append(fp) open(fp,'w').write('\n'.join(['#sample\ta\tb', '#comment line to skip', 'x \t y \t z ', ' ', '#more skip', 'i\tj\tk'])) obs = parse_mapping_file(fp) exp = ([['x','y','z'],['i','j','k']],\ ['sample','a','b'],\ ['comment line to skip','more skip']) self.assertEqual(obs, exp) def test_parse_mapping_file_handles_file_handle(self): """ parse_mapping_file handles being passed a mapping filepath """ fp = get_tmp_filename(prefix='test_parse_mapping_file', suffix='.txt') self.files_to_remove.append(fp) open(fp,'w').write('\n'.join(['#sample\ta\tb', '#comment line to skip', 'x \t y \t z ', ' ', '#more skip', 'i\tj\tk'])) obs = parse_mapping_file(open(fp)) exp = ([['x','y','z'],['i','j','k']],\ ['sample','a','b'],\ ['comment line to skip','more skip']) self.assertEqual(obs, exp) def test_parse_mapping_file_handles_errors(self): """parse_mapping_file handles bad mapping files""" # Empty file self.assertRaises(QiimeParseError, parse_mapping_file, []) # string self.assertRaises(QiimeParseError, parse_mapping_file, 'my_mapping_file.txt') # invalid format (no header line with leading # sign) self.assertRaises(QiimeParseError, parse_mapping_file, ['sampleID\ta\tb', '1\tf\t43', '2\tt\t44']) # invalid format (no non-header lines) self.assertRaises(QiimeParseError, parse_mapping_file, ['#sampleID\ta\tb']) # invalid format (no header line) self.assertRaises(QiimeParseError, parse_mapping_file, ['1\tf\t43', '2\tt\t44']) def test_parse_prefs_file(self): """parse_prefs_file should correctly eval prefs string. """ #Test good input ps1 = """{'bgcolor':'white','colors': {'id':'blue','name':'green'},'list':[1,2,3]}""" exp1 = {'bgcolor':'white','colors':{'id':'blue','name':'green'},\ 'list':[1,2,3]} self.assertEqual(parse_prefs_file(ps1),exp1) #Test bad input #list of valid input rather than multiline string should fail. ps_bad_1 = ["{'bgcolor':'white',", "'colors':{'id':'blue','name':'green'}",\ ",'list':[1,2,3]}"] self.assertRaises(QiimeParseError,parse_prefs_file,ps_bad_1) #bad data. Can be evaluated but not a dict. ps_bad_2 = "[1,2,3]" self.assertRaises(QiimeParseError,parse_prefs_file,ps_bad_2) def test_group_by_field(self): """group_by_field should group table by fields""" t = [ ['#sample', 'loc', 'age'], ['a','US','5'], ['b','US','10'], ['c','Mal','5'], ['d','Mal','10'], ['e','Ven','5'], ] self.assertEqual(group_by_field(t, 'loc'), \ {'US':['a','b'], 'Mal':['c','d'], 'Ven':['e']}) self.assertEqual(group_by_field(t, 'age'), \ {'5':['a','c','e'], '10':['b','d']}) def test_group_by_fields(self): """group_by_fields should group table by fields""" t = [ ['#sample', 'loc', 'age', 'mal'], ['a','US','5','n'], ['b','US','10','n'], ['c','Mal','5','y'], ['d','Mal','10','n'], ['e','Mal','5','y'], ] self.assertEqual(group_by_fields(t, ['age','loc']), \ {('5','US'):['a'], ('10','US'):['b'], ('5','Mal'):['c','e'], ('10','Mal'):['d']}) def test_parse_distmat(self): """parse_distmat should read distmat correctly""" lines = """\ta\tb\tc a\t0\t1\t2 b\t1\t0\t3.5 c\t1\t3.5\t0 """.splitlines() exp = (['a','b','c'], array([[0,1,2],[1,0,3.5],[1,3.5,0]])) obs = parse_distmat(lines) self.assertEqual(obs, exp) def test_parse_distmat_to_dict(self): """parse_distmat should return dict of distmat""" lines = """\ta\tb\tc a\t0\t1\t2 b\t1\t0\t3.5 c\t1\t3.5\t0 """.splitlines() exp = {'a': {'a': 0.0, 'c': 2.0, 'b': 1.0}, 'c': {'a': 1.0, 'c': 0.0, 'b': 3.5}, 'b': {'a': 1.0, 'c': 3.5, 'b': 0.0}} obs = parse_distmat_to_dict(lines) self.assertEqual(obs, exp) #should raise error because row and column headers don't match wrong_dist_mat ="""\ta\ty\tx a\t0\t1\t2 b\t1\t0\t3.5 c\t1\t3.5\t0 """.splitlines() self.failUnlessRaises(AssertionError, parse_distmat_to_dict, wrong_dist_mat) def test_parse_bootstrap_support(self): """parse_distmat should read distmat correctly""" input_txt = """#\ta\tb\tc. #more comments here node2\t0 17node\t0.11922 """ lines = input_txt.splitlines() exp = {'17node':0.11922, 'node2':0.00} obs = parse_bootstrap_support(lines) self.assertFloatEqual(obs, exp) def test_parse_rarefaction_data(self): self.data = {} self.data['headers'] = ['PD_whole_tree.txt', 'Antibiotics'] self.data['error'] = {'NA': [0.099969643842700004], \ 'Y': [0.105669693476, 1.08546135424, 1.5626248357999999], \ 'N': [0.101173002974]} self.data['options'] = ['Y', 'NA', 'N'] self.data['xaxis'] = [10.0, 310.0, 610.0, 910.0, 1210.0, 1510.0, \ 1810.0, 2110.0, 2410.0, 2710.0, 3010.0] self.data['series'] = {'NA': [0.88581050485400004], \ 'Y': [0.918845147059, 7.1758656176500004, 9.9186072941199992], \ 'N': [0.92636763785999998]} self.data['color'] = {'NA': '#00ff00', 'Y': '#ff0000', 'N': '#0000ff'} self.rarefaction_series_data = ['# PD_whole_tree.txt', '# Antibiotics', 'xaxis: 10.0\t310.0\t610.0\t910.0\t1210.0\t1510.0\t1810.0\t2110.0\ \t2410.0\t2710.0\t3010.0\t', 'xmax: 3310.0', '>> Y', 'color #ff0000', 'series 0.918845147059\t7.17586561765\t9.91860729412\t', 'error 0.105669693476\t1.08546135424\t1.5626248358\t', '>> NA', 'color #00ff00', 'series 0.885810504854\t', 'error 0.0999696438427\t', '>> N', 'color #0000ff', 'series 0.92636763786\t', 'error 0.101173002974' ] test = parse_rarefaction_data(self.rarefaction_series_data) self.assertEqual(test, self.data) def test_parse_rarefaction_record(self): self.rarefactionline1 = 'rare10.txt\t10\t0\t1.99181\t0.42877\t2.13996' test1 = parse_rarefaction_record(self.rarefactionline1) self.rarefactiondata1 = ('rare10.txt', [10.0, 0.0, \ 1.9918100000000001, 0.42876999999999998, 2.1399599999999999]) self.assertEqual(self.rarefactiondata1, test1) self.rarefactionline2 = 'rare10.txt\t10\t0\t1.99181\t0.42877\tNA' test2 = parse_rarefaction_record(self.rarefactionline2) self.rarefactiondata2 = ('rare10.txt', [10.0, 0.0, 1.9918100000000001,\ 0.42876999999999998, nan]) self.assertEqual(self.rarefactiondata2, test2) def test_parse_rarefaction_fname(self): """ parse_rarefaction_fname should return base, seqs/sam, iters, etc.""" fname = "alpha_rarefaction_900_3.txt" base, seqs, iter, ext = parse_rarefaction_fname(fname) self.assertEqual((base, seqs, iter, ext), ("alpha_rarefaction", 900, 3, ".txt")) def test_parse_rarefaction(self): self.rarefactionfile = ['\tsequences per sample\titeration\t123\t234\t345', 'rare10.txt\t10\t0\t1.99181\t0.42877\t2.13996', 'rare10.txt\t10\t1\t2.07163\t0.42877\t2.37055', 'rare310.txt\t310\t0\t8.83115\t0.42877\t11.00725', 'rare310.txt\t310\t1\t10.05242\t0.42877\t8.24474', 'rare610.txt\t610\t0\t12.03067\t0.42877\t11.58928', 'rare610.txt\t610\t1\t12.9862\t0.42877\t11.58642'] self.col_headers = ['', 'sequences per sample', 'iteration', '123', '234', '345'] self.comments = [] self.rarefaction_fns = ['rare10.txt', 'rare10.txt', 'rare310.txt', 'rare310.txt', 'rare610.txt', 'rare610.txt'] self.rarefaction_data = [[10.0, 0.0, 1.9918100000000001, 0.42876999999999998, 2.1399599999999999], [10.0, 1.0, 2.0716299999999999, 0.42876999999999998, 2.3705500000000002], [310.0, 0.0, 8.8311499999999992, 0.42876999999999998, 11.007250000000001], [310.0, 1.0, 10.05242, 0.42876999999999998, 8.2447400000000002], [610.0, 0.0, 12.030670000000001, 0.42876999999999998, 11.58928], [610.0, 1.0, 12.9862, 0.42876999999999998, 11.58642]] test_col_headers, test_comments, test_rarefaction_fns, test_rarefaction_data = parse_rarefaction(self.rarefactionfile) self.assertEqual(test_col_headers, self.col_headers) self.assertEqual(test_comments, self.comments) self.assertEqual(test_rarefaction_fns, self.rarefaction_fns) self.assertEqual(test_rarefaction_data, self.rarefaction_data) # def test_parse_rarefaction(self): # """parse_rarefaction should handle multiple recs""" # recs ="""#HEADER 97.0 NFkeyRightShift 1000 # #CHAO1 288.12903 241.27093 371.41733 # #ACE 294.74813 252.75930 361.30606 0.68654 # #SHANNON 3.71822 3.61146 3.82499 # #SIMPSON 0.08021 # #n rare rare_lci rare_hci # 1 1.000000 1.000000 1.000000 # 51 26.878000 26.032290 27.723710 # #HEADER 97.0 DMkeySpace 1000 # #CHAO1 90.20000 53.81120 196.98011 # #ACE 122.66234 68.48901 264.46888 1.53571 # #SHANNON 1.35156 1.12549 1.57762 # #SIMPSON 0.56002 # #n rare rare_lci rare_hci # 1 1.000000 1.000000 1.000000 # 51 10.707000 10.085986 11.328014 # 101 17.547000 17.046632 18.047368 # 151 23.410000 23.009805 23.810195 # #HEADER 97.0 RKkeyW 1000 # #CHAO1 251.57143 176.86438 401.83780 # #ACE 264.44294 192.23364 395.08515 0.89714 # #SHANNON 1.56521 1.44411 1.68630 # #SIMPSON 0.54772 # #n rare rare_lci rare_hci # 1 1.000000 1.000000 1.000000 # 51 11.739000 11.060658 12.417342 # 101 19.054000 18.458137 19.649863 # 151 24.994000 24.447174 25.540826 # """.splitlines() # obs = parse_rarefaction(recs) # self.assertEqual(set(obs.keys()), \ # set(['NFkeyRightShift','DMkeySpace','RKkeyW'])) def test_parse_coords(self): """parse_coords should handle coords file""" coords = """pc vector number\t1\t2\t3 A\t0.11\t0.09\t0.23 B\t0.03\t0.07\t-0.26 C\t0.12\t0.06\t-0.32 eigvals\t4.94\t1.79\t1.50 % variation explained\t14.3\t5.2\t4.3 """.splitlines() obs = parse_coords(coords) exp = (['A','B','C'], array([[.11,.09,.23],[.03,.07,-.26],[.12,.06,-.32]]), array([4.94,1.79,1.50]), array([14.3,5.2,4.3])) self.assertEqual(obs, exp) def test_parse_otu_table_legacy(self): """parse_otu_table functions as expected with legacy OTU table """ data = self.legacy_otu_table1 data_f = (data.split('\n')) obs = parse_otu_table(data_f) exp = (['Fing','Key','NA'], ['0','1','2','3','4'], array([[19111,44536,42],[1216,3500,6],[1803,1184,2], [1722,4903,17], [589,2074,34]]), self.expected_lineages1) self.assertEqual(obs, exp) def test_parse_otu_table(self): """parse_otu_table functions as expected with new-style OTU table """ data = self.otu_table1 data_f = (data.split('\n')) obs = parse_otu_table(data_f) exp = (['Fing','Key','NA'], ['0','1','2','3','4'], array([[19111,44536,42],[1216,3500,6],[1803,1184,2], [1722,4903,17], [589,2074,34]]), self.expected_lineages1) self.assertEqual(obs, exp) def test_parse_otu_table_floats_in_table(self): """parse_otu_table functions using an OTU table containing floats but cast as int....this will automatically cast into floats""" data = self.otu_table1_floats data_f = (data.split('\n')) obs = parse_otu_table(data_f) exp = (['Fing','Key','NA'], ['0','1','2','3','4'], array([[19111.0,44536.0,42.0],[1216.0,3500.0,6.0], [1803.0,1184.0,2.0],[1722.1,4903.2,17.0], [589.6,2074.4,34.5]]), self.expected_lineages1) self.assertEqual(obs, exp) def test_parse_otu_table_float_counts(self): """parse_otu_table should return correct result from small table""" data = """#Full OTU Counts #OTU ID Fing Key NA Consensus Lineage 0 19111 44536 42 Bacteria; Actinobacteria; Actinobacteridae; Propionibacterineae; Propionibacterium 1 1216 3500 6 Bacteria; Firmicutes; Alicyclobacillaceae; Bacilli; Lactobacillales; Lactobacillales; Streptococcaceae; Streptococcus 2 1803 1184 2 Bacteria; Actinobacteria; Actinobacteridae; Gordoniaceae; Corynebacteriaceae 3 1722 4903 17 Bacteria; Firmicutes; Alicyclobacillaceae; Bacilli; Staphylococcaceae 4 589 2074 34 Bacteria; Cyanobacteria; Chloroplasts; vectors""" data_f = (data.split('\n')) obs = parse_otu_table(data_f,count_map_f=float) exp = (['Fing','Key','NA'], ['0','1','2','3','4'], array([[19111.,44536.,42.],[1216.,3500.,6.],[1803.,1184.,2.],\ [1722.,4903.,17.], [589,2074.,34.]]), [['Bacteria','Actinobacteria','Actinobacteridae','Propionibacterineae','Propionibacterium'], ['Bacteria','Firmicutes','Alicyclobacillaceae','Bacilli','Lactobacillales','Lactobacillales','Streptococcaceae','Streptococcus'], ['Bacteria','Actinobacteria','Actinobacteridae','Gordoniaceae','Corynebacteriaceae'], ['Bacteria','Firmicutes','Alicyclobacillaceae','Bacilli','Staphylococcaceae'], ['Bacteria','Cyanobacteria','Chloroplasts','vectors']]) self.assertEqual(obs, exp) def test_parse_otu_table_file(self): """parse_otu_table should return correct result on fileio format object""" data = """#Full OTU Counts #OTU ID Fing Key NA Consensus Lineage 0 19111 44536 42 Bacteria; Actinobacteria; Actinobacteridae; Propionibacterineae; Propionibacterium 1 1216 3500 6 Bacteria; Firmicutes; Alicyclobacillaceae; Bacilli; Lactobacillales; Lactobacillales; Streptococcaceae; Streptococcus 2 1803 1184 2 Bacteria; Actinobacteria; Actinobacteridae; Gordoniaceae; Corynebacteriaceae 3 1722 4903 17 Bacteria; Firmicutes; Alicyclobacillaceae; Bacilli; Staphylococcaceae 4 589 2074 34 Bacteria; Cyanobacteria; Chloroplasts; vectors""" data_f = StringIO(data) obs = parse_otu_table(data_f) exp = (['Fing','Key','NA'], ['0','1','2','3','4'], array([[19111,44536,42],[1216,3500,6],[1803,1184,2],\ [1722,4903,17], [589,2074,34]]), [['Bacteria','Actinobacteria','Actinobacteridae','Propionibacterineae','Propionibacterium'], ['Bacteria','Firmicutes','Alicyclobacillaceae','Bacilli','Lactobacillales','Lactobacillales','Streptococcaceae','Streptococcus'], ['Bacteria','Actinobacteria','Actinobacteridae','Gordoniaceae','Corynebacteriaceae'], ['Bacteria','Firmicutes','Alicyclobacillaceae','Bacilli','Staphylococcaceae'], ['Bacteria','Cyanobacteria','Chloroplasts','vectors']]) self.assertEqual(obs, exp) def test_parse_otu_table_consensus_lineage(self): """parse_otu_table should accept 'consensusLineage'""" data = """#Full OTU Counts #OTU ID Fing Key NA consensusLineage 0 19111 44536 42 Bacteria; Actinobacteria; Actinobacteridae; Propionibacterineae; Propionibacterium 1 1216 3500 6 Bacteria; Firmicutes; Alicyclobacillaceae; Bacilli; Lactobacillales; Lactobacillales; Streptococcaceae; Streptococcus 2 1803 1184 2 Bacteria; Actinobacteria; Actinobacteridae; Gordoniaceae; Corynebacteriaceae 3 1722 4903 17 Bacteria; Firmicutes; Alicyclobacillaceae; Bacilli; Staphylococcaceae 4 589 2074 34 Bacteria; Cyanobacteria; Chloroplasts; vectors""" data_f = StringIO(data) obs = parse_otu_table(data_f) exp = (['Fing','Key','NA'], ['0','1','2','3','4'], array([[19111,44536,42],[1216,3500,6],[1803,1184,2],\ [1722,4903,17], [589,2074,34]]), [['Bacteria','Actinobacteria','Actinobacteridae','Propionibacterineae','Propionibacterium'], ['Bacteria','Firmicutes','Alicyclobacillaceae','Bacilli','Lactobacillales','Lactobacillales','Streptococcaceae','Streptococcus'], ['Bacteria','Actinobacteria','Actinobacteridae','Gordoniaceae','Corynebacteriaceae'], ['Bacteria','Firmicutes','Alicyclobacillaceae','Bacilli','Staphylococcaceae'], ['Bacteria','Cyanobacteria','Chloroplasts','vectors']]) self.assertEqual(obs, exp) def test_make_envs_dict(self): """ make_envs_dict should have the same abundance for each taxon as the matrix that made the dict""" envs = make_envs_dict(self.l19_data, self.l19_sample_names, self.l19_taxon_names) for key in envs.keys(): col_idx = self.l19_taxon_names.index(key) self.assertEqual(sum(envs[key].values()), self.l19_data[:,col_idx].sum()) def test_fields_to_dict(self): """fields_to_dict should make first field key, rest val""" test_data = \ """0 R27DLI_4812 R27DLI_600 R27DLI_727 U1PLI_403 U1PLI_8969 U1PLI_9080 U1PLI_9526 W3Cecum_6642 W3Cecum_8992 1 U1PLI_7889 2 W3Cecum_4858 3 R27DLI_3243 R27DLI_4562 R27DLI_6828 R27DLI_9097 U1PLI_2780 U1PLI_67 U9PSI_10475 U9PSI_4341 W3Cecum_5191""".splitlines() #output from cd-hit obs = fields_to_dict(test_data) exp = {'0':['R27DLI_4812','R27DLI_600','R27DLI_727','U1PLI_403','U1PLI_8969','U1PLI_9080','U1PLI_9526','W3Cecum_6642','W3Cecum_8992'], '1':['U1PLI_7889'], '2':['W3Cecum_4858'], '3':['R27DLI_3243','R27DLI_4562','R27DLI_6828','R27DLI_9097','U1PLI_2780','U1PLI_67','U9PSI_10475','U9PSI_4341','W3Cecum_5191']} self.assertEqual(obs, exp) def test_parse_qiime_parameters(self): """parse_qiime_parameters: functions with valid input """ lines = ["#Don't edit this file!",\ "pick_otus:similarity 0.94",\ "pick_otus:otu_picking_method\tcdhit",\ "align_seqs:verbose",\ "assign_taxonomy:use_rdp\ttRuE",\ "assign_taxonomy:something\tNone",\ "",\ "#some_script:fake_parameter\t99.0"] actual = parse_qiime_parameters(lines) expected = {'pick_otus':\ {'similarity':'0.94', 'otu_picking_method':'cdhit'},\ 'assign_taxonomy':\ {'use_rdp':None}} self.assertEqual(actual,expected) # default dict functions as expected -- looking up non-existant key # returns empty dict self.assertEqual(actual['some_other_script'],{}) def test_parse_sample_mapping(self): """parse_sample_mapping works""" lines = self.SampleMapping OTU_sample_info, all_sample_names = parse_sample_mapping(lines) self.assertEqual(OTU_sample_info, {'OTU2': {'sample1': '1', 'sample3': '0', 'sample2': '2'}, 'OTU1': {'sample1': '3', 'sample3': '2', 'sample2': '0'}}) self.assertEqual(all_sample_names, set(['sample1', 'sample3', 'sample2'])) #test that it works if no sample counts in there lines = self.SampleMapping2 OTU_sample_info, all_sample_names = parse_sample_mapping(lines) self.assertEqual(OTU_sample_info, {'OTU2': {'sample1': '1', 'sample3': '0', 'sample2': '1'}, 'OTU1': {'sample1': '1', 'sample3': '1', 'sample2': '0'}}) self.assertEqual(all_sample_names, set(['sample1', 'sample3', 'sample2'])) def test_sample_mapping_to_otu_table(self): """sample_mapping_to_otu_table works""" lines = self.SampleMapping result = sample_mapping_to_otu_table(lines) self.assertEqual(result, ['#Full OTU Counts',\ '#OTU ID\tsample1\tsample2\tsample3', 'OTU2\t1\t2\t0', \ 'OTU1\t3\t0\t2']) def test_parse_taxonomy(self): """ should parse taxonomy example, keeping otu id only""" example_tax = \ """412 PC.635_647 Root;Bacteria;Firmicutes;"Clostridia";Clostridiales 0.930 319 PC.355_281 Root;Bacteria;Bacteroidetes 0.970 353 PC.634_154 Root;Bacteria;Bacteroidetes 0.830 17 PC.607_302 Root;Bacteria;Bacteroidetes 0.960 13 PC.481_1214 Root;Bacteria;Firmicutes;"Clostridia";Clostridiales 0.870 338 PC.593_1314 Root;Bacteria 0.990 42556 Additional fields ignored""" res = parse_taxonomy(example_tax.split('\n')) self.assertEqual(res['412'], "Root;Bacteria;Firmicutes;\"Clostridia\";Clostridiales") self.assertEqual(res['338'], "Root;Bacteria") def test_parse_qiime_config_files(self): """ parse_qiime_config_files functions as expected """ fake_file1 = ['key1\tval1','key2 val2'] fake_file2 = ['key2\tval3'] actual = parse_qiime_config_files([fake_file1,fake_file2]) expected = {'key1':'val1','key2':'val3'} self.assertEqual(actual,expected) # looking up a non-existant value returns None self.assertEqual(actual['fake_key'],None) # empty dict on empty input self.assertEqual(parse_qiime_config_files([]),{}) def test_parse_metadata_state_descriptions(self): """parse_metadata_state_descriptions should return correct states from string.""" s = '' self.assertEqual(parse_metadata_state_descriptions(s), {}) s = 'Study:Twin,Hand,Dog;BodySite:Palm,Stool' self.assertEqual(parse_metadata_state_descriptions(s), {'Study':set(['Twin','Hand','Dog']), 'BodySite':set(['Palm','Stool'])}) def test_parse_illumina_line_barcode_in_header(self): """parse_illumina_line: handles barcode in header correctly """ illumina_line0 = illumina_read1[0] illumina_line1 = illumina_read1[1] actual = parse_illumina_line( illumina_line0,barcode_length=6,rev_comp_barcode=True) expected = {\ 'Full description':'HWI-6X_9267:1:1:4:1699#ACCACCC/1',\ 'Machine Name':'HWI-6X_9267',\ 'Channel Number':1,\ 'Tile Number':1,\ 'X Position':4,\ 'Y Position':1699,\ 'Barcode':'GGTGGT',\ 'Full Y Position Field':'1699#ACCACCC/1',\ 'Sequence':\ 'TACGGAGGGTGCGAGCGTTAATCGCCCCCCCCCCCCCCCCCCCCCCCCCCCC'+\ 'CCCCCCCCCCCCCCCCCCCCCCCGAAAAAAAAAAAAAAAAAAAAAAA',\ 'Quality Score':\ 'abbbbbbbbbb`_`bbbbbb`bb^aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'+\ 'aaaaaaaaaaaaaDaabbBBBBBBBBBBBBBBBBBBB'} self.assertEqual(actual,expected) actual = parse_illumina_line( illumina_line0,barcode_length=6,rev_comp_barcode=False) expected['Barcode'] = 'ACCACC' actual = parse_illumina_line( illumina_line1,barcode_length=6,rev_comp_barcode=True) expected = {\ 'Full description':'HWI-6X_9267:1:1:4:390#ACCTCCC/1',\ 'Machine Name':'HWI-6X_9267',\ 'Channel Number':1,\ 'Tile Number':1,\ 'X Position':4,\ 'Y Position':390,\ 'Barcode':'GGAGGT',\ 'Full Y Position Field':'390#ACCTCCC/1',\ 'Sequence':\ 'GACAGGAGGAGCAAGTGTTATTCAAATTATGCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCGG'+\ 'GGGGGGGGGGGGGAAAAAAAAAAAAAAAAAAAAAAA',\ 'Quality Score':\ 'aaaaaaaaaa```aa\^_aa``aVaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'+\ 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaBaaaaa'} self.assertEqual(actual,expected) actual = parse_illumina_line( illumina_line1,barcode_length=6,rev_comp_barcode=False) expected['Barcode'] = 'ACCTCC' def test_parse_illumina_line_barcode_in_sequence(self): """parse_illumina_line: handles barcode in sequence correctly """ illumina_line0 = illumina_read3[0] actual = parse_illumina_line( illumina_line0,barcode_length=12, rev_comp_barcode=False,barcode_in_sequence=True) expected = {\ 'Full description':'HWI-EAS440_0386:1:23:19516:1031#0/1', 'Machine Name':'HWI-EAS440_0386', 'Channel Number':1, 'Tile Number':23, 'X Position':19516, 'Y Position':1031, 'Barcode':'ACAGCTAGCTTG', 'Full Y Position Field':'1031#0/1', 'Sequence': 'TACGNAGGATCCGAGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGGTGGATT' 'GTTAAGTCAGTTGTGAAAGTTTGCGGCTCAACCGTAAAATTGCAGTTGATACTGGGTGTCTTGAGT' 'ACAGTAGAGGCAGGCGGAATTCGTGGGG', 'Quality Score': 'fffcGddd\_``_gggggggggggfgggggegggggcgggggggggggeeffafdcdfgbdggdbe]fbf' 'dddddbdadadcddaf`abb`cVNRNUScaa``aOY]]]_[_BBBBBBBBBBB' 'BBBBBBBBBBBBBBBBBBBBBBBBBBBBB'} self.assertEqual(actual,expected) def test_parse_qual_score(self): """qual_score should return dict of {id: qual_scores}""" scores = StringIO('>x\n5 10 5\n12\n>y\n30 40') self.assertEqual(parse_qual_score(scores), {'x':[5,10,5,12],'y':[30,40]}) #Check that a bad file, e.g. a fast raises Error bad_scores = StringIO('>x\nabcbd\n12\n>y\GATC') self.assertRaises(QiimeParseError, parse_qual_score, bad_scores) def test_parse_qual_scores(self): """qual_scores should return dict of {id:qual_scores}""" scores = StringIO('>x\n5 10 5\n12\n>y\n30 40') scores2= StringIO('>a\n5 10 5\n12\n>b\n30 40') self.assertEqual(parse_qual_scores([scores, scores2]), {'x':[5,10,5,12],'y':[30,40],'a':[5,10,5,12],'b':[30,40]}) def test_MinimalQualParser(self): """MinimalQualParser should yield (id_, quals)""" scores = ['>x','5 10 5','12', '>y','30 40', '>a','5 10 5','12', '>b','30 40'] gen = MinimalQualParser(scores) self.assertEqual(list(gen), [('x',[5,10,5,12]), ('y',[30,40]), ('a',[5,10,5,12]), ('b',[30,40])]) def test_parse_trflp(self): """ should return a header and otu_table lists""" data = \ """ Bin (10bp) Bin (20bp) Bin (30bp) Bin (40 bp) Samp-le 1 1000 2000 3000 4000 Sample 2 2000 3000 4000 Sample 3 3000 4000 Sample 4 4000 Sample 5 25 """ samples, otus, data = parse_trflp(data.split('\n')) samples_exp = ['Samp.le.1', 'Sample.2', 'Sample.3', 'Sample.4', 'Sample.5'] otus_exp = ['Bin__10bp_', 'Bin__20bp_', 'Bin__30bp_', 'Bin__40_bp_'] data_exp = array([[1000, 0, 0, 0, 25],\ [2000, 2000, 0, 0, 0],\ [3000, 3000, 3000, 0, 0],\ [4000, 4000, 4000, 4000, 0]]) self.assertEqual(samples, samples_exp) self.assertEqual(otus, otus_exp) self.assertEqual(data, data_exp) def test_parse_trflp_headerless(self): """ should return a header and otu_table lists""" data = \ """Samp-le 1 1000 2000 3000 4000 Sample 2 2000 3000 4000 Sample 3 3000 4000 Sample 4 4000 Sample_5__ 25 """ samples, otus, data = parse_trflp(data.split('\n')) samples_exp = ['Samp.le.1', 'Sample.2', 'Sample.3', 'Sample.4', 'Sample.5..'] otus_exp = ['Bin__0', 'Bin__1', 'Bin__2', 'Bin__3'] data_exp = array([[1000, 0, 0, 0, 25],\ [2000, 2000, 0, 0, 0],\ [3000, 3000, 3000, 0, 0],\ [4000, 4000, 4000, 4000, 0]]) self.assertEqual(samples, samples_exp) self.assertEqual(otus, otus_exp) self.assertEqual(data, data_exp) def test_parse_trflp_headerless_diff_row_len(self): """ should return a header and otu_table lists""" data = \ """Samp-le 1 1000 2000 Sample 2 2000 Sample 3 3000 Sample 4 4000 Sample 5 25 """ samples, otus, data = parse_trflp(data.split('\n')) samples_exp = ['Samp.le.1', 'Sample.2', 'Sample.3', 'Sample.4', 'Sample.5'] otus_exp = ['Bin__0', 'Bin__1', 'Bin__2', 'Bin__3'] data_exp = array([[1000, 0, 0, 0, 25],\ [2000, 2000, 0, 0, 0],\ [0, 0, 3000, 0, 0],\ [0, 0, 0, 4000, 0]]) self.assertEqual(samples, samples_exp) self.assertEqual(otus, otus_exp) self.assertEqual(data, data_exp) def test_parse_denoiser_mapping(self): """ parse_denoiser_mapping creates {} from denoiser mapping file """ actual = parse_denoiser_mapping(self.denoiser_mapping1) expected = {'Read1':['Read1','Read4','Read5 some comment'], 'Read2':['Read2'], 'Read3':['Read3','Read6']} self.assertEqual(actual,expected) illumina_read1 = """HWI-6X_9267:1:1:4:1699#ACCACCC/1:TACGGAGGGTGCGAGCGTTAATCGCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCGAAAAAAAAAAAAAAAAAAAAAAA:abbbbbbbbbb`_`bbbbbb`bb^aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaDaabbBBBBBBBBBBBBBBBBBBB HWI-6X_9267:1:1:4:390#ACCTCCC/1:GACAGGAGGAGCAAGTGTTATTCAAATTATGCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCGGGGGGGGGGGGGGGAAAAAAAAAAAAAAAAAAAAAAA:aaaaaaaaaa```aa\^_aa``aVaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaBaaaaa""".split('\n') illumina_read2 = """HWI-6X_9267:1:1:4:1699#ACCACCC/2:TTTTAAAAAAAAGGGGGGGGGGGCCCCCCCCCCCCCCCCCCCCCCCCTTTTTTTTTTTTTAAAAAAAAACCCCCCCGGGGGGGGTTTTTTTAATTATTC:aaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbcccccccccccccccccBcccccccccccccccc```````BBBB HWI-6X_9267:1:1:4:390#ACCTCCC/2:ACGTACGTACGTACGTACGTACGTACGTACGTACGTACGTACGTACGTACGTACGTACGTACGTACGTACGTACGTACGTACGTACGTACGTACGTACG:aaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbbbaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbb""".split('\n') illumina_read3 = """HWI-EAS440_0386:1:23:19516:1031#0/1:ACAGCTAGCTTGTACGNAGGATCCGAGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGGTGGATTGTTAAGTCAGTTGTGAAAGTTTGCGGCTCAACCGTAAAATTGCAGTTGATACTGGGTGTCTTGAGTACAGTAGAGGCAGGCGGAATTCGTGGGG:gggggggeggcffffcGddd\_``_gggggggggggfgggggegggggcgggggggggggeeffafdcdfgbdggdbe]fbfdddddbdadadcddaf`abb`cVNRNUScaa``aOY]]]_[_BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB HWI-EAS440_0386:1:23:19660:1034#0/1:CATATCGCAGTTTACGNAAGGTCCGAGCGTTGTCCGGAATCATTGGGCGTAAAGGGTACGTAGGCGGGTAAGCAAGTTAGAAGTGAAATCCTATAGCTCAACTATAGTAAGCTTTTAAAACTGCTCATCTTGAGGTATGGAAGGGAAAGTGGAATTCCTAGTTA:fhhghhhhfhghhhhcHdcddccddhhhhhhfhhhhghhhdghhhhhhhhhhfhhhdhghgghhhhhhbfdfdbagdgdgfffafa]dad_acdabZcaabad[a__^_`cbddefb_cd^]_L\]U_]^aaZ___]bBBBBBBBBBBBBBBBBBBBBBBBBBB""".split('\n') legacy_otu_table1 = """# some comment goes here #OTU ID Fing Key NA Consensus Lineage 0 19111 44536 42 Bacteria; Actinobacteria; Actinobacteridae; Propionibacterineae; Propionibacterium 1 1216 3500 6 Bacteria; Firmicutes; Alicyclobacillaceae; Bacilli; Lactobacillales; Lactobacillales; Streptococcaceae; Streptococcus 2 1803 1184 2 Bacteria; Actinobacteria; Actinobacteridae; Gordoniaceae; Corynebacteriaceae 3 1722 4903 17 Bacteria; Firmicutes; Alicyclobacillaceae; Bacilli; Staphylococcaceae 4 589 2074 34 Bacteria; Cyanobacteria; Chloroplasts; vectors """ otu_table1 = """# Some comment OTU ID Fing Key NA Consensus Lineage 0 19111 44536 42 Bacteria; Actinobacteria; Actinobacteridae; Propionibacterineae; Propionibacterium # some other comment 1 1216 3500 6 Bacteria; Firmicutes; Alicyclobacillaceae; Bacilli; Lactobacillales; Lactobacillales; Streptococcaceae; Streptococcus 2 1803 1184 2 Bacteria; Actinobacteria; Actinobacteridae; Gordoniaceae; Corynebacteriaceae # comments # everywhere! 3 1722 4903 17 Bacteria; Firmicutes; Alicyclobacillaceae; Bacilli; Staphylococcaceae 4 589 2074 34 Bacteria; Cyanobacteria; Chloroplasts; vectors """ otu_table1_floats = """# Some comment OTU ID Fing Key NA Consensus Lineage 0 19111.0 44536.0 42.0 Bacteria; Actinobacteria; Actinobacteridae; Propionibacterineae; Propionibacterium # some other comment 1 1216.0 3500.0 6.0 Bacteria; Firmicutes; Alicyclobacillaceae; Bacilli; Lactobacillales; Lactobacillales; Streptococcaceae; Streptococcus 2 1803.0 1184.0 2.0 Bacteria; Actinobacteria; Actinobacteridae; Gordoniaceae; Corynebacteriaceae # comments # everywhere! 3 1722.1 4903.2 17 Bacteria; Firmicutes; Alicyclobacillaceae; Bacilli; Staphylococcaceae 4 589.6 2074.4 34.5 Bacteria; Cyanobacteria; Chloroplasts; vectors """ taxa_summary1 = """#Full OTU Counts Taxon Even1 Even2 Even3 Bacteria;Actinobacteria;Actinobacteria(class);Actinobacteridae 0.0880247251673 0.0721968465746 0.081371761759 Bacteria;Bacteroidetes/Chlorobigroup;Bacteroidetes;Bacteroidia 0.192137761955 0.191095101593 0.188504131885 Bacteria;Firmicutes;Bacilli;Lactobacillales 0.0264895739603 0.0259942669171 0.0318460745596 # some comment Bacteria;Firmicutes;Clostridia;Clostridiales 0.491800007824 0.526186212556 0.49911159984 Bacteria;Firmicutes;Erysipelotrichi;Erysipelotrichales 0.0311411916592 0.0184083913576 0.0282325481054 Bacteria;Proteobacteria;Gammaproteobacteria;Enterobacteriales 0.166137214246 0.163087129528 0.168923372865 No blast hit;Other 0.00426952518811 0.00303205147361 0.0020105109874""" taxa_summary1_expected = (['Even1','Even2','Even3'], ['Bacteria;Actinobacteria;Actinobacteria(class);Actinobacteridae', 'Bacteria;Bacteroidetes/Chlorobigroup;Bacteroidetes;Bacteroidia', 'Bacteria;Firmicutes;Bacilli;Lactobacillales', 'Bacteria;Firmicutes;Clostridia;Clostridiales', 'Bacteria;Firmicutes;Erysipelotrichi;Erysipelotrichales', 'Bacteria;Proteobacteria;Gammaproteobacteria;Enterobacteriales', 'No blast hit;Other'], array([[0.0880247251673, 0.0721968465746, 0.081371761759], [0.192137761955, 0.191095101593, 0.188504131885], [0.0264895739603, 0.0259942669171, 0.0318460745596], [0.491800007824, 0.526186212556, 0.49911159984], [0.0311411916592, 0.0184083913576, 0.0282325481054], [0.166137214246, 0.163087129528, 0.168923372865], [0.00426952518811, 0.00303205147361, 0.0020105109874]])) expected_lineages1 = [['Bacteria','Actinobacteria','Actinobacteridae','Propionibacterineae','Propionibacterium'], ['Bacteria','Firmicutes','Alicyclobacillaceae','Bacilli','Lactobacillales','Lactobacillales','Streptococcaceae','Streptococcus'], ['Bacteria','Actinobacteria','Actinobacteridae','Gordoniaceae','Corynebacteriaceae'], ['Bacteria','Firmicutes','Alicyclobacillaceae','Bacilli','Staphylococcaceae'], ['Bacteria','Cyanobacteria','Chloroplasts','vectors']] denoiser_mapping1 = """Read1:\tRead4\tRead5 some comment Read2: Read3:\tRead6""" if __name__ =='__main__': main()