Author: Alastair McKinstry Description: Python3 porting patches Last-Updated: 2019-08-04 Forwarded: no Index: drslib-0.3.1.p3/drslib/cmip5.py =================================================================== --- drslib-0.3.1.p3.orig/drslib/cmip5.py +++ drslib-0.3.1.p3/drslib/cmip5.py @@ -76,7 +76,7 @@ for k in cmip3_models: raise Exception("Duplicate model %s" % k) model_institute_map[k] = cmip3_models[k] # Add overrides from the config file -for institute, models in config.institutes.items(): +for institute, models in list(config.institutes.items()): for model in models: model_institute_map[model] = institute @@ -85,7 +85,7 @@ class InstituteHandler(T.GenericComponen path_i = T.CMIP5_DRS.PATH_INSTITUTE file_i = None component = 'institute' - vocab = model_institute_map.values() + vocab = list(model_institute_map.values()) def filename_to_drs(self, context): context.drs.institute = self._deduce_institute(context) @@ -119,13 +119,13 @@ class ModelHandler(T.GenericComponentHan path_i = T.CMIP5_DRS.PATH_MODEL file_i = T.CMIP5_DRS.FILE_MODEL component = 'model' - vocab = model_institute_map.keys() + vocab = list(model_institute_map.keys()) def _validate(self, s): # Demote validation errors to a warning. try: return super(ModelHandler, self)._validate(s) - except T.TranslationError, e: + except T.TranslationError as e: log.warning('Model validation error: %s', e) return s @@ -142,7 +142,7 @@ class ExperimentHandler(T.GenericCompone super(ExperimentHandler, self).__init__(table_store) # Get valid experiment ids from MIP tables - for table in self.table_store.tables.values(): + for table in list(self.table_store.tables.values()): self.vocab.update(table.experiments) # Get valid experiment ids from metaconfig @@ -157,7 +157,7 @@ class FrequencyHandler(T.GenericComponen super(FrequencyHandler, self).__init__(table_store) self.vocab = set() - for table in self.table_store.tables.values(): + for table in list(self.table_store.tables.values()): try: self.vocab.add(table.frequency) except AttributeError: @@ -196,7 +196,7 @@ class RealmHandler(T.GenericComponentHan # Extract valid realms from the MIP tables self.vocab = set() - for table in table_store.tables.values(): + for table in list(table_store.tables.values()): for var in table.variables: try: realms = table.get_variable_attr(var, 'modeling_realm')[0] Index: drslib-0.3.1.p3/drslib/drs.py =================================================================== --- drslib-0.3.1.p3.orig/drslib/drs.py +++ drslib-0.3.1.p3/drslib/drs.py @@ -22,12 +22,17 @@ import re from abc import ABCMeta from drslib.exceptions import TranslationError +try: + import itertools.izip as zip +except ImportError: + pass + import logging log = logging.getLogger(__name__) -class BaseDRS(dict): +class BaseDRS(dict, metaclass=ABCMeta): """ Base class of classes representing DRS entries. @@ -48,7 +53,6 @@ class BaseDRS(dict): dataset-id. """ - __metaclass__ = ABCMeta DRS_ATTRS = NotImplemented PUBLISH_LEVEL = NotImplemented @@ -87,7 +91,7 @@ class BaseDRS(dict): assert len(argv) == 1 comps = {} - for (k, v) in argv[0].items(): + for (k, v) in list(argv[0].items()): if v is not None: comps[k] = v super(BaseDRS, self).update(comps) else: @@ -208,7 +212,7 @@ class BaseDRS(dict): """ parts = dataset_id.split('.') - for attr, val in itertools.izip(klass._iter_components(with_version=True, to_publish_level=True), parts): + for attr, val in zip(klass._iter_components(with_version=True, to_publish_level=True), parts): if val == '%': continue components[attr] = klass._decode_component(attr, val) @@ -427,7 +431,7 @@ class DRSFileSystem(object): p = [activity] + relpath.split('/') drs = self.drs_cls() - for val, attr in itertools.izip(p, drs._iter_components(with_version=False, to_publish_level=True)): + for val, attr in zip(p, drs._iter_components(with_version=False, to_publish_level=True)): drs[attr] = drs._decode_component(attr, val) log.debug('%s => %s' % (repr(path), drs)) Index: drslib-0.3.1.p3/drslib/drs_command.py =================================================================== --- drslib-0.3.1.p3.orig/drslib/drs_command.py +++ drslib-0.3.1.p3/drslib/drs_command.py @@ -10,10 +10,12 @@ Command-line access to drslib """ +from __future__ import print_function + import sys, os from optparse import OptionParser -from ConfigParser import NoSectionError, NoOptionError +from configparser import NoSectionError, NoOptionError import json from drslib.drs_tree import DRSTree @@ -268,21 +270,21 @@ class Command(object): def print_header(self): - print """\ + print("""\ ============================================================================== DRS Tree at %s ------------------------------------------------------------------------------\ -""" % self.drs_root +""" % self.drs_root) def print_sep(self): - print """\ + print("""\ ------------------------------------------------------------------------------\ -""" +""") def print_footer(self): - print """\ + print("""\ ==============================================================================\ -""" +""") @@ -304,31 +306,31 @@ class ListCommand(Command): elif pt.state != pt.STATE_VERSIONED: to_upgrade += 1 #!TODO: print update summary - print '%-70s %s' % (pt.version_drs().to_dataset_id(with_version=True), state_msg) + print ('%-70s %s' % (pt.version_drs().to_dataset_id(with_version=True), state_msg)) if self.drs_tree.incomplete: self.print_sep() - print 'Incompletely specified incoming datasets' + print ('Incompletely specified incoming datasets') self.print_sep() for dataset_id in sorted(self.drs_tree.incomplete_dataset_ids()): - print '%-70s' % dataset_id + print ('%-70s' % dataset_id) if to_upgrade or broken: self.print_sep() if to_upgrade: - print '%d datasets awaiting upgrade' % to_upgrade + print ('%d datasets awaiting upgrade' % to_upgrade) if broken: if config.check_latest: - print '%d datasets have broken latest versions' % broken + print ('%d datasets have broken latest versions' % broken) else: - print '%d datasets are broken' % broken + print ('%d datasets are broken' % broken) self.print_sep() - for pt in self.drs_tree.pub_trees.values(): + for pt in list(self.drs_tree.pub_trees.values()): if pt.has_failures(): - print 'FAIL %-70s' % pt.drs.to_dataset_id() + print ('FAIL %-70s' % pt.drs.to_dataset_id()) for line in pt.list_failures(): - print ' ', line + print (' ', line) self.print_footer() @@ -342,7 +344,7 @@ class TodoCommand(Command): if pt.count_todo() == 0: if not first: self.print_sep() - print 'Nothing todo for %s' % pt.drs.to_dataset_id() + print('Nothing todo for %s' % pt.drs.to_dataset_id()) first = False continue @@ -354,11 +356,11 @@ class TodoCommand(Command): todos = pt.list_todo(next_version) if not first: self.print_sep() - print "Publisher Tree %s todo for version %d" % (pt.drs.to_dataset_id(), - next_version) + print("Publisher Tree %s todo for version %d" % (pt.drs.to_dataset_id(), + next_version)) first = False - print - print '\n'.join(todos) + print() + print('\n'.join(todos)) self.print_footer() class UpgradeCommand(Command): @@ -374,12 +376,12 @@ class UpgradeCommand(Command): next_version = pt._next_version() if pt.state == pt.STATE_VERSIONED: - print 'Publisher Tree %s has no pending upgrades' % pt.drs.to_dataset_id() + print('Publisher Tree %s has no pending upgrades' % pt.drs.to_dataset_id()) else: - print ('Upgrading %s to version %d ...' % (pt.drs.to_dataset_id(), next_version)), + print(('Upgrading %s to version %d ...' % (pt.drs.to_dataset_id(), next_version)), end=' ') to_process = pt.count_todo() pt.do_version(next_version) - print 'done %d' % to_process + print('done %d' % to_process) self.print_footer() @@ -398,7 +400,7 @@ class MapfileCommand(Command): if len(self.drs_tree.pub_trees) == 0: raise Exception("No datasets selected") - pt = self.drs_tree.pub_trees.values()[0] + pt = list(self.drs_tree.pub_trees.values())[0] #!TODO: better argument handling if len(self.args) > 1: @@ -420,14 +422,14 @@ class HistoryCommand(Command): """ if len(self.drs_tree.pub_trees) != 1: raise Exception("You must select 1 dataset to list history. %d selected" % len(self.drs_tree.pub_trees)) - pt = self.drs_tree.pub_trees.values()[0] + pt = list(self.drs_tree.pub_trees.values())[0] self.print_header() - print "History of %s" % pt.drs.to_dataset_id() + print ("History of %s" % pt.drs.to_dataset_id()) self.print_sep() for version in sorted(pt.versions, reverse=True): vdrs = self.drs_fs.drs_cls(pt.drs, version=version) - print vdrs.to_dataset_id(with_version=True) + print (vdrs.to_dataset_id(with_version=True)) self.print_footer() @@ -443,17 +445,17 @@ class InitCommand(Command): from drslib.p_cmip5.init import init init(self.shelve_dir, config.table_path) - print "CMIP5 configuration data written to %s" % repr(self.shelve_dir) + print("CMIP5 configuration data written to %s" % repr(self.shelve_dir)) class RepairCommand(Command): def do(self): - for drs_id, pt in self.drs_tree.pub_trees.items(): + for drs_id, pt in list(self.drs_tree.pub_trees.items()): if pt.has_failures(): - print 'FIXING %-70s' % drs_id + print ('FIXING %-70s' % drs_id) pt.repair() for line in pt.list_failures(): - print ' ', line + print (' ', line) class DiffCommand(Command): @@ -472,7 +474,7 @@ class DiffCommand(Command): if len(self.drs_tree.pub_trees) == 0: raise Exception("No datasets selected") - pt =self.drs_tree.pub_trees.values()[0] + pt =list(self.drs_tree.pub_trees.values())[0] #!TODO: better argument handling args = self.args[:] @@ -493,7 +495,7 @@ class DiffCommand(Command): v2_msg = v2 else: v2_msg = 'todo list' - print 'Diff between %s and %s' % (v1, v2_msg) + print('Diff between %s and %s' % (v1, v2_msg)) self.print_sep() #!FIXME: Just compare file sizes at the moment! @@ -503,15 +505,15 @@ class DiffCommand(Command): continue # Don't compare by path if using the todo list elif (v2 is not None) and (diff_type & pt.DIFF_PATH == pt.DIFF_PATH): - print 'PATH\t\t%s' % filename + print('PATH\t\t%s' % filename) elif diff_type & pt.DIFF_SIZE == pt.DIFF_SIZE: - print 'SIZE\t\t%s' % filename + print('SIZE\t\t%s' % filename) elif diff_type & pt.DIFF_TRACKING_ID == pt.DIFF_TRACKING_ID: - print 'TRACKING_ID\t%s' % filename + print('TRACKING_ID\t%s' % filename) elif diff_type & pt.DIFF_V1_ONLY == pt.DIFF_V1_ONLY: - print '%s\t\t%s' % (v1, filename) + print('%s\t\t%s' % (v1, filename)) elif diff_type & pt.DIFF_V2_ONLY == pt.DIFF_V2_ONLY: - print '%s\t\t%s' % (v2_msg, filename) + print('%s\t\t%s' % (v2_msg, filename)) else: assert False Index: drslib-0.3.1.p3/drslib/drs_tree.py =================================================================== --- drslib-0.3.1.p3.orig/drslib/drs_tree.py +++ drslib-0.3.1.p3/drslib/drs_tree.py @@ -156,7 +156,7 @@ class DRSTree(object): continue log.debug('File %s => %s' % (repr(filename), drs)) - for k, v in components.items(): + for k, v in list(components.items()): if v is None: continue # If component is present in drs act as a filter @@ -201,7 +201,7 @@ class DRSTree(object): if drs_id not in self.pub_trees: self.pub_trees[drs_id] = PublisherTree(drs, self) - for pt in self.pub_trees.values(): + for pt in list(self.pub_trees.values()): pt.deduce_state() def discover_incoming_fromfiles(self, files_iter, **components): @@ -335,7 +335,7 @@ class DRSList(list): """ items = self - for k, v in kw.items(): + for k, v in list(kw.items()): if type(v) == list: items = [x for x in items if x[1].get(k, None) in v] else: Index: drslib-0.3.1.p3/drslib/mapfile.py =================================================================== --- drslib-0.3.1.p3.orig/drslib/mapfile.py +++ drslib-0.3.1.p3/drslib/mapfile.py @@ -10,6 +10,8 @@ Generate mapfiles from streams of DRS ob """ +from __future__ import print_function + #!TODO: check againsts similar code in datanode_admin and merge CHECKSUM_BLOCKSIZE = 2**20 @@ -39,7 +41,7 @@ def write_mapfile(stream, fh, checksum_f params.append('checksum_type=%s' % checksum_type) params.append('checksum=%s' % checksum) - print >>fh, ' | '.join(params) + print (' | '.join(params), file=fh) Index: drslib-0.3.1.p3/drslib/mip_table.py =================================================================== --- drslib-0.3.1.p3.orig/drslib/mip_table.py +++ drslib-0.3.1.p3/drslib/mip_table.py @@ -143,11 +143,11 @@ class MIPTable(object): @property def variables(self): - return self._vardict.keys() + return list(self._vardict.keys()) @property def experiments(self): - return self._exptdict.keys() + return list(self._exptdict.keys()) @property def frequency(self): @@ -282,8 +282,8 @@ def read_model_table(table_csv): table_reader = csv.reader(fh) # Check first 2 lines look like the right file - header1 = table_reader.next() - header2 = table_reader.next() + header1 = next(table_reader) + header2 = next(table_reader) assert "CMIP5 Modeling Groups" in header1[0] assert 'Abbreviated name of center or group' in header2[1] Index: drslib-0.3.1.p3/drslib/publisher_tree.py =================================================================== --- drslib-0.3.1.p3.orig/drslib/publisher_tree.py +++ drslib-0.3.1.p3/drslib/publisher_tree.py @@ -206,7 +206,7 @@ class PublisherTree(object): for filepath, drs in fl: files2[os.path.basename(filepath)] = filepath - for file in set(files1.keys() + files2.keys()): + for file in set(list(files1.keys()) + list(files2.keys())): if file in files1 and file in files2: yield (self._diff_file(files1[file], files2[file], by_tracking_id), @@ -303,7 +303,7 @@ class PublisherTree(object): """ for checker in self._checker_failures: yield '%-20s: ======== %s ========' % (checker.get_name(), checker.get_message()) - for stat, count in checker.get_stats().items(): + for stat, count in list(checker.get_stats().items()): yield '%-20s: %30s = %d' % (checker.get_name(), stat, count) def has_failures(self): Index: drslib-0.3.1.p3/drslib/thredds.py =================================================================== --- drslib-0.3.1.p3.orig/drslib/thredds.py +++ drslib-0.3.1.p3/drslib/thredds.py @@ -22,8 +22,12 @@ import re from lxml import etree as ET from drslib.drs import CmipDRS from drslib.cmip5 import make_translator -import urlparse from optparse import OptionParser +try: + from urllib.parse import urlparse +except ImportError: + # Python 2 + from urlparse import urlparse import logging log = logging.getLogger(__name__) @@ -102,7 +106,7 @@ def run_checks(etree, checks, environ=No check = CheckClass(environ) try: check.check(etree) - except InvalidThreddsException, e: + except InvalidThreddsException as e: log.error(e) except CheckNotPossible: log.warn('Check %s aborted' % CheckClass.__name__) Index: drslib-0.3.1.p3/drslib/translate_cmip3.py =================================================================== --- drslib-0.3.1.p3.orig/drslib/translate_cmip3.py +++ drslib-0.3.1.p3/drslib/translate_cmip3.py @@ -81,7 +81,7 @@ def check_dirnames(dirpath, dirnames): log.debug('dirnames remaining %s' % dirnames) -def _mkdirs(name, mode=0777): +def _mkdirs(name, mode=0o777): log.info('mkdir -p %s' % name) if not dry_run: os.makedirs(name, mode) @@ -113,7 +113,7 @@ def trans_files(cmip3_path, cmip5_path): try: drs = cmip3_t.path_to_drs(dirpath) path = cmip5_t.drs_to_path(drs) - except TranslationError, e: + except TranslationError as e: log.error('Failed to translate path %s: %s' % (dirpath, e)) continue except: @@ -130,7 +130,7 @@ def trans_files(cmip3_path, cmip5_path): try: drs2 = cmip3_t.filepath_to_drs(os.path.join(dirpath, filename)) filename2 = cmip5_t.drs_to_file(drs2) - except TranslationError, e: + except TranslationError as e: log.error('Failed to translate filename %s: %s' % (filename, e)) continue Index: drslib-0.3.1.p3/drslib/p_cmip5/__init__.py =================================================================== --- drslib-0.3.1.p3.orig/drslib/p_cmip5/__init__.py +++ drslib-0.3.1.p3/drslib/p_cmip5/__init__.py @@ -28,8 +28,8 @@ New in this version: """ -import product -import init +from . import product +from . import init # Exception raised by product detection failures -from product import ProductException, cmip5_product, version, version_date +from .product import ProductException, cmip5_product, version, version_date Index: drslib-0.3.1.p3/drslib/p_cmip5/init.py =================================================================== --- drslib-0.3.1.p3.orig/drslib/p_cmip5/init.py +++ drslib-0.3.1.p3/drslib/p_cmip5/init.py @@ -4,10 +4,16 @@ initialise the drslib.p_cmip5 module. """ +from __future__ import print_function + import os, sys import xlrd, string, shelve import re, glob -from whichdb import whichdb +try: + from dbm import whichdb +except ImportError: + # Python 2 + from whichdb import whichdb import logging log = logging.getLogger(__name__) @@ -43,7 +49,7 @@ day_f10 = ['huss','tasmax','tasmin','tas def scan_table(mip,dir=CMOR_TABLE_DIR): ll = open( '%s/CMIP5_%s' % (dir, mip), 'r' ).readlines() - lll = map( string.strip, ll ) + lll = list(map( string.strip, ll )) ssss = string.join( lll, ':::' ) vitems = string.split( ssss, ':::variable_entry:' )[1:] @@ -76,8 +82,8 @@ def scan_table_csv(mip,ee,dir=CMOR_TABLE ll = open( os.path.join(dir, '%s.csv' % mip), 'r' ).readlines() ## strip out white space, so that variable can be identified unambiguously by ',var,' - lll = map( lambda x: string.replace( string.strip(x), ' ',''), ll ) - vlist = map( lambda x: x[0], ee ) + lll = [string.replace( string.strip(x), ' ','') for x in ll] + vlist = [x[0] for x in ee] ee2 = [] for x in ee: @@ -160,7 +166,7 @@ def init(shelve_dir,mip_dir,mip_csv_dir= version_file = os.path.join(shelve_dir, SHELVE_VERSION_FILE) fh = open(version_file, 'w') - print >>fh, SHELVE_VERSION + print(SHELVE_VERSION, file=fh) fh.close() #--------------------------------------------------------------------------- @@ -169,7 +175,7 @@ def init(shelve_dir,mip_dir,mip_csv_dir= def helper_year( val ): if type( val ) == type( 1. ): return int(val) - elif type(val) in [type('x'), type(u'x')]: + elif type(val) in [type('x'), type('x')]: if string.strip(val) == '': return None if val[-1] == '*': @@ -228,7 +234,7 @@ class workflow: self.status = None def allowed(self,next): - if next not in self.statuses.keys(): + if next not in list(self.statuses.keys()): return False if self.status == None: @@ -308,13 +314,13 @@ class request_importer: key = 'historical%s' % y sh[key] = (93, '7.3',str(this_row[0].value), str(this_row[1].value)) - rl1 = range(46,52) + range(56,97) + range(100,110) + rl1 = list(range(46,52)) + list(range(56,97)) + list(range(100,110)) for r in rl1: this_row = sheet.row(r) key = string.strip( str(this_row[10].value ) ) - if key in mappings.keys(): + if key in list(mappings.keys()): key = mappings[key] - while key in sh.keys(): + while key in list(sh.keys()): key += '+' sh[key] = (r,str(this_row[2].value), str(this_row[0].value), str(this_row[1].value)) sh.close() @@ -329,7 +335,7 @@ class request_importer: def import_other(self,sh,book): oo = book.sheet_by_name('other output') - rl1 = range(39,51) + range(54,95) + range(98,108) + rl1 = list(range(39,51)) + list(range(54,95)) + list(range(98,108)) colh = ['M','N','Q','R'] kkk = 0 for col in [12,13,16,17]: @@ -380,11 +386,11 @@ class request_importer: ll = [this_type] for b in bits: if string.find( b, ':' ) != -1: - st,en,dd = map( int, string.split(b,':') ) + st,en,dd = list(map( int, string.split(b,':') )) for y in range(st,en+1,dd): ll.append( ('year',y) ) elif string.find( b, '-' ) !=-1: - st,en = map( int, string.split(b,'-') ) + st,en = list(map( int, string.split(b,'-') )) ll.append( ('slice',st,en) ) else: log.info('%s %s' % (bits,b) ) @@ -397,7 +403,7 @@ class request_importer: def import_cfmip(self,sh,book): oo = book.sheet_by_name('CFMIP output') - rl1 = range(6,27) + rl1 = list(range(6,27)) ee = {} # # for each row, check for non-zero date ranges, and append to list. NB there may be multiple @@ -481,14 +487,14 @@ class mip_importer_rev: if __name__ == '__main__': import sys if len( sys.argv ) != 3: - print 'usage: init.py go ' + print('usage: init.py go ') sys.exit(0) init( sys.argv[2],CMOR_TABLE_DIR, xls_dir=CMIP5_REQUEST_XLS ) sh = shelve.open( sys.argv[2] + '/standard_output_mip', 'r' ) tlist = ['cf3hr', 'cfSites', 'Oyr','Omon','aero','day','6hrPlev','3hr','cfMon'] - print sh.keys(), len(sh.keys()) + print(list(sh.keys()), len(list(sh.keys()))) for t in tlist: l1 = sh[t] ee = scan_table( t ) @@ -517,18 +523,18 @@ if __name__ == '__main__': if e[3] == 1: kk1 += 1 - print t, len(ee), len(ee2), len(l1), kkk, kk1 + print(t, len(ee), len(ee2), len(l1), kkk, kk1) if kkk > 0: - print '===================================' + print('===================================') for e in ee2: if e[2] == None: - print e - print '===================================' + print(e) + print('===================================') if t == 'day' and kk1 != 10: - print 'error',ee2[0:10] + print('error',ee2[0:10]) if t == 'day': - print ee2[0:10] + print(ee2[0:10]) if len(ee) != len(l1): eevl = [] @@ -540,7 +546,7 @@ if __name__ == '__main__': rold = l1[i] vold = string.strip(rold[5]) if vold not in eevl: - print t,':: not found: ',vold + print(t,':: not found: ',vold) knf += 1 else: kmd.append( eevl.index( vold ) ) @@ -549,11 +555,11 @@ if __name__ == '__main__': if i not in kmd: knmd.append( eevl[i] ) knmd.sort() - print knmd + print(knmd) else: for i in range(len(l1)): rold = l1[i] rnew = ee[i] if rnew[0] != rold[5]: - print t,i,rnew[0],rold[5] + print(t,i,rnew[0],rold[5]) Index: drslib-0.3.1.p3/drslib/p_cmip5/product.py =================================================================== --- drslib-0.3.1.p3.orig/drslib/p_cmip5/product.py +++ drslib-0.3.1.p3/drslib/p_cmip5/product.py @@ -45,7 +45,7 @@ class DebugException(ProductException): def __str__(self): for t in self.tup: - print t + print (t) return repr(self.tup) class ProductScope(ProductException): @@ -111,7 +111,7 @@ class cmip5_product: self.mip_sh = shelve.open( mip_table_shelve, flag='r' ) self.tmpl = shelve.open( template, flag='r' ) self.stdo = shelve.open( stdo, flag='r' ) - self.tmpl_keys = self.tmpl.keys() + self.tmpl_keys = list(self.tmpl.keys()) self.tmpl_keys.sort( ddsort(self.tmpl,0).cmp ) self.pos_in_table = 999 self.config = config @@ -168,17 +168,17 @@ class cmip5_product: fpat = '%s_%s_%s_%s_*.nc' % (self.var,self.table,self.model,self.expt) else: fpat = '*.nc' - fl = map( lambda x: string.split(x, '/')[-1], glob.glob( dir + fpat ) ) + fl = [string.split(x, '/')[-1] for x in glob.glob( dir + fpat )] assert len(fl) != 0,'No files matching %s found in %s' % (fpat,self.path) fl.sort() if self.path_output1 != None or self.path_output2 != None: if self.path_output1 != None: fl1b = glob.glob( self.path_output1 + '/*.nc' ) - fl1 = map( lambda x: string.split(x, '/')[-1], glob.glob( self.path_output1 + '/*.nc' ) ) + fl1 = [string.split(x, '/')[-1] for x in glob.glob( self.path_output1 + '/*.nc' )] else: fl1 = [] if self.path_output2 != None: - fl2 = map( lambda x: string.split(x, '/')[-1], glob.glob( self.path_output2 + '/*.nc' ) ) + fl2 = [string.split(x, '/')[-1] for x in glob.glob( self.path_output2 + '/*.nc' )] else: fl2 = [] if len(fl1) == 0 and len(fl2) == 0: @@ -189,18 +189,18 @@ class cmip5_product: fld[f] = 'N' for f in fl1: assert f not in fl2, 'Files should not be in output1 and output2: %s, %s' % (self.path_output1,self.path_output2) - if fld.has_key(f): + if f in fld: fld[f] = 'R1' else: fld[f] = 'O1' - keys = fld.keys() + keys = list(fld.keys()) for f in fl2: - if fld.has_key(f): + if f in fld: fld[f] = 'R2' else: fld[f] = 'O2' fli = fl[:] - fl = fld.keys() + fl = list(fld.keys()) fl.sort() self.file_dict = fld @@ -358,7 +358,7 @@ class cmip5_product: return self.vline[1] def get_cfmip_request_spec(self): - keys = self.stdo['cfmip'].keys() + keys = list(self.stdo['cfmip'].keys()) log.debug( 'get_cfmip_request_spec: %s' % self.rei[1] ) if self.rei[1] not in keys: log.info( '%s not in keys:: %s' % (self.rei[1],str(keys)) ) @@ -388,7 +388,7 @@ class cmip5_product: def get_request_spec(self): tlist = self.stdo[self.request_col] self.requested_years_list = [] - if self.rei[0]-2 in tlist.keys(): + if self.rei[0]-2 in list(tlist.keys()): tli = self.rei[0]-2 ssp = tlist[tli] self.request_spec = ssp @@ -416,8 +416,8 @@ class cmip5_product: def load_config(self): assert self.config_exists, 'load_config: need a valid configuration file at this point' if not self.config_loaded: - import ConfigParser - self.cp = ConfigParser.SafeConfigParser() + import configparser + self.cp = configparser.SafeConfigParser() self.cp.read( self.config ) self.config_loaded = True @@ -480,7 +480,7 @@ class cmip5_product: #################### #################### def find_product_step_one(self,var,table,expt,model,verbose=False): - if table not in self.mip_sh.keys(): + if table not in list(self.mip_sh.keys()): return self.not_ok( 'Bad mip table:: %s ' % table, 'ERR008' ) ## offset_status has 3 levels: -1: not set, 0: set by default, 1: set using info from configuration file. self.offset_status = -1 @@ -568,7 +568,7 @@ class cmip5_product: if len(self.expt) > 7 and self.expt[0:7] == 'decadal': time_datum = int( self.expt[7:11] ) if time_datum in [1960, 1980, 2005]: - requested_years = map( lambda x: x+time_datum, [10,20,30] ) + requested_years = [x+time_datum for x in [10,20,30]] else: requested_years = (time_datum + 10, ) if self.verbose: @@ -608,18 +608,18 @@ class cmip5_product: y_ec2eh = int( self.cp.get( self.model, 'branch_year_esmControl_to_esmHistorical' ) ) y_ehist0 = int( self.cp.get( self.model, 'base_year_esmHistorical' ) ) offset = y_ec2eh - y_ehist0 - requested_years = map( lambda x: x + offset, self.requested_years_list ) + requested_years = [ x + offset for x in self.requested_years_list ] return self.select_year_list( requested_years, 'output1', rc='OK300.02' ) elif (self.table == '3hr' and self.expt in ['1pctCO2','abrupt4xCO2'] ): offset = self._get_base_year( self.expt ) assert offset != None, 'offset not found for %s, %s, %s' % (self.model, self.expt, self.table) if self.expt == '1pctCO2': - requested_years = map( lambda x: offset + 110 + x, range(30) ) + requested_years = [ offset + 110 + x for x in range(30) ] nrq = 30 else: - requested_years = map( lambda x: offset + x, range(5) ) + \ - map( lambda x: offset + 120 + x, range(30) ) + requested_years = [ offset + x for x in range(5) ] + \ + [ offset + 120 + x for x in range(30) ] nrq = 35 if self.select_year_list( requested_years, 'output1', force_complete = True, rc='OK300.03', no_except=True ): return True @@ -638,7 +638,7 @@ class cmip5_product: elif (self.expt == 'piControl' and self.table == '3hr'): offset = self._get_base_year( '1pctCO2' ) if offset != None: - requested_years = map( lambda x: offset - 1 + x, self.requested_years_list ) + requested_years = [offset - 1 + x for x in self.requested_years_list] result = self.select_year_list( requested_years, 'output1', force_complete = True, rc='OK300.04' ) if self.rc != 'ERR005': return result @@ -661,14 +661,14 @@ class cmip5_product: offset == None if offset != None: - requested_years = map( lambda x: offset - 1 + x, self.requested_years_list ) + requested_years = [offset - 1 + x for x in self.requested_years_list] result = self.select_year_list( requested_years, 'output1', force_complete = True, rc='OK300.05' ) if self.rc != 'ERR005': return result ## return years requested relative to first year if number submitted is greater than 145 if self.nyears_submitted > 145: offset = self.ads_time_period[0] - requested_years = map( lambda x: offset - 1 + x, self.requested_years_list ) + requested_years = [offset - 1 + x for x in self.requested_years_list] result = self.select_year_list( requested_years, 'output1', force_complete = True, rc='OK300.06' ) if self.rc != 'ERR005': return result @@ -682,7 +682,7 @@ class cmip5_product: y_pic2h = int( self.cp.get( self.model, 'base_year_1pctCO2' ) ) ##y_hist0 = int( self.cp.get( self.model, 'base_year_historical' ) ) offset = y_pic2h - requested_years = map( lambda x: x + offset, self.requested_years_list ) + requested_years = [ x + offset for x in self.requested_years_list ] result = self.select_year_list( requested_years, 'output1', force_complete = True, rc='OK300.07' ) if self.rc != 'ERR005': return result Index: drslib-0.3.1.p3/drslib/p_cmip5/time_slices.py =================================================================== --- drslib-0.3.1.p3.orig/drslib/p_cmip5/time_slices.py +++ drslib-0.3.1.p3/drslib/p_cmip5/time_slices.py @@ -53,8 +53,8 @@ class request_time_slice: def load_config(self): assert self.parent.config_exists, 'load_config: need a valid configuration file at this point' if not self.parent.config_loaded: - import ConfigParser - self.cp = ConfigParser.SafeConfigParser() + import configparser + self.cp = configparser.SafeConfigParser() self.cp.read( self.parent.config ) self.parent.config_loaded = True Index: drslib-0.3.1.p3/test/__init__.py =================================================================== --- drslib-0.3.1.p3.orig/test/__init__.py +++ drslib-0.3.1.p3/test/__init__.py @@ -28,7 +28,7 @@ def test_1(): """ drs = get_drs1() - print drs.__dict__ + print (drs.__dict__) def test_2(): """ @@ -70,11 +70,11 @@ def test_4(): def test_5(): # Regression test for a bug - print translator.path_to_drs('cmip5/output/MOHC/HadCM3/historicalNat/mon/atmos/Amon/r1i1p1/v3/tas/tas_Amon_HadCM3_historicalNat_r1i1p1_185001-200512.nc') + print (translator.path_to_drs('cmip5/output/MOHC/HadCM3/historicalNat/mon/atmos/Amon/r1i1p1/v3/tas/tas_Amon_HadCM3_historicalNat_r1i1p1_185001-200512.nc')) def test_6(): # Bug reported by Ag - print translator.filename_to_drs('cct_Amon_HadGEM2-ES_piControl_r1i1p1.nc') + print (translator.filename_to_drs('cct_Amon_HadGEM2-ES_piControl_r1i1p1.nc')) def test_7(): @@ -129,7 +129,7 @@ def test_11(): def test_11(): # Check single ensemble number is allowed drs = translator_noversion.filename_to_drs('hur_Amon_HadGEM2-ES_historical_r1_186212-186311.nc') - print drs.to_dataset_id() + print (drs.to_dataset_id()) def test_11(): # Detect regression where 0-indexed ensembles are printed wrong @@ -173,7 +173,7 @@ def test_16(): def test_17(): # From Estani for bug 112 drs = translator.filename_to_drs('psl_6hrPlev_MPI-ESM-LR_amip_r1i1p1_1979010100-197912311800.nc') - print translator.drs_to_file(drs) + print (translator.drs_to_file(drs)) def test_18(): # From Estani for bug 86 @@ -216,8 +216,8 @@ def test_22(): drs = translator.filename_to_drs(fn) calculated_fn = translator.drs_to_file(drs) - print fn - print calculated_fn + print (fn) + print (calculated_fn) assert calculated_fn == fn @@ -233,4 +233,4 @@ def test_23(): drs.product = 'output1' drs.version = 20120101 - print translator.drs_to_filepath(drs) + print (translator.drs_to_filepath(drs)) Index: drslib-0.3.1.p3/drslib/config.py =================================================================== --- drslib-0.3.1.p3.orig/drslib/config.py +++ drslib-0.3.1.p3/drslib/config.py @@ -146,7 +146,7 @@ except: try: checksum_func_str = config.get('hooks', 'checksum_func') except: - import mapfile + from . import mapfile checksum_func = mapfile.calc_md5 else: #!TODO: Move this into metaconfig Index: drslib-0.3.1.p3/test/test_tamip.py =================================================================== --- drslib-0.3.1.p3.orig/test/test_tamip.py +++ drslib-0.3.1.p3/test/test_tamip.py @@ -26,5 +26,5 @@ def do(filename): drs.product = 'output' drs.activity == 'tamip' - print drs + print (drs) assert drs.is_complete() Index: drslib-0.3.1.p3/drslib/drs_tree_check.py =================================================================== --- drslib-0.3.1.p3.orig/drslib/drs_tree_check.py +++ drslib-0.3.1.p3/drslib/drs_tree_check.py @@ -107,7 +107,7 @@ class TreeChecker(object): return set(int(x[1:]) for x in os.listdir(pt.pub_dir) if x[0] == 'v') def _all_versions(self, pt): - return self._fs_versions(pt).union(pt.versions.keys()) + return self._fs_versions(pt).union(list(pt.versions.keys())) def _latest_version(self, pt): versions = self._all_versions(pt) Index: drslib-0.3.1.p3/drslib/translate_iface.py =================================================================== --- drslib-0.3.1.p3.orig/drslib/translate_iface.py +++ drslib-0.3.1.p3/drslib/translate_iface.py @@ -7,7 +7,6 @@ from abc import ABCMeta - #----------------------------------------------------------------------------- # BaseTranslator interface @@ -26,7 +25,7 @@ class BaseTranslator(object): """ __metaclass__ = ABCMeta - + def __init__(self, prefix=''): raise NotImplementedError Index: drslib-0.3.1.p3/test/drs_tree_shared.py =================================================================== --- drslib-0.3.1.p3.orig/test/drs_tree_shared.py +++ drslib-0.3.1.p3/test/drs_tree_shared.py @@ -10,12 +10,12 @@ import tempfile import shutil import sys, os from glob import glob -from StringIO import StringIO +from io import StringIO import datetime from unittest import TestCase -import gen_drs +from . import gen_drs from drslib.drs_tree import DRSTree from drslib import config from drslib.cmip5 import CMIP5FileSystem @@ -61,4 +61,4 @@ class TestListing(TestEg): assert pt.state == pt.STATE_INITIAL pt.do_version() assert pt.state == pt.STATE_VERSIONED - assert pt.versions.keys() == [self.today] + assert list(pt.versions.keys()) == [self.today] Index: drslib-0.3.1.p3/test/gen_drs.py =================================================================== --- drslib-0.3.1.p3.orig/test/gen_drs.py +++ drslib-0.3.1.p3/test/gen_drs.py @@ -20,7 +20,6 @@ import os.path as op from drslib.drs import CmipDRS from drslib import cmip5, config -from itertools import izip # Python 2.5 compatibility try: @@ -29,7 +28,7 @@ except ImportError: def product(*args, **kwds): # product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy # product(range(2), repeat=3) --> 000 001 010 011 100 101 110 111 - pools = map(tuple, args) * kwds.get('repeat', 1) + pools = list(map(tuple, args)) * kwds.get('repeat', 1) result = [[]] for pool in pools: result = [x+[y] for x in result for y in pool] @@ -75,19 +74,19 @@ def iter_drs_template(drs_template, drs_ """ #!NOTE: Assuming the ordering is the same - attrs = drs_attr_values.keys() - pos_values = drs_attr_values.values() + attrs = list(drs_attr_values.keys()) + pos_values = list(drs_attr_values.values()) for values in product(*pos_values): drs = copy(drs_template) - for attr, value in izip(attrs, values): + for attr, value in zip(attrs, values): setattr(drs, attr, value) yield drs def emember_range(imax, rmax=1, pmax=1): - for i in xrange(imax): - for r in xrange(rmax): - for p in xrange(pmax): + for i in range(imax): + for r in range(rmax): + for p in range(pmax): yield (i+1, r+1, p+1) def subset_range(date1, date2, clim, n): @@ -95,7 +94,7 @@ def subset_range(date1, date2, clim, n): # It therefore isn't suitable for making CMOR-correct filenames t_delta = (date2 - date1) / n - for i in xrange(n): + for i in range(n): dt1 = date1 + t_delta*i dt2 = date1 + t_delta*(i+1) N1 = (dt1.year, dt1.month, dt1.day, dt1.hour, None, None) Index: drslib-0.3.1.p3/test/test_mapfile.py =================================================================== --- drslib-0.3.1.p3.orig/test/test_mapfile.py +++ drslib-0.3.1.p3/test/test_mapfile.py @@ -10,12 +10,12 @@ import subprocess as S from drslib.drs_tree import DRSTree -from drs_tree_shared import TestEg, test_dir -import gen_drs +from .drs_tree_shared import TestEg, test_dir +from . import gen_drs from drslib.cmip5 import CMIP5FileSystem from drslib.mapfile import calc_md5 -import StringIO +import io class TestMapfile(TestEg): __test__ = True @@ -29,12 +29,12 @@ class TestMapfile(TestEg): dt = DRSTree(drs_fs) dt.discover(self.incoming, activity='cmip5', product='output1', institute='MOHC', model='HadCM3') - self.pt = dt.pub_trees.values()[0] + self.pt = list(dt.pub_trees.values())[0] self.pt.do_version() assert self.pt.state == self.pt.STATE_VERSIONED def test1(self): - fh = StringIO.StringIO() + fh = io.StringIO() self.pt.version_to_mapfile(self.pt.latest, fh, checksum_func=calc_md5) # Verify checksums match standard md5 tool @@ -44,6 +44,6 @@ class TestMapfile(TestEg): p = S.Popen('md5sum %s' % path, shell=True, stdout=S.PIPE) output = p.stdout.read() if not checksum in output: - print 'LINE:', line.strip() - print 'MD5: ', output.strip() + print ('LINE:', line.strip()) + print ('MD5: ', output.strip()) assert False Index: drslib-0.3.1.p3/test/test_repair.py =================================================================== --- drslib-0.3.1.p3.orig/test/test_repair.py +++ drslib-0.3.1.p3/test/test_repair.py @@ -21,8 +21,9 @@ from glob import glob from drslib.drs_tree import DRSTree from drslib.drs_tree_check import CheckLatest, VERSIONING_LATEST_DIR, VERSIONING_FILES_DIR -from drs_tree_shared import TestEg, test_dir -import gen_drs +from .drs_tree_shared import TestEg, test_dir +from . import gen_drs + import os.path as op @@ -37,7 +38,7 @@ class TestRepair(TestEg): dt = DRSTree(self.tmpdir) dt.discover(self.incoming, activity='cmip5', product='output1', institute='MOHC', model='HadCM3') - self.pt = dt.pub_trees.values()[0] + self.pt = list(dt.pub_trees.values())[0] self.pt.do_version() assert self.pt.state == self.pt.STATE_VERSIONED @@ -59,7 +60,7 @@ class TestRepair(TestEg): if self.pt.has_failures(): self.pt.repair() if self.pt.state == self.pt.STATE_BROKEN: - print '\n'.join(self.pt.list_failures()), '\n' + print ('\n'.join(self.pt.list_failures()), '\n') raise AssertionError() # For tests that need 2 versions @@ -87,7 +88,7 @@ class TestRepair2(TestRepair): self.dt.discover(self.incoming, activity='cmip5', product='output1', institute='MOHC', model='HadCM3') - (self.pt, ) = self.dt.pub_trees.values() + (self.pt, ) = list(self.dt.pub_trees.values()) def _cmor2(self): genfunc = self.genfuncs[1] @@ -150,14 +151,14 @@ class TestRepairVersionContents(TestRepa __test__ = True def breakme(self): - version = self.pt.versions.keys()[0] + version = list(self.pt.versions.keys())[0] variable = 'pr' var_dir = op.join(self.pt.pub_dir, 'v%d/%s' % (version, variable)) # Delete every 3rd file for path in glob('%s/*.nc' % var_dir)[::3]: - print 'REMOVING %s' % path + print ('REMOVING %s' % path) os.remove(path) @@ -178,7 +179,7 @@ class TestLsRepair(TestRepair): dt = DRSTree(self.tmpdir) dt.discover(self.incoming, **self.drs_components) - self.pt = dt.pub_trees.values()[0] + self.pt = list(dt.pub_trees.values())[0] def breakme(self): # Already setup and broken Index: drslib-0.3.1.p3/test/test_drs_tree_listings.py =================================================================== --- drslib-0.3.1.p3.orig/test/test_drs_tree_listings.py +++ drslib-0.3.1.p3/test/test_drs_tree_listings.py @@ -10,15 +10,15 @@ import tempfile import shutil import sys, os, re from glob import glob -from StringIO import StringIO +from io import StringIO import datetime -import gen_drs +from . import gen_drs from drslib.drs_tree import DRSTree from drslib import config from drslib.cmip5 import CMIP5FileSystem -from drs_tree_shared import TestEg, TestListing, test_dir +from .drs_tree_shared import TestEg, TestListing, test_dir @@ -30,12 +30,12 @@ class TestListing1(TestListing): def test_1(self): self._discover('MPI-M', 'ECHAM6-MPIOM-HR') - pt = self.dt.pub_trees.values()[0] + pt = list(self.dt.pub_trees.values())[0] self._do_version(pt) def test_2(self): self._discover('MPI-M', 'ECHAM6-MPIOM-LR') - pt = self.dt.pub_trees.values()[0] + pt = list(self.dt.pub_trees.values())[0] self._do_version(pt) class TestListingOptVar(TestListing): @@ -60,7 +60,7 @@ class TestListingOptVar(TestListing): model='HadGEM2-ES', realm='atmos') - pt = self.dt.pub_trees.values()[0] + pt = list(self.dt.pub_trees.values())[0] self._do_version(pt) #!TODO: confirm realm set right @@ -110,7 +110,7 @@ class TestMapfile(TestListing): mapfile = fh.getvalue() - print mapfile + print (mapfile) assert 'cmip5.output1.MPI-M.ECHAM6-MPIOM-HR.rcp45.mon.ocean.Omon.r1i1p1' in mapfile assert 'output1/MPI-M/ECHAM6-MPIOM-HR/rcp45/mon/ocean/Omon/r1i1p1/v%s'%self.today in mapfile @@ -176,7 +176,7 @@ class TestThreeway(TestEg): def test1(self): v = 1 for listing_path in self.listing_iter: - print 'Doing version %d' % v + print ('Doing version %d' % v) self._discover() assert len(self.dt.pub_trees) == 1 pt = self.dt.pub_trees.values()[0] Index: drslib-0.3.1.p3/test/test_specs.py =================================================================== --- drslib-0.3.1.p3.orig/test/test_specs.py +++ drslib-0.3.1.p3/test/test_specs.py @@ -7,7 +7,7 @@ NOTE: this structure is not finalised! from drslib.specs import SpecsFileSystem, SpecsDRS -from drs_tree_shared import TestEg, TestListing +from .drs_tree_shared import TestEg, TestListing specs_fs = SpecsFileSystem('/specs') @@ -79,7 +79,7 @@ def test_4(): assert drs.experiment == 'decadal' assert drs.start_date[:3] == (1962, 1, 1) assert drs.ensemble == (3,1,1) - assert drs.subset[0][:2] == (1962, 01) + assert drs.subset[0][:2] == (1962, 0o1) assert drs.subset[1][:2] == (1971, 12) def test_5(): @@ -116,7 +116,7 @@ class TestSpecsListing1(TestListing): institute='IPSL', ) - print len(self.dt.pub_trees) + print (len(self.dt.pub_trees)) assert len(self.dt.pub_trees) == 3 @@ -128,14 +128,14 @@ class TestSpecsListing1(TestListing): institute='IPSL', ) - print len(self.dt.pub_trees) + print (len(self.dt.pub_trees)) assert len(self.dt.pub_trees) == 3 - pt = self.dt.pub_trees.values()[0] + pt = list(self.dt.pub_trees.values())[0] self._do_version(pt) pt.deduce_state() - assert len(pt.versions.values()[0]) == 1 + assert len(list(pt.versions.values())[0]) == 1 class TestSpecsListing2(TestSpecsListing1): @@ -163,7 +163,7 @@ class TestSpecsMapfile(TestListing): frequency='mon', institute='IPSL', ) - self.pt = self.dt.pub_trees.values()[0] + self.pt = list(self.dt.pub_trees.values())[0] self._do_version(self.pt) def _init_drs_fs(self): @@ -172,10 +172,10 @@ class TestSpecsMapfile(TestListing): # def version_to_mapfile(self, version, fh=None, checksum_func=None): def test_mapfile(self): - from cStringIO import StringIO + from io import StringIO fh = StringIO() - self.pt.version_to_mapfile(self.pt.versions.keys()[0], fh) + self.pt.version_to_mapfile(list(self.pt.versions.keys())[0], fh) assert 'specs.output.IPSL.IPSL-CM5A-LR.decadal.S19630101.mon.ocean.day.clt.r3i1p1' in fh.getvalue() Index: drslib-0.3.1.p3/test/test_p_cmip5.py =================================================================== --- drslib-0.3.1.p3.orig/test/test_p_cmip5.py +++ drslib-0.3.1.p3/test/test_p_cmip5.py @@ -13,7 +13,7 @@ from test.gen_drs import write_listing_s from drslib import config from nose import with_setup -from drs_tree_shared import test_dir +from .drs_tree_shared import test_dir verbose = False @@ -103,22 +103,22 @@ def test_gen2(): yield check_product3, ('hus', '6hrLev', 'historical', 'tmp/a2_1930_2000', 1949), {'selective_ads_scan':True}, ('output1', 'OK013' ) def test_gen3(): - print '3d aero field' + print ('3d aero field') yield check_product3, ( 'sconcdust', 'aero', 'rcp85', 'tmp/a_2010_2020', 2015), {}, 'output2' yield check_product3, ( 'rhs', 'day', 'historical', 'tmp/a_2005_2100', -999), {}, 'split' yield check_product3, ( 'rhs', 'day', 'historicalxxx', 'tmp/a_2005_2100', -999), {}, 'output1' yield check_product3, ( 'rhs', 'day', 'historical', 'tmp/a_1930_2000', -999), {}, 'split' yield check_product3, ( 'rhs', 'day', 'historical', 'tmp/a_1930_2000', -999), dict( pci=pc2 ), 'split' yield check_product3, ( 'rhs', 'day', 'piControl', 'tmp/a_1930_2000', -999), {}, 'split' - print '------------ noVolc1980 --------------' + print ('------------ noVolc1980 --------------') yield check_product3, ( 'sconcno3', 'aero', 'noVolc1980', 'tmp/a_2005_2100', 2015 ), {'model':'HadGEM2-ES'}, ('output2', 'OK300.08') yield check_product3, ( 'sconcno3', 'aero', 'noVolc1980', 'tmp/a_2005_2100', 2010 ), {'model':'HadGEM2-ES'}, ('output2', 'OK300.08') - print '------------ volcIn2010 --------------' + print ('------------ volcIn2010 --------------') yield check_product3, ( 'sconcno3', 'aero', 'volcIn2010', 'tmp/a_2005_2100', 2015 ), {'model':'HadGEM2-ES'}, ('output2', 'OK300.09') yield check_product3, ( 'sconcno3', 'aero', 'volcIn2010', 'tmp/a_2005_2100', 2014 ), {'model':'HadGEM2-ES'}, ('output1', 'OK300.09') def test_gen4(): - print 'test using sample_2.ini, in which there is a 30 year offset between dating in historical and piControl' + print ('test using sample_2.ini, in which there is a 30 year offset between dating in historical and piControl') yield check_product3, ( 'rhs', 'day', 'piControl', 'tmp/a_1930_2000', -999 ), dict( verbose=verbose, pci=pc2 ), 'split' yield check_product3, ( 'tas', '3hr', 'piControl', 'tmp/a_2005_2100', -999 ), {},'split' yield check_product3, ( 'tas', '3hr', 'historical', 'tmp/a_1930_2000', 1990 ), {},'output1' @@ -137,33 +137,33 @@ def test_gen4(): yield check_product3, ( 'sconcnh4', 'aero', 'decadal2001', 'tmp/a_2005_2100', -999 ), {}, 'split' def test_gen5(): - print 'cfMon, section 1' + print ('cfMon, section 1') yield check_product3, ( 'rlu', 'cfMon', 'amip', 'tmp/a_1930_2000', 1990 ), {}, ('output1','OK300.08') yield check_product3, ( 'rlu', 'cfMon', 'amip', 'tmp/single', None ), {}, ('output1','OK012') def test_gen6(): - print 'cfMon, section 2' + print ('cfMon, section 2') yield check_product3, ( 'rlut4co2', 'cfMon', 'piControl', 'tmp/a_1930_2000', 1950 ), {}, ('output1','OK100') def test_gen7(): - print 'cfMon, section 3' + print ('cfMon, section 3') yield check_product3, ( 'rlu4co2', 'cfMon', 'amip', 'tmp/a_1930_2000', 1980 ), {}, ('output1','OK300.08') yield check_product3, ( 'rlu4co2', 'cfMon', 'piControl', 'tmp/a_1930_2000', 1950 ), {}, ('output1','OK008.2') def test_gen8(): - print '------------ cfMon, section 4 --------------' + print ('------------ cfMon, section 4 --------------') yield check_product3, ( 'clisccp', 'cfMon', 'amip', 'tmp/a_1930_2000', 1980 ), {}, ('output1', 'OK300.08') yield check_product3, ( 'clisccp', 'cfMon', 'piControl', 'tmp/a_2005_2100', 2030 ), {}, ('output2', 'OK200.06') yield check_product3, ( 'clisccp', 'cfMon', 'piControl', 'tmp/a_2005_2100', 2029 ), {}, ('output1', 'OK200.06') yield check_product3, ( 'clisccp', 'cfMon', 'abrupt4xCO2', 'tmp/a_2010_2020', 2015 ), {}, ('output1','OK009.2') - print '------------ 1pctCO2 --------------' + print ('------------ 1pctCO2 --------------') yield check_product3, ( 'va', '6hrPlev', '1pctCO2', 'tmp/a_2010_2020', 1980 ), {'model':'HadGEM2-ES'}, ('output1', 'OK008.1') yield check_product3, ( 'tas', '3hr', 'abrupt4xCO2', 'tmp/a_2010_2020', 2012 ), {'model':'HadGEM2-ES'}, ('output1', 'OK009.2') yield check_product3, ( 'tas', '3hr', 'abrupt4xCO2', 'tmp/a_2005_2100', 2007 ), {'model':'HADCM3'}, ('output1', 'OK200.03') yield check_product3, ( 'tas', '3hr', 'abrupt4xCO2', 'tmp/a_2005_2100', 2016 ), {'model':'HADCM3'}, ('output2', 'OK200.03') def test_gen9(): - print '------------ mohc, first batch --------------' + print ('------------ mohc, first batch --------------') yield check_product3, ( 'hus', '6hrLev', 'historical', 'tmp/mohc1', 1980 ), {'selective_ads_scan':True, 'model':'HadGEM2-ES'}, ('output1', 'OK013') yield check_product3, ( 'ps', '6hrLev', 'historical', 'tmp/mohc1', 1980 ), {'selective_ads_scan':True, 'model':'HadGEM2-ES'}, ('output1', 'OK013') yield check_product3, ( 'ta', '6hrLev', 'historical', 'tmp/mohc1', 1980 ), {'selective_ads_scan':True, 'model':'HadGEM2-ES'}, ('output1', 'OK013') @@ -237,7 +237,7 @@ def test_regression_3(): drs.model, prefix, startyear=drs.subset[0][0]) assert status - print '%s startyear=%d product=%s' % (filename, drs.subset[0][0], pc1.product) + print ('%s startyear=%d product=%s' % (filename, drs.subset[0][0], pc1.product)) assert pc1.product=='output2' @@ -301,7 +301,7 @@ def test_p_cmip5_data_perms(): try: - os.chmod(shelve_file, 0400) + os.chmod(shelve_file, 0o400) # Reload shelves pc1 = p.cmip5_product(mip_table_shelve=shelves['stdo_mip'], template=shelves['template'], @@ -310,7 +310,7 @@ def test_p_cmip5_data_perms(): # Repeat test test_drs_tree() finally: - os.chmod(shelve_file, 0644) + os.chmod(shelve_file, 0o644) pc1 = p.cmip5_product(mip_table_shelve=shelves['stdo_mip'], template=shelves['template'], stdo=shelves['stdo'], Index: drslib-0.3.1.p3/test/test_drs_tree_json.py =================================================================== --- drslib-0.3.1.p3.orig/test/test_drs_tree_json.py +++ drslib-0.3.1.p3/test/test_drs_tree_json.py @@ -7,7 +7,7 @@ from drslib.cordex import CordexFileSystem, CordexDRS from drslib.specs import SpecsFileSystem, SpecsDRS -from drs_tree_shared import TestEg, TestListing, test_dir +from .drs_tree_shared import TestEg, TestListing, test_dir from drslib.drs_tree import DRSTree import json @@ -37,7 +37,7 @@ class TestJson(TestEg): drs_id = 'specs.output.IPSL.IPSL-CM5A-LR.decadal.S20130101.mon.seaIce.OImon.sic.r3i1p1' assert drs_id in drs_tree.pub_trees - p = drs_tree.pub_trees.values()[0] + p = list(drs_tree.pub_trees.values())[0] p_vars = set(drs.variable for (drs_str, drs) in p._todo) # All DRS objects should be for the same variable Index: drslib-0.3.1.p3/test/test_drs_tree_product.py =================================================================== --- drslib-0.3.1.p3.orig/test/test_drs_tree_product.py +++ drslib-0.3.1.p3/test/test_drs_tree_product.py @@ -9,7 +9,7 @@ from drslib.cmip5 import CMIP5FileSystem from drslib import p_cmip5 from drslib import drs_tree -import gen_drs +from . import gen_drs import os, shutil import metaconfig @@ -25,7 +25,7 @@ def setup_module(): tmpdir = tempfile.mkdtemp(prefix='drs_tree-product-') - print 'TMPDIR ',tmpdir + print ('TMPDIR ',tmpdir) shelves = p_cmip5.init._find_shelves(shelve_dir) config_file = os.path.join(os.path.dirname(__file__), 'ukmo_sample.ini') Index: drslib-0.3.1.p3/test/test_drs_tree.py =================================================================== --- drslib-0.3.1.p3.orig/test/test_drs_tree.py +++ drslib-0.3.1.p3/test/test_drs_tree.py @@ -10,19 +10,19 @@ import tempfile import shutil import sys, os from glob import glob -from StringIO import StringIO +from io import StringIO import datetime import os.path as op from unittest import TestCase -import gen_drs +from . import gen_drs from drslib.drs_tree import DRSTree from drslib.drs import CmipDRS from drslib.cmip5 import CMIP5FileSystem from drslib import config -from drs_tree_shared import TestEg, TestListing +from .drs_tree_shared import TestEg, TestListing @@ -43,7 +43,7 @@ class TestEg1(TestEg): experiment='1pctto4x', realm='atmos') assert len(dt.pub_trees) == 3 - k = sorted(dt.pub_trees.keys())[2] + k = sorted(list(dt.pub_trees.keys()))[2] assert k == 'cmip5.output1.MOHC.HadCM3.1pctto4x.day.atmos.day.r3i1p1' pt = dt.pub_trees[k] @@ -59,7 +59,7 @@ class TestEg1(TestEg): product='output1', institute='MOHC', model='HadCM3') assert len(dt.pub_trees) == 3 - pt = dt.pub_trees.values()[0] + pt = list(dt.pub_trees.values())[0] assert pt.drs.realm == 'atmos' def test_3(self): @@ -67,14 +67,14 @@ class TestEg1(TestEg): dt.discover(self.incoming, activity='cmip5', product='output1', institute='MOHC', model='HadCM3') - pt = dt.pub_trees.values()[0] + pt = list(dt.pub_trees.values())[0] assert pt.state == pt.STATE_INITIAL pt.do_version() assert pt.state == pt.STATE_VERSIONED - assert len(pt.versions.keys()) == 1 + assert len(list(pt.versions.keys())) == 1 - assert self.today in pt.versions.keys() + assert self.today in list(pt.versions.keys()) class TestEg2(TestEg): __test__ = True @@ -91,7 +91,7 @@ class TestEg2(TestEg): product='output1', institute='MOHC', model='HadCM3') assert len(dt.pub_trees) == 2 - assert set([x.drs.realm for x in dt.pub_trees.values()]) == set(['atmos', 'ocean']) + assert set([x.drs.realm for x in list(dt.pub_trees.values())]) == set(['atmos', 'ocean']) class TestEg2_1(TestEg2): __test__ = True @@ -113,7 +113,7 @@ class TestEg2_1(TestEg2): dt.discover_incoming(self.tmpdir, realm='atmos', **components) assert len(dt.pub_trees) == 2 - assert set([x.drs.realm for x in dt.pub_trees.values()]) == set(['atmos', 'ocean']) + assert set([x.drs.realm for x in list(dt.pub_trees.values())]) == set(['atmos', 'ocean']) def test_2(self): """Test incremental discovery without calling discover() first.""" @@ -130,7 +130,7 @@ class TestEg2_1(TestEg2): dt.discover_incoming(self.tmpdir, realm='atmos', **components) assert len(dt.pub_trees) == 2 - assert set([x.drs.realm for x in dt.pub_trees.values()]) == set(['atmos', 'ocean']) + assert set([x.drs.realm for x in list(dt.pub_trees.values())]) == set(['atmos', 'ocean']) @@ -153,7 +153,7 @@ class TestEg3(TestEg): self.dt.discover(self.incoming, activity='cmip5', product='output1', institute='MOHC', model='HadCM3') - (self.pt, ) = self.dt.pub_trees.values() + (self.pt, ) = list(self.dt.pub_trees.values()) def _cmor2(self): gen_drs.write_eg3_2(self.tmpdir) @@ -274,7 +274,7 @@ class TestEg3_1(TestEg3): self.dt2 = DRSTree(self.drs_fs) self.dt2.discover_incoming(self.incoming, activity='cmip5', product='output1') - (self.pt, ) = self.dt2.pub_trees.values() + (self.pt, ) = list(self.dt2.pub_trees.values()) @@ -290,7 +290,7 @@ class TestEg4(TestEg3): self.dt.discover(self.incoming, activity='cmip5', product='output1', institute='MOHC', model='HadCM3') - (self.pt, ) = self.dt.pub_trees.values() + (self.pt, ) = list(self.dt.pub_trees.values()) def _cmor2(self): gen_drs.write_eg4_2(self.tmpdir) @@ -352,7 +352,7 @@ class TestEg4_1(TestEg4): self.dt2.discover_incoming(self.incoming, activity='cmip5', product='output1') - (self.pt, ) = self.dt2.pub_trees.values() + (self.pt, ) = list(self.dt2.pub_trees.values()) @@ -369,7 +369,7 @@ class TestEg5(TestEg4): self.dt.discover(self.incoming, activity='cmip5', product='output1', institute='MOHC', model='HadCM3') - (self.pt, ) = self.dt.pub_trees.values() + (self.pt, ) = list(self.dt.pub_trees.values()) def _cmor2(self): gen_drs.write_eg5_2(self.tmpdir) @@ -429,7 +429,7 @@ class TestEg5_1(TestEg5): self.dt2 = DRSTree(self.drs_fs) self.dt2.discover_incoming(self.incoming, activity='cmip5', product='output1') - (self.pt, ) = self.dt2.pub_trees.values() + (self.pt, ) = list(self.dt2.pub_trees.values()) @@ -454,7 +454,7 @@ class TestSymlinks(TestEg): dt.discover(self.incoming, activity='cmip5', product='output1', institute='MOHC', model='HadCM3') - pt = dt.pub_trees.values()[0] + pt = list(dt.pub_trees.values())[0] assert pt.state == pt.STATE_INITIAL pt.do_version() @@ -486,7 +486,7 @@ class TestEg6(TestEg): for i, delivery in enumerate(self.deliveries): self.dt.discover_incoming(op.join(self.incoming, str(i)), activity='cmip5', product='output1', institute='MOHC') - for drs_id, pt in self.dt.pub_trees.items(): + for drs_id, pt in list(self.dt.pub_trees.items()): pt.do_version(i) def setupIncoming(self): @@ -500,11 +500,11 @@ class TestEg6(TestEg): def test_1(self): assert len(self.dt.pub_trees) == 1 - pt = self.dt.pub_trees.values()[0] + pt = list(self.dt.pub_trees.values())[0] - file_counts = set((k, len(v)) for (k, v) in pt.versions.items()) + file_counts = set((k, len(v)) for (k, v) in list(pt.versions.items())) - print file_counts + print (file_counts) assert file_counts == set([(0, 2), (1, 3), (2, 5)]) Index: drslib-0.3.1.p3/test/test_dups.py =================================================================== --- drslib-0.3.1.p3.orig/test/test_dups.py +++ drslib-0.3.1.p3/test/test_dups.py @@ -9,9 +9,9 @@ from unittest import TestCase import shutil import tempfile -import gen_drs +from . import gen_drs from drslib.drs_tree import DRSTree -from drs_tree_shared import TestEg, test_dir +from .drs_tree_shared import TestEg, test_dir CHANGE_FILE = 'hfss_Amon_HadGEM2-ES_rcp45_r1i1p1_209012-209911.nc' @@ -27,7 +27,7 @@ class TestDups(TestEg): self.dt = DRSTree(self.tmpdir) self.dt.discover(self.incoming, activity='cmip5', product='output1', institute='MOHC', model='HadCM3') - self.pt = dt.pub_trees.values()[0] + self.pt = list(dt.pub_trees.values())[0] self.pt.do_version() def tearDown(self): @@ -46,7 +46,7 @@ class TestDups(TestEg): self._make_incoming2() fh = open(os.path.join(self.incoming, CHANGE_FILE), 'a') - print >>fh, 'File has grown' + print ('File has grown', file=fh) def _make_incoming4(self): # As incoming2 except one of the dups only differs by contents Index: drslib-0.3.1.p3/test/test_cmip3.py =================================================================== --- drslib-0.3.1.p3.orig/test/test_cmip3.py +++ drslib-0.3.1.p3/test/test_cmip3.py @@ -34,11 +34,11 @@ def test_listing(): yield convert, line.strip() def convert(filepath): - print ('%s -->' % filepath), + print (('%s -->' % filepath),) drs = translator.filepath_to_drs(filepath) cmip5_filepath = cmip5_translator.drs_to_filepath(drs) - print cmip5_filepath + print (cmip5_filepath) return cmip5_filepath Index: drslib-0.3.1.p3/test/test_cordex.py =================================================================== --- drslib-0.3.1.p3.orig/test/test_cordex.py +++ drslib-0.3.1.p3/test/test_cordex.py @@ -5,7 +5,7 @@ Test CORDEX DRS structure. from drslib.cordex import CordexFileSystem, CordexDRS -from drs_tree_shared import TestEg, TestListing +from .drs_tree_shared import TestEg, TestListing cordex_fs = CordexFileSystem('/cordex') @@ -94,7 +94,7 @@ class TestCordexListing1(TestListing): self.dt.discover(self.incoming, activity='cordex', product='output') - print len(self.dt.pub_trees) + print (len(self.dt.pub_trees)) assert len(self.dt.pub_trees) == 168 def test_2(self): @@ -102,7 +102,7 @@ class TestCordexListing1(TestListing): product='output', frequency='day') - print len(self.dt.pub_trees) + print (len(self.dt.pub_trees)) assert len(self.dt.pub_trees) == 50 def test_3(self): @@ -111,7 +111,7 @@ class TestCordexListing1(TestListing): frequency='day', variable='vas') - print len(self.dt.pub_trees) + print (len(self.dt.pub_trees)) assert len(self.dt.pub_trees) == 1 pt = self.dt.pub_trees.values()[0] Index: drslib-0.3.1.p3/test/test_decadal.py =================================================================== --- drslib-0.3.1.p3.orig/test/test_decadal.py +++ drslib-0.3.1.p3/test/test_decadal.py @@ -22,7 +22,7 @@ def test(): for year in years: filename = filepat % year - print year, + print (year, ) try: drs_obj = cmip5_trans.filename_to_drs(filename)