import sys
import pkgutil
import imp
import os,os.path
import types
import time
import inspect
import cPickle
import json
del sys.path[0]

def get_builtin_mods(mods):
    builtins = set(sys.builtin_module_names)
    for m in builtins:
        mods[m]=imp.find_module(m)
    return mods

def get_package_mods(mods,mod_list,prefix='',path=None,level=0):
    for m in mod_list:
        try:
            fp, pathname, description = imp.find_module(m,path)
            mod = __import__(prefix+m)
            ##mod = imp.load_module(m, fp, pathname, description)
        except:
            import traceback
            print prefix,m,path
            print traceback.format_exception(*sys.exc_info())
            #traceback.print_exception(*sys.exc_info())
            print 'ERROR: skipping',prefix+m
            continue
        try:
            chpath=mod.__path__
        except:
            chpath=None
        fullname=prefix+m
        mods[fullname]=None
        chmods=pkgutil.iter_modules([pathname])
        mlist=[n[1] for n in chmods if not n[1].startswith('_') and not n[1].startswith('test') and not n[1].startswith('try')]
        get_package_mods(mods,mlist,fullname+'.',chpath,level+1)

chtypes=[types.ClassType, types.ObjectType, types.ModuleType, types.TypeType] #types.InstanceType
obtypes=[types.ClassType, types.ObjectType, types.TypeType] #types.InstanceType

def parse_objs(symbols,obj_list,follow=True,prefix=''):
    if(type(symbols)!=types.DictType):
        print '***NOT DICT',prefix,type(symbols)
    for o in obj_list:
        if o[0].startswith('_'):
            continue
        chsyms=None
        args=None
        path = None
        line = None
        try:
            doc=str(o[1].__doc__)
        except:
            doc=''
        try:
            path = inspect.getsourcefile(o[1])
        except TypeError:
            pass
        try:
            lines,line = inspect.getsourcelines(o[1])
        except:
            pass
        try:
            o[1].__bases__
            has_bases=True
        except:
            has_bases=False
        if type(o[1]) in obtypes or has_bases:
            ch=inspect.getmembers(o[1])
            if follow:
                print '###follow',prefix+o[0],type(o[1])
                chsyms={}
                parse_objs(chsyms,ch,False,prefix+o[0]+'.')
        if type(o[1]) in [types.BuiltinFunctionType, types.BuiltinMethodType, types.FunctionType,
                        types.LambdaType, types.UnboundMethodType, types.MethodType]:
            try:
                a=inspect.getargspec(o[1])
                args=', '.join(a.args)
                if a.varargs is not None:
                    args+= ', *'+a.varargs
                if a.keywords is not None:
                    args+= ', **'+a.keywords
            except:
                pass
        symbols[o[0]]=(str(type(o[1])),args,path,line,doc,chsyms)

def parse_mods(mods):
    symbols={}  ##Symbol is formatted key:value key=symbol, value=tuple(type,args,docstring,children)
    for m in sorted(mods):
        try:
            print m
            a=m.find('.')
            if a>=0:
                mod=__import__(m,fromlist=[m[a+1:]])
            else:
                mod=__import__(m)
        except:
            continue
        msyms=m.split('.')
        psymbols=symbols
        for s in msyms:
            if s not in psymbols:
                psymbols[s]=[str(types.ModuleType),None,mod.__doc__,{}]
            elif type(psymbols[s][-1])!=types.DictType:
                print 'WARNING FOUND',msyms,type(psymbols[s][-1])
                psymbols[s]=[str(types.ModuleType),None,mod.__doc__,{}]
            psymbols=psymbols[s][-1]
        parse_objs(psymbols,inspect.getmembers(mod),True,m+'.')
    return symbols


def create(dest='STDLIB',use_json=True):
    '''
    creates the standard lib from path dest
    '''
    mods={}

    #get_builtin_mods(mods)
    #std_mod_list=[line.strip().split(' ')[0] for line in open("pymods.txt","r").readlines()]
    #for m in std_mod_list:
    #    mods[m]=None
    #pkg_mod_list=['gtk','numpy','scipy','pandas','statsmodels']
    pkg_mod_list=['numpy']
    get_package_mods(mods,pkg_mod_list)

    print 'MODULES'
    print '##########'
    for m in sorted(mods):
        print m

    syms=parse_mods(mods)
    for s in syms:
        print s

    t=time.time()
    f=open(dest,'wb')
    if use_json:
        f.write(json.dumps(syms))
    else:
        cPickle.dump(syms,f)
    f.close()
    print 'write took',time.time()-t
    return syms

def load(src='STDLIB',use_json=True):
    '''
    loads the stdlib from path src
    '''
    try:
        if os.path.exists(src):
            t=time.time()
            f=open(src,'rb')
            if use_json:
                mods=json.loads(f.read())
            else:
                mods=cPickle.load(f)
            f.close()
            print 'read took',time.time()-t
            print 'main scope'
            for s in sorted(mods):
                print s
            return mods
        else:
            return None
    except:
        return None

if __name__=='__main__':
#    symbols=create()
    syms=load()
    print 'numpy' in syms
    api = syms['numpy'][-1]
    for s in sorted(api):
        print s,api[s][:-2]
#    ols = api['OLS'][-1]
#    for s in sorted(ols):
#        print s,ols[s][:-2]
#    datasets = api['datasets'][-1]
#    datasets2 = syms['statsmodels'][-1]['datasets'][-1]
#    for s in sorted(datasets):
#        print s
#    for s in sorted(datasets2):
#        print s
#    print datasets == datasets2