#!/usr/bin/python
# -*- coding: utf-8 -*-
## import all required classes from the /src/ folder 
## - Note: Classes which are created in the /src/ folder have to be declared public in the file src/__init__.py
##         before they can be used here
from primalintegral_src import * 

## import system-related I/O to have access to terminal commands etc.
import sys
import numpy as np
import os

'''This is the main script to be called to generate Primal Integrals.
 
  The script reads .out files generated by our 'make test(cluster)(gurobi,cpx,xpress,cbc)'
  and needs a .solu file for optimal or best known solutions to base the integral on
  It will generate two .tex source files to contain instancewise integrals and average integrals
  to files specified by two variables, 'integralsfilename' and 'meanintegralsfilename', respectively.
  These can be specified by modifying the script or via command line arguments.
  
  An empty .solu-file will cause the program to crash
  
  Note that all instances which are known to be infeasible are automatically excluded from the primal integral
  calculation.
  
  Example calls:
  1)
  python primalintegral.py -files ../PATH/TO/check.TESTSET.*.out -solufilename ../PATH/TOSOLUFILE/TESTSET.solu
  
  will run over all .out log files matching the expression and obtain the Solution information from the specified
  solufilename and write the integrals and plots to 'integrals.tex' and 'meanintegrals.tex' to this directory
  
  
  2)
  python primalintegral.py -files ../PATH/TO/check.TESTSET.*.out -solufilename ../PATH/TOSOLUFILE/TESTSET.solu
      -integralfilesname PATH/TO/somefilename.tex -meanintegralsfilename PATH/TO/someotherfilename.tex
      
   will generate the .tex output in the specified files instead  
    
'''


## rename the classes for a quicker access
DataCollector = DataCollector.DataCollector
GeneralInformationReader = StatisticReader_GeneralInformationReader.GeneralInformationReader
PrimalBoundHistoryReader = StatisticReader_PrimalBoundHistoryReader.PrimalBoundHistoryReader
PrimalBoundReader = StatisticReader_PrimalBoundReader.PrimalBoundReader
ProblemNameReader = StatisticReader_ProblemNameReader.ProblemNameReader
#PrimalBoundAfterRootNodeReader = StatisticReader_PrimalBoundAfterRootNode.PrimalBoundAfterRootNodeReader

SoluFileReader = StatisticReader_SoluFileReader.SoluFileReader
SolvingTimeReader = StatisticReader_SolvingTimeReader.SolvingTimeReader
StatisticReader = StatisticReader.StatisticReader
TimeLimitReader = StatisticReader_TimeLimitReader.TimeLimitReader

TestRun = TestRun.TestRun
Comparator = Comparator.Comparator

###  the list of .out files evaluated by this script. Usually entered by typing '-files', followed by the .out-files to be part of the evaluation
filelist = [
#   "../new-scip-git/check/results/check.short.scip-3.0.1.1.linux.x86_64.gnu.opt.cpx.opt55.default.out",
#   "../new-scip-git/check/results/check.short.scip-3.0.1.1.linux.x86_64.gnu.opt.cpx.opt54.proxyunlimited.out"
]

#solufilename = '../../scip-git/check/testset/MMM.solu'
solufilename = ''

EXECUTE_PLUGIN = True

### exclude problems manually by explicitly naming them here
excludeProbs = [#"ns1758913", "timetab1", "timetab2", "momentum2" 
#"snp-001-04.lp"
]

# # the timelimit to be taken as default timelimit
DEFAULT_TIMELIMIT = 36000

### enter the list of exclude reasons here. Every instance matching one of the listed reasons like, e. g., 
### it is considered infeasible, will be excluded from the evaluated set of instances, if 'Comparator.EXCLUDE_REASON_INFEASIBLE'
### is an element of this list.
excludereasons = [Comparator.EXCLUDE_REASON_INFEASIBLE,Comparator.EXCLUDE_REASON_NOSOLUTIONKNOWN]

options = ['-files', '-solufilename', '-meanintegralsfilename', '-integralsfilename', '-timelimit']

# ## choose the names of the LaTeX files to be generated for the geometric mean and the explicit instance wise table, respectively
meanintegralsfilename = 'meanintegrals.tex'
integralsfilename = 'integrals.tex'

## initialize default names
testsetname = ''
## install all required readers in this list. Remove readers which are not required for your specific task to speed up the evaluation process
listofreaders = [
                 GeneralInformationReader(),
                 PrimalBoundHistoryReader(),
                 PrimalBoundReader(),
                 ProblemNameReader(),
                 SolvingTimeReader(), 
                 TimeLimitReader(), 
                 ]

datakeys = []  ### initialize the data key list
lastoption = None

### read all arguments from the command line 
for eacharg in sys.argv:
#    print eacharg
   if eacharg in options:
      lastoption = eacharg
   elif lastoption == '-files':
      filelist.append(eacharg)
   elif lastoption == '-solufilename':
      solufilename = eacharg
   elif lastoption == '-timelimit':
      DEFAULT_TIMELIMIT = float(eacharg)
   elif lastoption == '-meanintegralsfilename':
      meanintegralsfilename = eacharg
   elif lastoption == '-integralsfilename':
      integralsfilename = eacharg

execute = True ### The 'execute' flag prevents the script from being run in case of an invalid data set.


writemeanintegrals=True
# # the maximum primal-optimal gap considered for the evaluation
cutoffGap = 100;

# # 
usexlogaxis = False
useylogaxis = False


def changeNameForSoluFile(nameOfProb): 
   '''workaround to get access to .solu-file data even in case of corrupted instance names. 
   
      Keyword arguments:
      nameofProb -- the name of the instance to be potentially modified
      
   '''
   return nameOfProb
      
def getXFloat(somefloat):
   if usexlogaxis:
      return np.log10(max(1.0, somefloat))
   else:
      return somefloat

def getYFloat(somefloat):
   if useylogaxis:
      return np.log10(max(1.0, somefloat))
   else:
      return somefloat
      
def calcIntegralValue(thedatalist, optSol, cutoffgap, timelimit=DEFAULT_TIMELIMIT): 
   '''
      calculates the value of the primal-optimal integral over time. 
      
      Keyword arguments:
      thedatalist -- a list of string tuples (time,pb) representing the improvement of the incumbent objective pb over time
      optSol -- the value of an optimal or best known solution for the calculation of the gap
      cutoffgap -- an upper bound to the value of an evaluated gap
      timelimit -- the time limit as an upper bound for the time-abscissa (default DEFAULT_TIMELIMIT)
   '''
   
   if thedatalist == []:  
      return getXFloat(timelimit) * getYFloat(cutoffgap)
   else:
      integralVal = 0.0
      lastPointInTime = 0.0
      lastgap = getYFloat(cutoffgap)
      for element in thedatalist:
         time, pb = element
         time = getXFloat(min(float(time), timelimit))
         pb = float(pb)
         nextgap = getYFloat(min(Misc.getGap(pb, optSol, useCplexGap=True), cutoffgap))
         
         integralVal += (time - lastPointInTime) * lastgap
         lastgap = nextgap
         lastPointInTime = time
      else:
         # if the end of the data list has been reached, but a gap remains, the integral value is completed
         # by adding the rectangle until the timelimit
         if lastgap > 0.0:
            integralVal += lastgap * (getXFloat(timelimit) - lastPointInTime)
            
      return integralVal
      
      
      
def getMaxTime(probname, testruns):
   '''
      returns the maximum solving time by which a test run found a new solution for a specific instance
      
      Keyword arguments:
      probname -- the name of the instance
   '''
   themax = 0.0
   for testrun in testruns:
      thelist = testrun.problemGetData(probname, PrimalBoundHistoryReader.datakey)
      if thelist != []:
         time = thelist[- 1][0]
         themax = max(themax, time)
   return themax
   
def getMaxGap(probname, testruns, optimum):
   '''
      returns the maximum primal optimal gap of a solution objective for a specific instance
      
      Keyword arguments:
      probname -- the name of the instance
   '''
   themax = 0.0
   
   for testrun in testruns:
      thelist = testrun.problemGetData(probname, PrimalBoundHistoryReader.datakey)
      if thelist != []:
         bound = thelist[0][1]
         gap = min(Misc.getGap(bound, optimum, useCplexGap=True), cutoffGap)
         themax = max(themax, gap)
   return themax            



def getTexHeader(settingsForGeometry=''):
   '''
      returns a possible LaTeX-file header as string object. Settings for the geometry package are an optional argument
      
      Keyword arguments:
      settingsForGeometry -- special package options for the geometry package,e.g., 'twoside,bottom=2cm' (default '')
   '''
   return ('''\documentclass[titlepage, oneside, 10pt]{book}
 \\usepackage[utf8x]{inputenc}
 \\usepackage{amsmath}
 \\usepackage{amssymb}
 \\usepackage{amsthm}
\\usepackage{paralist}
\\usepackage{booktabs}
\\usepackage{overpic}
\\usepackage{color}
\\usepackage{pgfplots}
\\usepackage{tikz}
\\usepackage{pgfplotstable}
 \\usepackage{booktabs}
 \\usepackage{array}
 \\usepackage{colortbl}
\\usepackage{longtable}
\\usepackage{pdflscape}
\\usepackage{url}%s\n''' % defineGeometry(settingsForGeometry))
# \\usepackage[a4paper,bindingoffset=0.5cm,twoside,includehead,bottom=2cm]{geometry} 


def getTexAuthor():
   '''
      returns the author of the document
   '''
   return '''\\author{Gregor Hendel \\thanks{Konrad Zuse Institut f\"ur Informationstechnik Berlin,
               hendel@zib.de}}'''
               

def getTexPgfplotsset(settingspath, settings=''):
   '''
      returns a string which can be used in a LaTeX source file as PgfPlots set up. Use the empty string '' as settings to 
      have  a set of standard options for the settings path, or modify the settings.
      
      Keyword arguments:
      settingspath -- the Pgf path to store this setting as settingspath/.style, use e.g. settingspath='every axis'
                       to manipulate all plots in an axis environment.
      settings -- a string with PgfPlots style options for the specified Pgf path
      
   '''
   if settings == '':
      return '''\\pgfplotsset{
      %s/.style={
      width = \\linewidth,
      height = 5cm,
      xmin = 0,
      max space between ticks=50,
      ymin = 0,
      xlabel=Time(sec.),
      ylabel= Gap,
      axis x line=bottom,
      axis y line=left,
      const plot
      }
      }\n''' % settingspath
   else:
      return '''\\pgfplotsset{
      %s/.style={
      %s
      }
      }\n''' % (settingspath, settings)


def getTexSection(title):
   '''
      returns a string to open a new LaTeX Section.
      
      Keyword arguments:
      title -- the title of the section
   '''
   return "\\section{%s}\n" % Misc.getTexName(title)  

def defineGeometry(settingsForGeometry):
   '''
      returns a string which is used as definition for the geometry package in the method 'getTexHeader(...)'
      
      Keyword arguments:
      settingsForGeometry -- a string with package options for the geometry-package like, e.g., 'a4paper,bindingoffset=0.5cm,twoside'
   '''
   return '\\usepackage[%s]{geometry}\n' % settingsForGeometry


def getAxisPlot(dictOfPlots, dictOfSettings, settings='', legend=False):
   '''
      returns a string which can be embedded in a LaTeX file and produces a complete TikZ picture where all requested plots
      are drawn in a single axis environment. 
      
      Keyword arguments:
      dictOfPlots -- a dictionary of all plots in which each plot has to be represented via a list of (x,f(x)) 
      dictOfSetto
   '''
   theplots = {}
   for plotname in dictOfSettings.keys():
      theplots[plotname] = "\\addplot+[%s] coordinates {" % dictOfSettings[plotname]
      lastycoord = -1;
      lastelem = dictOfPlots[plotname][-1]
      for element in dictOfPlots[plotname]:
         xcoord, ycoord = element
         if ycoord != lastycoord or element == lastelem:
            theplots[plotname] += '(%5.1f,%4.1f)' % (xcoord, ycoord)
            lastycoord = ycoord
      
      else:
         theplots[plotname] += "} \\closedcycle ;"
         
   thefinalaxis = "\n\\begin{tikzpicture} \n"
   if usexlogaxis and useylogaxis:
      thefinalaxis += "\\begin{loglogaxis}"
   elif usexlogaxis:
      thefinalaxis += "\\begin{semilogxaxis}"
   elif useylogaxis:
      thefinalaxis += "\\begin{semilogyaxis}"
   else:
      thefinalaxis += "\\begin{axis}"
      
   thefinalaxis += "\n"
   if settings != '':
      thefinalaxis += '[%s]' % settings
   
   for plotname in dictOfPlots.keys():
      thefinalaxis += theplots[plotname] + "\n"
      if legend:
         thefinalaxis += "\\addlegendentry{%s}" % (Misc.getTexName(plotname)) + "\n"
   
   if usexlogaxis and useylogaxis:
      thefinalaxis += "\\end{loglogaxis}"
   elif usexlogaxis:
      thefinalaxis += "\\end{semilogxaxis}"
   elif useylogaxis:
      thefinalaxis += "\\end{semilogyaxis}"
   else:
      thefinalaxis += "\\end{axis}"
         
   thefinalaxis += "\n\\end{tikzpicture} \n"
   
   return thefinalaxis




def getNameOfPicFile(testrunname, probname):
   '''
      defines the name of a single picture file
      the file name is generic in the sense that its file extension is missing, but can be 
      accessed via the format-command of python, e.g., filename.format('.tex')
   '''
   return testrunname + probname + '{0}'

def checkPath(filename):
   ''' test if directory to contain a specified filename exists '''
   pathname = os.path.dirname(filename)
   if pathname == '':
      return True
   if not os.path.exists(pathname):
      print 'Making the directory', pathname
      os.makedirs(pathname)
      print os.path.exists(pathname), ' path existiert'
   return os.path.exists(pathname)
   
def testrungetGapAtTime(testrun, probname, time):
   listofpoints = testrun.problemGetData(probname, PrimalBoundHistoryReader.datakey)
   optimum = float(testrun.problemGetOptimalSolution(changeNameForSoluFile(probname)))
   if listofpoints == []:
      return cutoffGap;
   
   idx = -1
   while idx < len(listofpoints) -1 and float(listofpoints[idx + 1][0]) <= time:
      idx += 1 
   
   if idx == -1:
      assert float(listofpoints[0][0]) > time
      return cutoffGap
   
   assert float(listofpoints[idx][0]) <= time and (idx == len(listofpoints) - 1 or float(listofpoints[idx + 1][0]) > time)
   
   return Misc.getGap(float(listofpoints[idx][1]), optimum , useCplexGap=True)

def execute(comparator, myprobnamelist):
   
   THE_TIMELIMIT = max(comparator.testruns[0].problemlistGetData(myprobnamelist, TimeLimitReader.datakey))
   THE_TIMELIMIT = min(THE_TIMELIMIT, DEFAULT_TIMELIMIT)
   print "time limit for mean integrals set to", THE_TIMELIMIT
   # # mean integral evaluation points
   meanintegralpoints = int(THE_TIMELIMIT)
   scale = THE_TIMELIMIT / float(meanintegralpoints)

   therange = np.arange(meanintegralpoints)

   integralvalues = {}
   meanintegrals = {}
   virtualtestrun = np.zeros(meanintegralpoints)
   
   for testrun in comparator.testruns:
      integralvalues[testrun] = {}
      meanintegrals[testrun] = np.zeros(meanintegralpoints)
   
   nprocessedinstances = 0
   
   
   try:
      checkPath(integralsfilename)
      f = open(integralsfilename, 'w')
   except IOError:
      print 'File', integralsfilename, "doesn't exist!!!"
       
   # ## print head of LaTeX file
   f.write(getTexHeader('a4paper,bindingoffset=0.5cm,twoside,includehead,bottom=2cm') + "\n")

   f.write(getTexAuthor() + "\n")

   f.write(getTexPgfplotsset('every axis') + "\n")



   f.write('''\\begin{document}

   \centering

   ''')
   for probname in myprobnamelist :
      
      optimalsolution = comparator.testruns[0].problemGetOptimalSolution(changeNameForSoluFile(probname))

         
      optimalfloat = float(optimalsolution)
      
      f.write(getTexSection(probname))
      assert(optimalfloat != Misc.FLOAT_INFINITY), probname
      
      timelimithit = False;
      nprocessedinstances += 1
      
      for testrun in comparator.testruns:
         timelimithit = timelimithit or testrun.timeLimitHit(probname)
      
      optimum = comparator.testruns[0].problemGetOptimalSolution(changeNameForSoluFile(probname))
      optimum = float(optimum)
   
      maximumgap = max(1, getMaxGap(probname, comparator.testruns, optimum))
      if not timelimithit:
         maximumtime = getMaxTime(probname, comparator.testruns)
      else:
         maximumtime = comparator.testruns[0].problemGetData(probname, TimeLimitReader.datakey)
      
      thelocalsettings = '''xmax = %f,
                           ymax = %f''' % (maximumtime, maximumgap)
                           
      f.write(getTexPgfplotsset('local settings', thelocalsettings) + "\n")
      
      for testrun in comparator.testruns:
               
         mydatalist = testrun.problemGetData(probname, PrimalBoundHistoryReader.datakey)
         
         if mydatalist == []:
            firstpointintime = testrun.problemGetData(probname, TimeLimitReader.datakey);
            # continue
         else :
            time, bound = mydatalist[0];
            firstpointintime = float(time);
            
         dictOfPlots = {}
         dictOfSettings = {}
         firstplotname = 'firstplot'
         dictOfSettings[firstplotname] = 'fill=orange!40,mark=none,draw=red'
         dictOfPlots[firstplotname] = [(0.0, 1.1 * max(maximumgap, 1)), (firstpointintime, 1.1 * max(maximumgap, 1))]
         secondplotname = 'secondplot'
         dictOfPlots[secondplotname] = []
         dictOfSettings[secondplotname] = 'fill=orange'
         gap = 100.0
         for datatoken in mydatalist:
            time, bound = datatoken
            boundfloat = float(bound);
            gap = min(gap, Misc.getGap(boundfloat, optimalfloat, useCplexGap=True));
               
            dictOfPlots[secondplotname].append((float(time), gap))
               
         else:
            if testrun.timeLimitHit(probname) or gap > 10e-6:
               dictOfPlots[secondplotname].append((testrun.problemGetData(probname, TimeLimitReader.datakey), gap))
               
         solvingtime = float(testrun.problemGetData(probname, SolvingTimeReader.datakey))
         integralvalues[testrun][probname] = calcIntegralValue(mydatalist, optimalfloat, cutoffGap, min(solvingtime, testrun.problemGetData(probname, TimeLimitReader.datakey)))
         settings = '''local settings,
         title={settings: %s, integral value: %f}''' % (Misc.getTexName(testrun.getIdentification()), integralvalues[testrun][probname])
            
         f.write(getAxisPlot(dictOfPlots, dictOfSettings, settings))
      else:
         f.write("\\newpage")
            
   f.write('\\end{document}')



   for probname in myprobnamelist:
      for i in range(meanintegralpoints):
         theminattimeI = cutoffGap
         for testrun in comparator.testruns:
            gap = testrungetGapAtTime(testrun, probname, scale * i)
            
            meanintegrals[testrun][i] += gap 
            if  gap < theminattimeI:
               theminattimeI = gap
         virtualtestrun[i] += theminattimeI
         
      
   virtualtestrun /= float(nprocessedinstances)
   
   for testrun in comparator.testruns:
      meanintegrals[testrun] = meanintegrals[testrun] / float(nprocessedinstances)
      

     
   if writemeanintegrals:
      try:
         checkPath(meanintegralsfilename)
         f = open(meanintegralsfilename, 'w')
      except IOError:
         print 'File doesnt exist'
         
      colors = ['blue', 'red', 'green', 'yellow', 'black', 'gray']
      dictionary = {}
      settingsdictionary = {} 
      for testrun in comparator.testruns:
         thepointlist = []
         for i in therange:
            thepointlist.append((scale * i, meanintegrals[testrun][i]))
         
         ident = testrun.getIdentification()  # .split(':')[-1]
         dictionary[ident] = thepointlist
         settingsdictionary[ident] = '%s!50!black, fill=%s!30, mark=none, opacity=0.5' % (colors[comparator.testruns.index(testrun)%len(colors)], colors[comparator.testruns.index(testrun)%len(colors)])
      
      virtuallist = []
      for i in therange:
         virtuallist.append((scale * i, virtualtestrun[i]))
      dictionary['virtual'] = virtuallist
      settingsdictionary['virtual'] = '%s!50!black, fill=%s!30, mark=none, opacity=0.5' % (colors[-1], colors[-1])
      f.write(getTexHeader())
      # f.write('\pagestyle{empty}\n')
      f.write(getTexPgfplotsset('every axis'))
      f.write('''\\begin{document}''')
      f.write(getAxisPlot(dictionary, settingsdictionary, legend=True))
      
      for testrun in comparator.testruns:
         thepointlist = []
         for i in therange:
            thepointlist.append((scale * i, meanintegrals[testrun][i]))
         
         dictionary = {'theonlyplot' : thepointlist}    
         settingsdictionary = {'theonlyplot' : 'fill=orange, mark=none'}
         integralvalslist = []
         for probname in myprobnamelist:
            integralvalslist.append(integralvalues[testrun][probname])
         
         
            
         print "Mean integral for ", testrun.getIdentification(), ": ", np.sum(meanintegrals[testrun]) * scale
         f.write(getTexSection("Settings: %s, Integral value: %f" % (testrun.getIdentification(), np.sum(meanintegrals[testrun]) * scale)))
         f.write(getAxisPlot(dictionary, settingsdictionary))
      f.write("\\end{document}")   
      f.close()

## execute the main function of the module
if __name__ == "__main__":
   if filelist == []: ## don't continue if no .out file is given for evaluation
      execute = False
      print "Empty file list ---> Cancelling because nothing to do"
     
   if solufilename == '':
      execute = False
      print "Error: No .solu file specified"
   if execute:
      ## initialize the comparator object
      comparator = Comparator(filelist, listofreaders, testsetname, solufilename) 
      
      print "Collecting Data"
      comparator.collectData()

      myprobnamelist = comparator.probnamelist[:] ## copies the list of instance names found during data collection
      deletedproblems = [] ## list to record the deleted instances
      
      for probname in comparator.probnamelist: # search the instance list for problems which should be excluded
         index = myprobnamelist.index(probname)
        
         if probname in excludeProbs or comparator.excludeProb(changeNameForSoluFile(probname), excludereasons):
            print probname, "EXCLUDED"
            myprobnamelist.remove(probname)
            deletedproblems.append(probname)
      
      print "Calculating integrals and storing them to files ", integralsfilename, "and", meanintegralsfilename      
      execute(comparator, myprobnamelist)
      print "Total number of solutions: ", PrimalBoundHistoryReader.totalnumberofsols