import os
import sys
import numpy as np
from datetime import datetime
from collections import OrderedDict
import copy
from copy import deepcopy
from doris.doris_stack.main_code.resdata import ResData
from doris.doris_stack.main_code.dorisparameters import DorisParameters
import collections
from .jobs import Jobs
from doris.doris_stack.functions.baselines import baselines


class SingleMaster(object):

    def __init__(self, start_date='', end_date='', master_date='', stack_folder='', processing_folder='',
                 input_files=''):
        # This function loads in a datastack to create a single master stack. Optional are the start date, end date and
        # master date. If they are not defined all dates will be loaded. The master date can be loaded later using the
        # master function. If you want a random master value, choose master_date='random'
        # Dates should be given as 'yyyy-mm-dd'. If stack_read is True information is read from the stack_folder. Otherwise
        # the datastack from an StackData object should be given as input.

        Jobs.id = 0

        doris_parameters = DorisParameters(os.path.dirname(processing_folder)) # (assuming it is stored in the stackfolder
        self.doris_parameters = doris_parameters
        
        if not start_date:
            self.start_date = doris_parameters.start_date_default
        else:
            self.start_date = start_date
        if not end_date:
            self.end_date = doris_parameters.end_date_default
        else:
            self.end_date = end_date

        self.start_date = datetime.strptime(self.start_date,'%Y-%m-%d')
        self.end_date = datetime.strptime(self.end_date, '%Y-%m-%d')

        self.nr_of_jobs = doris_parameters.nr_of_jobs
        self.parallel = doris_parameters.parallel

        self.stack = dict()
        self.full_swath = dict()
        self.master_date = ''
        self.master_key = ''
        self.folder = processing_folder
        self.stack_folder = stack_folder

        self.doris_path = doris_parameters.doris_path
        self.cpxfiddle = doris_parameters.cpxfiddle_path  # '/...../cpxfiddle'
        self.function_path = doris_parameters.function_path
        self.input_files = input_files
        self.ESD_shift = dict()
        self.ESD_angle_pixel = dict()
        self.swath = dict()
        self.overlapping = []

        # Initialize ESD variables
        self.diff_matrix = dict()
        self.var_matrix = dict()
        self.to_angle_matrix = dict()
        self.weight_matrix = dict()

        if master_date:
            master_date = datetime.strptime(master_date, '%Y-%m-%d')
            self.master(master_date)

        if self.folder:
            self.processing_read()

        self.coreg_dates = [d for d in self.stack.keys() if d != self.master_date]

    def processing_read(self):
        # This function reads a processing datastack based on the processing folder

        folders = next(os.walk(self.folder))[1]
        folders = [fold for fold in folders if len(fold) == 8]

        self.stack = dict()

        for fold in folders:
            s_date = fold[:4] + '-' + fold[4:6] + '-' + fold[6:8]
            s_datetime = datetime.strptime(s_date,'%Y-%m-%d')

            if self.start_date <= s_datetime <= self.end_date:
                self.stack[s_date] = dict()
                self.full_swath[s_date] = dict()

                swaths = next(os.walk(os.path.join(self.folder, fold)))[1]
                swaths = [fol for fol in swaths if len(fol) == 7]

                for swath in swaths:
                    bursts = next(os.walk(os.path.join(self.folder, fold, swath)))[1]

                    for burst in bursts:
                        burst_name = swath + '_' + burst
                        self.stack[s_date][burst_name] = dict()

        self.read_res()

    def remove_finished(self, step='unwrap'):
        # Checks which processing folders are already finished based on the final step.
        # possible steps are ('dem_assist', 'comp_coregpm', 'interfero', 'coherence', 'subtr_refphase', 'subtr_refdem',
        # 'filtphase', 'unwrap', 'geocoding'
        self.read_res()

        for date in self.stack.keys():
            if 'ifgs' in self.full_swath[date].keys():
                if self.full_swath[date]['ifgs'].process_control[step] == '1':
                    self.coreg_dates.remove(date)

        print('Dates that will be processed are:')
        for date in self.coreg_dates:
            print(date)
        print('End list')

    def master(self,master_date):
        # Load master date
        self.master_date = master_date.strftime('%Y-%m-%d')
        self.master_key = self.master_date[:4] + self.master_date[5:7] + self.master_date[8:10]

        if not master_date in self.stack.keys():
            print('Master date is not part of the datastack. If you do not need to initialize anymore this is not a problem.')

    def baseline(self):
        # Create baseline plot of datastack. Usefull to select the right master
        baselines(self.stack_folder,self.start_date,self.end_date)

    def initialize(self, path='',cascade=False):
        # If path is not defined the stack will be initialized in the same folder as the datastack. This function does:
        # - copy .res and .raw files

        os.chdir(self.folder)

        for date in self.stack.keys():
            date_folder = self.image_path(date)
            if not os.path.exists(date_folder):
                os.mkdir(date_folder)

            for burst in self.stack[date].keys():
                swath_folder = self.swath_path(date, burst)
                if not os.path.exists(swath_folder):
                    os.mkdir(swath_folder)

                burst_folder = self.burst_path(date, burst, full_path=True)
                if not os.path.exists(burst_folder):
                    os.mkdir(burst_folder)
                os.chdir(burst_folder)

                # Copy data files
                m_source = self.burst_path(key=burst, date=self.master_date, dat_type='slave', full_path=True)
                m_dest = self.burst_path(key=burst, date=date, dat_type='master', full_path=True)

                if not os.path.exists(m_dest):
                    os.symlink(m_source, m_dest)

                # Write res files
                new_filename = os.path.join(burst_folder, 'master.res')
                if not os.path.exists(new_filename):
                    res = deepcopy(self.stack[self.master_date][burst]['slave'])
                    res.processes['crop']['Data_output_file'] = 'master' + res.processes['crop']['Data_output_file'][5:]
                    res.write(new_filename=new_filename)

        self.read_res()

        self.create_full_swath()
        self.coarse_orbits(dates=[self.master_date])   # Create ifgs.res files for master.

        del self.stack[self.master_date]
        del self.full_swath[self.master_date]

    def create_full_swath(self):
        # Create folders with full swath for individual interferogram.

        dates = list(self.stack.keys())

        # Change the res files.
        for date in dates:
            print(date)
            bursts = list(self.stack[date].keys())

            if 'slave' in self.full_swath[date].keys() and 'master' in self.full_swath[date].keys():
                continue

            for dat_type in ['master', 'slave']:
                self.full_swath[date][dat_type] = copy.deepcopy(self.stack[date][bursts[0]][dat_type])

                # Information slave images
                az_time = self.full_swath[date][dat_type].processes['readfiles']['First_pixel_azimuth_time (UTC)']
                az_time = datetime.strptime(az_time,'%Y-%b-%d %H:%M:%S.%f')
                range_time = float(self.full_swath[date][dat_type].processes['readfiles']['Range_time_to_first_pixel (2way) (ms)'])

                # First adjust pixel and range times for pixel (1,1)
                for burst in bursts:
                    az_burst = self.stack[date][burst][dat_type].processes['readfiles']['First_pixel_azimuth_time (UTC)']
                    az_burst = datetime.strptime(az_burst,'%Y-%b-%d %H:%M:%S.%f')
                    range_burst = float(self.stack[date][burst][dat_type].processes['readfiles']['Range_time_to_first_pixel (2way) (ms)'])

                    if az_burst < az_time:
                        az_time = az_burst
                    if range_burst < range_time:
                        range_time = range_burst

                az_time = az_time.strftime('%Y-%b-%d %H:%M:%S.%f')
                range_time = "{0:.15f}".format(range_time)
                self.full_swath[date][dat_type].processes['readfiles']['First_pixel_azimuth_time (UTC)'] = az_time
                self.full_swath[date][dat_type].processes['readfiles']['Range_time_to_first_pixel (2way) (ms)'] = range_time

                # Then change information on image size and crop.
                no_lines = self.full_swath[date][dat_type].processes['readfiles']['Number_of_lines_output_image']
                no_pixels = self.full_swath[date][dat_type].processes['readfiles']['Number_of_pixels_output_image']

                self.full_swath[date][dat_type].processes['readfiles']['Number_of_lines_original'] = no_lines
                self.full_swath[date][dat_type].processes['readfiles']['Number_of_pixels_original'] = no_pixels

                # Change readfiles
                self.full_swath[date][dat_type].processes['readfiles']['First_line (w.r.t. output_image)'] = '1'
                self.full_swath[date][dat_type].processes['readfiles']['Last_line (w.r.t. output_image)'] = no_lines
                self.full_swath[date][dat_type].processes['readfiles']['First_pixel (w.r.t. output_image)'] = '1'
                self.full_swath[date][dat_type].processes['readfiles']['Last_pixel (w.r.t. output_image)'] = no_pixels

                # Change in crop
                self.full_swath[date][dat_type].processes['crop']['First_line (w.r.t. original_image)'] = '1'
                self.full_swath[date][dat_type].processes['crop']['Last_line (w.r.t. original_image)'] = no_lines
                self.full_swath[date][dat_type].processes['crop']['First_pixel (w.r.t. original_image)'] = '1'
                self.full_swath[date][dat_type].processes['crop']['Last_pixel (w.r.t. original_image)'] = no_pixels
                if dat_type == 'master':
                    self.full_swath[date][dat_type].processes['crop']['Data_output_file'] = self.master_date + '.raw'
                else:
                    self.full_swath[date][dat_type].processes['crop']['Data_output_file'] = date + '.raw'
                self.full_swath[date][dat_type].processes['crop'].pop('First_line (w.r.t. tiff_image)')
                self.full_swath[date][dat_type].processes['crop'].pop('Last_line (w.r.t. tiff_image)')

                # Write data to folder
                folder = self.image_path(date)
                os.chdir(folder)

                # Create res_file
                master_path = self.image_path(date, dat_type + '.res')
                self.full_swath[date][dat_type].write(new_filename=master_path)

        self.read_res()

    def coarse_orbits(self, dates=[]):
        # Run coarse orbits for all bursts.

        if not dates:
            if len(self.coreg_dates) == 0:
                return
            dates = self.coreg_dates
            self.read_res(dates=self.coreg_dates)
            stack = self.stack
            full_swath = self.full_swath
        else:  # In the case we process for another date.
            stack = dict()
            full_swath = dict()
            self.read_res(dates=dates, image_stack=full_swath, burst_stack=stack)

        job_list1 = []
        job_list2 = []

        bursts = list(self.stack[dates[0]].keys())

        for date in dates:
            for burst in bursts:
                if 'ifgs' not in self.stack[date][burst].keys(): # If there is an ifgs file, coarse orbits are done...
                    path = self.burst_path(date, burst, full_path=True)
                    command2 = self.doris_path + ' ' + os.path.join(self.input_files, 'input.coarseorb')
                    job_list2.append({"path": path, "command": command2})
                    if not self.parallel:
                        os.chdir(path)
                        os.system(command2)
            if self.parallel:
                jobs = Jobs(self.nr_of_jobs, self.doris_parameters)
                jobs.run(job_list2)

            # Run coarse orbits for full swath
            if 'ifgs' not in self.full_swath[date].keys():  # If there is an ifgs file, coarse orbits are done...
                path = self.image_path(date)
                command1 = self.doris_path + ' ' + os.path.join(self.input_files, 'input.coarseorb')
                job_list1.append({"path": path, "command": command1})
                if not self.parallel:
                    os.chdir(path)
                    os.system(command1)
            if self.parallel:
                jobs = Jobs(self.nr_of_jobs, self.doris_parameters)
                jobs.run(job_list1)

    def coarse_correlation(self, ps=False):
        # Run coarse correlation.

        if len(self.coreg_dates) == 0:
            return
        self.read_res(dates=self.coreg_dates)

        for date in self.coreg_dates:
            job_list1 = []
            job_list2 = []
            for burst in self.stack[date].keys():
                if self.stack[date][burst]['ifgs'].process_control['coarse_correl'] != '1':
                    path = self.burst_path(date, burst, full_path=True)
                    os.chdir(path)
                    if ps is True:
                        master_file = self.burst_path(key=burst,dat_type='master',full_path=False)
                        command1 = sys.executable + ' -m ' + 'get_winpos' + ' ' + master_file + ' master.res 21 winpos_cc.asc'
                        job_list1.append({"path": path, "command": command1})
                        command2 = self.doris_path + ' ' + os.path.join(self.input_files, 'input.coarsecorr')
                        job_list2.append({"path": path, "command": command2})
                        if not self.parallel:
                            os.system(sys.executable + ' -m ' + 'get_winpos' + ' ' + master_file + ' master.res 21 winpos_cc.asc')
                            os.system(command2)
                    if ps is False:
                        command = self.doris_path + ' ' + os.path.join(self.input_files, 'input.coarsecorr')
                        job_list1.append({"path": path, "command": command})
                        if not self.parallel:
                            os.system(command)

            if (self.parallel):
                jobs = Jobs(self.nr_of_jobs, self.doris_parameters)
                jobs.run(job_list1)
                jobs.run(job_list2)

        self.fake_master_steps(step='coarse_correl', full_swath=False)

    def correct_coarse_correlation(self):
        # Correct coarse orbits to same reference system for whole image.

        if len(self.coreg_dates) == 0:
            return
        self.read_res(dates=self.coreg_dates)

        for date in self.coreg_dates:

            bursts = list(self.stack[date].keys())
            real_trans_p = []
            real_trans_l = []
            crop_shift_p = []
            crop_shift_l = []

            for burst in bursts:

                s_first_pix = self.stack[date][burst]['slave'].processes['readfiles']['First_pixel (w.r.t. output_image)']
                s_first_line = self.stack[date][burst]['slave'].processes['readfiles']['First_line (w.r.t. output_image)']
                m_first_pix = self.stack[date][burst]['master'].processes['readfiles']['First_pixel (w.r.t. output_image)']
                m_first_line = self.stack[date][burst]['master'].processes['readfiles']['First_line (w.r.t. output_image)']

                s_first_pix_c = self.stack[date][burst]['slave'].processes['crop']['First_pixel (w.r.t. original_image)']
                s_first_line_c = self.stack[date][burst]['slave'].processes['crop']['First_line (w.r.t. original_image)']
                m_first_pix_c = self.stack[date][burst]['master'].processes['crop']['First_pixel (w.r.t. original_image)']
                m_first_line_c = self.stack[date][burst]['master'].processes['crop']['First_line (w.r.t. original_image)']

                coarse_p = self.stack[date][burst]['ifgs'].processes['coarse_correl']['Coarse_correlation_translation_pixels']
                coarse_l = self.stack[date][burst]['ifgs'].processes['coarse_correl']['Coarse_correlation_translation_lines']

                crop_p = int(s_first_pix) - int(m_first_pix) - int(s_first_pix_c) + int(m_first_pix_c)
                crop_l = int(s_first_line) - int(m_first_line) - int(s_first_line_c) + int(m_first_line_c)
                crop_shift_p.append(crop_p)
                crop_shift_l.append(crop_l)
                real_trans_p.append(int(coarse_p) + crop_p)
                real_trans_l.append(int(coarse_l) + crop_l)

            im_trans_p = int(round(np.median(real_trans_p)))
            im_trans_l = int(round(np.median(real_trans_l)))

            for burst, p_shift, l_shift in zip(bursts, crop_shift_p, crop_shift_l):

                trans_l = str(im_trans_l - l_shift)
                trans_p = str(im_trans_p - p_shift)
                self.stack[date][burst]['ifgs'].processes['coarse_correl']['Coarse_correlation_translation_pixels'] = trans_p
                self.stack[date][burst]['ifgs'].processes['coarse_correl']['Coarse_correlation_translation_lines'] = trans_l
                self.stack[date][burst]['ifgs'].processes['coarse_correl']['Initial_Offset_CoarseCorr_pixels'] = trans_p
                self.stack[date][burst]['ifgs'].processes['coarse_correl']['Initial_Offset_CoarseCorr_lines'] = trans_l
                self.stack[date][burst]['ifgs'].processes['coarse_correl']['Slope_CoarseCorr_pixels'] = '0'
                self.stack[date][burst]['ifgs'].processes['coarse_correl']['Slope_CoarseCorr_lines'] = '0'
            self.full_swath[date]['ifgs'].processes['coarse_orbits']['Coarse_orbits_translation_pixels'] = str(im_trans_p)
            self.full_swath[date]['ifgs'].processes['coarse_orbits']['Coarse_orbits_translation_lines'] = str(im_trans_l)

        self.update_res(dates=self.coreg_dates)

        self.fake_master_steps(step='coarse_correl', burst_proc=False)

    def deramp(self, master=True):
        # Deramp slave and masters and slaves of bursts.

        if len(self.coreg_dates) == 0:
            return
        self.read_res(dates=self.coreg_dates)

        job_list1 = []
        job_list2 = []

        # Deramp slaves
        bursts = list(self.stack[self.coreg_dates[0]].keys())

        for date in self.coreg_dates:
            for burst in bursts:
                path = self.burst_path(date, burst, full_path=True)
                slave_file = self.burst_path(key=burst, dat_type='slave', full_path=False)
                slave_deramped = self.burst_path(key=burst, dat_type='slave_deramped', full_path=False)

                if not os.path.exists(os.path.join(path, slave_deramped)):
                    command2 = sys.executable + ' ' + os.path.join(self.function_path, 'do_deramp_SLC.py') + ' ' + slave_file + ' slave.res'
                    job_list2.append({"path": path, "command": command2})
                    if not self.parallel:
                        os.chdir(path)
                        os.system(command2)

                if self.stack[date][burst]['slave'].processes['crop']['Data_output_file'] != os.path.basename(slave_deramped):
                    self.stack[date][burst]['slave'].processes['crop']['Data_output_file'] = os.path.basename(slave_deramped)
                if self.stack[date][burst]['slave'].processes['readfiles']['deramp'] != '1':
                    self.stack[date][burst]['slave'].processes['readfiles']['deramp'] = '1'
                if self.stack[date][burst]['master'].processes['readfiles']['reramp'] != '0':
                    self.stack[date][burst]['master'].processes['readfiles']['reramp'] = '0'

        # Deramp master
        date = self.master_date

        for burst in bursts:
            path = self.burst_path(date, burst, full_path=True)
            master_file = self.burst_path(key=burst, dat_type='slave', full_path=False)
            master_deramped = self.burst_path(key=burst, dat_type='slave_deramped', full_path=False)

            if not os.path.exists(os.path.join(path, master_deramped)) or not master:
                command1 = sys.executable + ' ' + os.path.join(self.function_path, 'do_deramp_SLC.py') + ' ' + master_file + ' slave.res'
                job_list1.append({"path": path, "command": command1})
                if not self.parallel:
                    os.chdir(path)
                    os.system(command1)

        if self.parallel:
            jobs = Jobs(self.nr_of_jobs, self.doris_parameters)
            jobs.run(job_list1)
            jobs.run(job_list2)

        # Create links for master if needed.
        for burst in bursts:
            master_file = self.burst_path(key=burst, date=date, dat_type='slave_deramped', full_path=True)

            for date_slave in self.coreg_dates:
                slave_file = self.burst_path(key=burst, date=date_slave, dat_type='master_deramped', full_path=True)
                if not os.path.exists(slave_file):
                    os.symlink(master_file, slave_file)
                if self.stack[date_slave][burst]['master'].processes['crop']['Data_output_file'] != os.path.basename(slave_file):
                    self.stack[date_slave][burst]['master'].processes['crop']['Data_output_file'] = os.path.basename(slave_file)
                if self.stack[date_slave][burst]['master'].processes['readfiles']['deramp'] != '1':
                    self.stack[date_slave][burst]['master'].processes['readfiles']['deramp'] = '1'
                if self.stack[date_slave][burst]['master'].processes['readfiles']['reramp'] != '0':
                    self.stack[date_slave][burst]['master'].processes['readfiles']['reramp'] = '0'

        self.update_res(dates=self.coreg_dates)

    def icc_burst(self, ps=False):
        # Do the icc per burst

        if len(self.coreg_dates) == 0:
            return
        self.read_res(dates=self.coreg_dates)

        job_list1 = []
        job_list2 = []

        for date in self.coreg_dates:
            for burst in self.stack[date].keys():
                if self.stack[date][burst]['ifgs'].process_control['fine_coreg'] != '1':
                    path = self.burst_path(date, burst, full_path=True)
                    master_file = self.burst_path(key=burst,dat_type='master',full_path=False)
                    if not(self.parallel):
                        os.chdir(path)
                    if ps == True:
                        command1 = sys.executable + ' -m' + 'get_winpos'  + ' ' + master_file + ' master.res 101 winpos_fine.asc'
                        job_list1.append({"path": path, "command": command1})
                        command2 = self.doris_path + ' ' + os.path.join(self.input_files,'input.finecoreg_icc_pointscat')
                        job_list2.append({"path": path, "command": command2})
                        if (not(self.parallel)):
                            os.system(command1)
                            os.system(command2)
                    elif ps == False:
                        command = self.doris_path + ' ' + os.path.join(self.input_files,'input.finecoreg')
                        job_list1.append({"path": path, "command": command})
                        if not (self.parallel):
                            os.system(command)
        if self.parallel:
            jobs = Jobs(self.nr_of_jobs, self.doris_parameters)
            jobs.run(job_list1)
            jobs.run(job_list2)

        self.fake_master_steps(step='fine_coreg', full_swath=False)

    def coreg_full_swath(self):
        # Do the combined icc and dem coregistration for the full swath

        # First read all .res files again.
        if len(self.coreg_dates) == 0:
            return
        self.read_res(dates=self.coreg_dates)

        for date in self.coreg_dates:
            # We start by adding the windows of the first burst.
            no_offset = 0
            bursts = list(self.stack[date].keys())
            new_icc = copy.deepcopy(self.stack[date][bursts[0]]['ifgs'].processes['fine_coreg'])

            im_trans_p = self.full_swath[date]['ifgs'].processes['coarse_orbits']['Coarse_orbits_translation_pixels']
            im_trans_l = self.full_swath[date]['ifgs'].processes['coarse_orbits']['Coarse_orbits_translation_lines']

            for burst in bursts:

                icc = self.stack[date][burst]['ifgs'].processes['fine_coreg']
                position = self.stack[date][burst]['master'].processes['readfiles']

                trans_p = self.stack[date][burst]['ifgs'].processes['coarse_correl']['Coarse_correlation_translation_pixels']
                trans_l = self.stack[date][burst]['ifgs'].processes['coarse_correl']['Coarse_correlation_translation_lines']
                p_shift_offset = int(im_trans_p) - int(trans_p)
                l_shift_offset = int(im_trans_l) - int(trans_l)

                p_offset = int(position['First_pixel (w.r.t. output_image)']) - 1
                l_offset = int(position['First_line (w.r.t. output_image)']) - 1
                window_no = int(icc['Number_of_correlation_windows'])

                for row in range(1,window_no+1):
                    dat = copy.deepcopy(icc['row_' + str(row)])
                    dat[0] = str(no_offset)
                    dat[1] = str(int(dat[1]) + l_offset)
                    dat[2] = str(int(dat[2]) + p_offset)
                    dat[3] = str(float(dat[3]) + float(l_shift_offset))
                    dat[4] = str(float(dat[4]) + float(p_shift_offset))
                    new_icc['row_' + str(no_offset + 1)] = dat

                    no_offset += 1

            new_icc['Number_of_correlation_windows'] = str(no_offset)

            # Finally save to .res file
            self.full_swath[date]['ifgs'].insert(new_icc,'fine_coreg')
            # And write .res file
            res_path = self.image_path(date,file_path='ifgs.res')
            self.full_swath[date]['ifgs'].write(new_filename=res_path)

        self.fake_master_steps(step='fine_coreg', burst_proc=False)

    def dac_bursts(self):
        # Do the DEM coregistration and coregpm for the full swath

        if len(self.coreg_dates) == 0:
            return
        self.read_res(dates=self.coreg_dates)

        for date in self.coreg_dates:
            job_list = []
            for burst in self.stack[date].keys():
                # If this step is not run yet.
                if self.stack[date][burst]['ifgs'].process_control['dem_assist'] != '1':
                    path = self.burst_path(date, burst, full_path=True)
                    command = self.doris_path + ' ' + os.path.join(self.input_files,'input.dembased')
                    job_list.append({"path": path, "command": command})
                    if not self.parallel:
                        os.chdir(path)
                        os.system(command)
            if (self.parallel):
                jobs = Jobs(self.nr_of_jobs, self.doris_parameters)
                jobs.run(job_list)

        self.fake_master_steps(step='dem_assist', full_swath=False)

    def coreg_bursts(self,no_poly=True):
        # Write coregistration results from full swath to individual bursts

        if len(self.coreg_dates) == 0:
            return
        self.read_res(dates=self.coreg_dates)

        for date in self.coreg_dates:
            # First read the polynomials and normalization lines/pixels from full swath

            path = self.image_path(date)
            os.chdir(path)
            os.system(self.doris_path + ' ' + os.path.join(self.input_files,'input.coregpm'))

            self.read_res(dates=[date])

            coreg = copy.deepcopy(self.full_swath[date]['ifgs'].processes['comp_coregpm'])
            norm_line = [float(coreg['Normalization_Lines'].split()[0]),float(coreg['Normalization_Lines'].split()[1])]
            norm_pix = [float(coreg['Normalization_Pixels'].split()[0]),float(coreg['Normalization_Pixels'].split()[1])]
            degree = int(coreg['Degree_cpm'])

            La = 0; Lb = 0; Lc = 0; Ld = 0; Le = 0; Lf = 0
            Pa = 0; Pb = 0; Pc = 0; Pd = 0; Pe = 0; Pf = 0

            # Load the polynomial from the full swath
            if degree == 0 and no_poly == False:
                Lf = float(coreg['row_0'][0])
                Pf = float(coreg['row_1'][0])
            if degree == 1 and no_poly == False:
                Lf = float(coreg['row_0'][0])
                Le = float(coreg['row_1'][0])
                Ld = float(coreg['row_2'][0])
                Pf = float(coreg['row_3'][0])
                Pe = float(coreg['row_4'][0])
                Pd = float(coreg['row_5'][0])
            if degree == 2 and no_poly == False:
                Lf = float(coreg['row_0'][0])
                Le = float(coreg['row_1'][0])
                Ld = float(coreg['row_2'][0])
                Lc = float(coreg['row_4'][0])
                Lb = float(coreg['row_3'][0])
                La = float(coreg['row_5'][0])
                Pf = float(coreg['row_6'][0])
                Pe = float(coreg['row_7'][0])
                Pd = float(coreg['row_9'][0])
                Pc = float(coreg['row_8'][0])
                Pb = float(coreg['row_10'][0])
                Pa = float(coreg['row_11'][0])

            for burst in self.stack[date].keys():
                # Now convert to burst coreg using the pixel and line offset
                line_burst = int(self.stack[date][burst]['master'].processes['readfiles']['First_line (w.r.t. output_image)'])
                pixel_burst = int(self.stack[date][burst]['master'].processes['readfiles']['First_pixel (w.r.t. output_image)'])

                # And convert to an offset in the [-2,2] domain
                l0 = (line_burst-norm_line[0]) / (norm_line[1]-norm_line[0]) * 4
                p0 = (pixel_burst-norm_pix[0]) / (norm_pix[1]-norm_pix[0]) * 4

                # Finally convert variables. We assume a 2 degree polynomial.
                # y = ax'^2+bz'^2+cx'z'+(2ax0+cz0+d)x'+(2bz0+cx0+e)z'+(a0^2+bz0^2+cx0z0+dx0+ez0+f)
                p_poly = [0,0,0,0,0,0]
                p_poly[0] = Pa*p0**2 + Pb*l0**2 + Pc*p0*l0 + Pd*p0 + Pe*l0 + Pf
                p_poly[1] = 2*Pb*l0 + Pc*p0 + Pe
                p_poly[2] = 2*Pa*l0 + Pc*p0 + Pd
                p_poly[3] = Pc
                p_poly[4] = Pb
                p_poly[5] = Pa

                l_poly = [0,0,0,0,0,0]
                l_poly[0] = La*l0**2 + Lb*p0**2 + Lc*p0*l0 + Ld*l0 + Le*p0 + Lf
                l_poly[1] = 2*Lb*p0 + Lc*l0 + Le
                l_poly[2] = 2*La*p0 + Lc*l0 + Ld
                l_poly[3] = Lc
                l_poly[4] = Lb
                l_poly[5] = La

                # lambda function for pixel and line coordinates
                l_eq = lambda l,p: l_poly[5]*l**2 + l_poly[4]*p**2 + l_poly[3]*l*p + l_poly[2]*l + l_poly[1]*p + l_poly[0]
                p_eq = lambda l,p: p_poly[5]*p**2 + p_poly[4]*l**2 + p_poly[3]*l*p + p_poly[2]*p + p_poly[1]*l + p_poly[0]

                # Save new coregistration function to burst
                coreg['Degree_cpm'] = str(degree)
                if degree == 0:
                    coreg['row_0'] = ["{0:.8e}".format(l_poly[0]), '0', '0']
                    coreg['row_1'] = ["{0:.8e}".format(p_poly[0]), '0', '0']
                if degree == 1:
                    coreg['row_0'] = ["{0:.8e}".format(l_poly[0]), '0', '0']
                    coreg['row_1'] = ["{0:.8e}".format(l_poly[1]), '1', '0']
                    coreg['row_2'] = ["{0:.8e}".format(l_poly[2]), '0', '1']
                    coreg['row_3'] = ["{0:.8e}".format(p_poly[0]), '0', '0']
                    coreg['row_4'] = ["{0:.8e}".format(p_poly[1]), '1', '0']
                    coreg['row_5'] = ["{0:.8e}".format(p_poly[2]), '0', '1']
                if degree == 2:
                    coreg['row_0'] = ["{0:.8e}".format(l_poly[0]), '0', '0']
                    coreg['row_1'] = ["{0:.8e}".format(l_poly[1]), '1', '0']
                    coreg['row_2'] = ["{0:.8e}".format(l_poly[2]), '0', '1']
                    coreg['row_3'] = ["{0:.8e}".format(l_poly[4]), '2', '0']
                    coreg['row_4'] = ["{0:.8e}".format(l_poly[3]), '1', '1']
                    coreg['row_5'] = ["{0:.8e}".format(l_poly[5]), '0', '2']
                    coreg['row_6'] = ["{0:.8e}".format(p_poly[0]), '0', '0']
                    coreg['row_7'] = ["{0:.8e}".format(p_poly[1]), '1', '0']
                    coreg['row_8'] = ["{0:.8e}".format(p_poly[2]), '0', '1']
                    coreg['row_9'] = ["{0:.8e}".format(p_poly[4]), '2', '0']
                    coreg['row_10'] = ["{0:.8e}".format(p_poly[3]), '1', '1']
                    coreg['row_11'] = ["{0:.8e}".format(p_poly[5]), '0', '2']

                coreg['Deltaline_slave00_poly'] = "{0:.8e}".format(-l_eq(-2.0, -2.0))
                coreg['Deltapixel_slave00_poly'] = "{0:.8e}".format(-p_eq(-2.0, -2.0))
                coreg['Deltaline_slave0N_poly'] = "{0:.8e}".format(-l_eq(-2.0, 2.0))
                coreg['Deltapixel_slave0N_poly'] = "{0:.8e}".format(-p_eq(-2, 2.0))
                coreg['Deltaline_slaveN0_poly'] = "{0:.8e}".format(-l_eq(2.0, -2.0))
                coreg['Deltapixel_slaveN0_poly'] = "{0:.8e}".format(-p_eq(2.0, -2.0))
                coreg['Deltaline_slaveNN_poly'] = "{0:.8e}".format(-l_eq(2.0, 2.0))
                coreg['Deltapixel_slaveNN_poly'] = "{0:.8e}".format(-p_eq(2.0, 2.0))

                # Finally add the Normalization lines / pixels
                lines = (int(self.stack[date][burst]['master'].processes['crop']['Last_line (w.r.t. original_image)']) -
                         int(self.stack[date][burst]['master'].processes['crop']['First_line (w.r.t. original_image)']))
                pixels = (int(self.stack[date][burst]['master'].processes['crop']['Last_pixel (w.r.t. original_image)']) -
                         int(self.stack[date][burst]['master'].processes['crop']['First_pixel (w.r.t. original_image)']))

                # Save pixels lines
                coreg['Normalization_Lines'] = "{0:.8e}".format(1) + ' ' + "{0:.8e}".format(lines)
                coreg['Normalization_Pixels'] = "{0:.8e}".format(1) + ' ' + "{0:.8e}".format(pixels)

                # Copy coregistration from full swath to burst
                try:
                    self.stack[date][burst]['ifgs'].insert(coreg,'comp_coregpm')
                except:
                    self.stack[date][burst]['ifgs'].update(coreg,'comp_coregpm')

            self.update_res(dates=[date])

        # Save .res files.
        self.update_res(dates=self.coreg_dates)

    def fake_fine_coreg(self):
        # This function is used if only geometrical coregistatration is used.

        if len(self.coreg_dates) == 0:
            return
        self.read_res(dates=self.coreg_dates)

        coreg = OrderedDict()
        coreg['Initial offsets (l,p)'] = '0, 0'
        coreg['Window_size_L_for_correlation'] = '64'
        coreg['Window_size_P_for_correlation'] = '64'
        coreg['Max. offset that can be estimated'] = '32'
        coreg['Peak search ovs window (l,p)'] = '16 , 16'
        coreg['Oversampling factor'] = '32'
        coreg['Number_of_correlation_windows'] = '0'

        for date in self.coreg_dates:
            for burst in self.stack[date].keys():
                # Insert fake coregistration
                if not self.stack[date][burst]['ifgs'].process_control['fine_coreg'] == '1':
                    self.stack[date][burst]['ifgs'].insert(coreg,'fine_coreg')

        self.update_res(dates=self.coreg_dates)

        self.fake_master_steps(step='fine_coreg', full_swath=False)

    def fake_coregmp(self):
        # This function is used if only geometrical coregistatration is used.

        if len(self.coreg_dates) == 0:
            return
        self.read_res(dates=self.coreg_dates)

        coreg = OrderedDict()
        coreg['Degree_cpm'] = '0'
        coreg['Normalization_Lines'] = ''
        coreg['Normalization_Pixels'] = ''
        coreg['Estimated_coefficientsL'] = ''
        coreg['row_0'] = ["{0:.8e}".format(0), '0', '0']
        coreg['Estimated_coefficientsP'] = ''
        coreg['row_1'] = ["{0:.8e}".format(0), '0', '0']

        coreg['Deltaline_slave00_poly'] = "{0:.8e}".format(0)
        coreg['Deltapixel_slave00_poly'] = "{0:.8e}".format(0)
        coreg['Deltaline_slave0N_poly'] = "{0:.8e}".format(0)
        coreg['Deltapixel_slave0N_poly'] = "{0:.8e}".format(0)
        coreg['Deltaline_slaveN0_poly'] = "{0:.8e}".format(0)
        coreg['Deltapixel_slaveN0_poly'] = "{0:.8e}".format(0)
        coreg['Deltaline_slaveNN_poly'] = "{0:.8e}".format(0)
        coreg['Deltapixel_slaveNN_poly'] = "{0:.8e}".format(0)

        for date in self.coreg_dates:
            for burst in self.stack[date].keys():
                # Now convert to burst coreg using the pixel and line offset
                lines = (int(self.stack[date][burst]['master'].processes['crop']['Last_line (w.r.t. original_image)']) -
                         int(self.stack[date][burst]['master'].processes['crop']['First_line (w.r.t. original_image)']))
                pixels = (int(self.stack[date][burst]['master'].processes['crop']['Last_pixel (w.r.t. original_image)']) -
                         int(self.stack[date][burst]['master'].processes['crop']['First_pixel (w.r.t. original_image)']))

                # Save pixels lines
                coreg['Normalization_Lines'] = "{0:.8e}".format(1) + ' ' + "{0:.8e}".format(lines)
                coreg['Normalization_Pixels'] = "{0:.8e}".format(1) + ' ' + "{0:.8e}".format(pixels)

                # Copy coregistration from full swath to burst
                if not self.stack[date][burst]['ifgs'].process_control['comp_coregpm'] == '1':
                    self.stack[date][burst]['ifgs'].insert(coreg,'comp_coregpm')

        self.update_res(dates=self.coreg_dates)

        self.fake_master_steps(step='comp_coregpm', full_swath=False)

    def dac_full_swath(self):
        # This function reads the dem shift result files from the full swath and saves them to both data and result
        # files of individual bursts.

        if len(self.coreg_dates) == 0:
            return
        self.read_res(dates=self.coreg_dates)

        for date in self.coreg_dates:
            for burst in self.stack[date].keys():

                master_dat = self.stack[date][burst]['master'].processes['crop']
                lines = int(master_dat['Last_line (w.r.t. original_image)']) - int(master_dat['First_line (w.r.t. original_image)'])
                pixels = int(master_dat['Last_pixel (w.r.t. original_image)']) - int(master_dat['First_pixel (w.r.t. original_image)'])
                ref_offset_p = int(self.full_swath[date]['ifgs'].processes['coarse_orbits']['Coarse_orbits_translation_pixels'])
                ref_offset_l = int(self.full_swath[date]['ifgs'].processes['coarse_orbits']['Coarse_orbits_translation_lines'])
                offset_p = int(self.stack[date][burst]['ifgs'].processes['coarse_correl']['Coarse_correlation_translation_pixels'])
                offset_l = int(self.stack[date][burst]['ifgs'].processes['coarse_correl']['Coarse_correlation_translation_lines'])

                file_path = self.burst_path(date, burst, file_path='dac_delta_pixel.raw', full_path=True)

                if not os.path.exists(file_path + '.new'):
                    d_pixel = np.memmap(file_path, dtype=np.dtype('float64'), shape=(lines+1,pixels+1))
                    n_pixel = np.memmap(file_path + '.new', mode='w+', dtype=np.dtype('float64'), shape=(lines+1,pixels+1))
                    n_pixel[:,:] = d_pixel[:,:] - (offset_p - ref_offset_p)
                    n_pixel.flush()

                if not os.path.exists(file_path + '.new'):
                    file_path = self.burst_path(date, burst, file_path='dac_delta_line.raw', full_path=True)
                    d_line = np.memmap(file_path, dtype=np.dtype('float64'), shape=(lines+1,pixels+1))
                    n_line = np.memmap(file_path + '.n', mode='w+', dtype=np.dtype('float64'), shape=(lines+1,pixels+1))
                    n_line[:,:] = d_line[:,:] - (offset_l - ref_offset_l)
                    n_line.flush()

        # Write delta line/pixel to burst folder
        self.concatenate('dac_delta_line.raw.new', 'dac_delta_line.raw.new',dt=np.dtype('float64'))
        self.concatenate('dac_delta_pixel.raw.new', 'dac_delta_pixel.raw.new',dt=np.dtype('float64'))

        for date in self.coreg_dates:

            bursts = list(self.stack[date].keys())

            res_dem = deepcopy(self.stack[date][bursts[0]]['ifgs'].processes['dem_assist'])
            master_crop = deepcopy(self.full_swath[date]['master'].processes['crop'])

            # Update fields to dimensions of master burst.
            res_dem['First_line (w.r.t. original_master)'] = master_crop['First_line (w.r.t. original_image)']
            res_dem['Last_line (w.r.t. original_master)'] = master_crop['Last_line (w.r.t. original_image)']
            res_dem['First_pixel (w.r.t. original_master)'] = master_crop['First_pixel (w.r.t. original_image)']
            res_dem['Last_pixel (w.r.t. original_master)'] = master_crop['Last_pixel (w.r.t. original_image)']
            lines = int(master_crop['Last_line (w.r.t. original_image)']) - int(master_crop['First_line (w.r.t. original_image)'])
            res_dem['Number of lines'] = str(lines + 1)
            pixels = int(master_crop['Last_pixel (w.r.t. original_image)']) - int(master_crop['First_pixel (w.r.t. original_image)'])
            res_dem['Number of pixels'] = str(pixels + 1)

            # Load image data
            file_path = self.image_path(date, file_path='dac_delta_pixel.raw')
            command = 'mv ' + file_path + '.new ' + file_path
            os.system(command)
            d_pixel = np.memmap(file_path, dtype=np.dtype('float64'), shape=(lines+1,pixels+1))
            file_path = self.image_path(date, file_path='dac_delta_line.raw')
            command = 'mv ' + file_path + '.new ' + file_path
            os.system(command)
            d_line = np.memmap(file_path, dtype=np.dtype('float64'), shape=(lines+1,pixels+1))

            # Correct for corner information.
            res_dem['Number of pixels'] = str(pixels + 1)
            res_dem['Deltaline_slave00_dem'] = str(-d_line[0,0])
            res_dem['Deltapixel_slave00_dem'] = str(-d_pixel[0,0])
            res_dem['Deltaline_slave0N_dem'] = str(-d_line[0,-1])
            res_dem['Deltapixel_slave0N_dem'] = str(-d_pixel[0,-1])
            res_dem['Deltaline_slaveN0_dem'] = str(-d_line[-1,0])
            res_dem['Deltapixel_slaveN0_dem'] = str(-d_pixel[-1,0])
            res_dem['Deltaline_slaveNN_dem'] = str(-d_line[-1,-1])
            res_dem['Deltapixel_slaveNN_dem'] = str(-d_pixel[-1,-1])

            self.full_swath[date]['ifgs'].insert(res_dem,process='dem_assist')

        self.update_res(dates=self.coreg_dates)

        self.fake_master_steps(step='dem_assist', burst_proc=False)

    def resample(self, type=''):
        # Resample slave bursts

        if len(self.coreg_dates) == 0:
            return

        jobList1 = []
        jobList2 = []

        for date in self.coreg_dates:
            for burst in self.stack[date].keys():

                if self.stack[date][burst]['slave'].process_control['resample'] != '1':
                    path = self.burst_path(date, burst, full_path=True)
                    command1 = self.doris_path + ' ' + os.path.join(self.input_files, 'input.resample')

                    jobList1.append({"path": path, "command": command1})

                    if not self.parallel:
                        os.chdir(path)
                        # Resample
                        os.system(command1)

        if self.parallel:
            jobs = Jobs(self.nr_of_jobs, self.doris_parameters)

            jobs.run(jobList1)
            jobs.run(jobList2)

    def reramp(self, type=''):
        # This function reramps the radar data. If master is True, we assume that there is still an original master file
        # Which means that it is not needed to reramp that one. If master is false, only the slave is reramped.

        if len(self.coreg_dates) == 0:
            return
        self.read_res(dates=self.coreg_dates)

        bursts = list(self.stack[self.coreg_dates[0]].keys())

        jobList1 = []

        for date in self.coreg_dates:
            for burst in self.stack[date].keys():
                path = self.burst_path(date, burst, full_path=True)

                if not os.path.exists(os.path.join(path, 'slave_rsmp_reramped.raw')):
                    # If we are before the ESD step and reramp is not jet done.
                    command1 = sys.executable + ' ' + os.path.join(self.function_path, 'do_reramp_SLC.py') + ' slave_rsmp.raw slave.res'
                    jobList1.append({"path": path, "command": command1})
                    if not self.parallel:
                        os.chdir(path)
                        os.system(command1)

                if self.stack[date][burst]['slave'].processes['resample']['Data_output_file'] != 'slave_rsmp_reramped.raw':
                    self.stack[date][burst]['slave'].processes['resample']['Data_output_file'] = 'slave_rsmp_reramped.raw'
                if self.stack[date][burst]['slave'].processes['readfiles']['reramp'] != '1':
                    self.stack[date][burst]['slave'].processes['readfiles']['reramp'] = '1'

        # Create links for master if needed.
        for burst in bursts:
            for date in self.coreg_dates:
                # TODO If steps like simamp are added, we have to link back to these files.
                slave_file = self.burst_path(key=burst, date=date, dat_type='master', full_path=True)
                if self.stack[date][burst]['master'].processes['crop']['Data_output_file'] != os.path.basename(slave_file):
                    self.stack[date][burst]['master'].processes['crop']['Data_output_file'] = os.path.basename(slave_file)
                if self.stack[date][burst]['master'].processes['readfiles']['reramp'] != '1':
                    self.stack[date][burst]['master'].processes['readfiles']['reramp'] = '1'

        if self.parallel:
            jobs = Jobs(self.nr_of_jobs, self.doris_parameters)
            jobs.run(jobList1)

        self.update_res(dates=self.coreg_dates)

        self.fake_master_steps(step='resample')

    def fake_master_resample(self):
        # This script fakes a resample step for the master file (this is of course not really needed)
        # This will save us a lot of time in exception handling in steps later on....
        # - create a master.res / copy slave.res
        # - add resampling step (We assume a reramped result)
        # - add slave_rsmp.raw and slave_rsmp_reramped.raw in complex64 format.
        # - add the master original and deramp step same as the slave file.

        date = self.master_date
        date_1 = list(self.stack.keys())[0]
        bursts = list(self.stack[date_1].keys())
        burst_res = dict()
        image_res = dict()
        self.read_res(dates=[self.master_date], bursts=bursts, burst_stack=burst_res, image_stack=image_res)

        for burst in self.stack[self.coreg_dates[0]].keys():
            # burst_path
            if not burst_res[date][burst]['slave'].process_control['resample'] == '1':
                burst_res[date][burst]['slave'].insert(self.stack[date_1][burst]['slave'].processes['resample'], 'resample')

            # Now create symlink to master data
            slave_dat = self.burst_path(self.master_date, burst, full_path=True, dat_type='slave')
            slave_deramped_dat = self.burst_path(self.master_date, burst, full_path=True, dat_type='slave_deramped')
            master_dat = self.burst_path(self.master_date, burst, full_path=True, dat_type='master')
            master_deramped_dat = self.burst_path(self.master_date, burst, full_path=True, dat_type='master_deramped')

            if not os.path.exists(master_dat):
                os.symlink(slave_dat, master_dat)
            if not os.path.exists(master_deramped_dat):
                os.symlink(slave_deramped_dat, master_deramped_dat)

            # Finally copy the resampled files in complex64 format.
            resample_dat = self.burst_path(self.master_date, burst, full_path=True, file_path='slave_rsmp.raw')
            resample_reramped_dat = self.burst_path(self.master_date, burst, full_path=True, file_path='slave_rsmp_reramped.raw')

            lines = int(burst_res[date][burst]['slave'].processes['readfiles']['Last_line (w.r.t. output_image)']) - \
                    int(burst_res[date][burst]['slave'].processes['readfiles']['First_line (w.r.t. output_image)']) + 1
            pixels = int(burst_res[date][burst]['slave'].processes['readfiles']['Last_pixel (w.r.t. output_image)']) - \
                     int(burst_res[date][burst]['slave'].processes['readfiles']['First_pixel (w.r.t. output_image)']) + 1
            dtype = np.dtype([('re', np.int16), ('im', np.int16)])

            if not os.path.exists(resample_dat):
                resample = np.memmap(resample_dat, dtype='complex64', mode='w+', shape=(lines, pixels))
                slc_dat = np.memmap(slave_deramped_dat, dtype=dtype, mode='r', shape=(lines, pixels)).view(
                    np.int16).astype(np.float32).view(np.complex64)
                resample[:, :] = slc_dat
                resample.flush()

            if not os.path.exists(resample_reramped_dat):
                resample_reramped = np.memmap(resample_reramped_dat, dtype='complex64', mode='w+', shape=(lines, pixels))
                slc_ramped_dat = np.memmap(slave_dat, dtype=dtype, mode='r', shape=(lines, pixels)).view(
                    np.int16).astype(np.float32).view(np.complex64)
                resample_reramped[:, :] = slc_ramped_dat
                resample_reramped.flush()

        self.update_res(dates=[date], image_stack=image_res, burst_stack=burst_res)

    def fake_master_steps(self, step='subtr_refdem', network=True, burst_proc=True, full_swath=True):
        # This fakes different steps for the ifgs for the master date. These are needed for further processing in a
        # network setup. Note that we just copy data from one of the other resampled datasets, which means that the actual
        # information in the .res files is not correct.

        steps = ['coarse_correl', 'fine_coreg', 'dem_assist', 'comp_coregpm', 'interfero', 'comp_refphase',
                 'subtr_refphase', 'comp_refdem', 'subtr_refdem', 'filtphase', 'unwrap', 'coherence']
        file_steps = {'subtr_refphase': 'cint_srp.raw', 'subtr_refdem': 'cint_srd.raw',
                      'filtphase': 'cint.0.2filtered', 'unwrapped': 'unwrapped.raw'}

        if not step in steps:
            print('Step ' + step + ' does not exist in processing')
            return
        elif step == 'resample':
            self.fake_master_resample()
            return

        date = self.master_date
        date_1 = list(self.stack.keys())[0]
        bursts = list(self.stack[date_1].keys())
        burst_res = dict()
        image_res = dict()
        self.read_res()  # Read the information from other steps first.
        self.read_res(dates=[self.master_date], bursts=bursts, burst_stack=burst_res, image_stack=image_res)

        if burst_proc:
            for burst in bursts:
                if burst_res[date][burst]['ifgs'].process_control[step] != '1':
                    lines = int(burst_res[date][burst]['master'].processes['readfiles']['Last_line (w.r.t. output_image)']) - \
                            int(burst_res[date][burst]['master'].processes['readfiles']['First_line (w.r.t. output_image)']) + 1
                    pixels = int(burst_res[date][burst]['master'].processes['readfiles']['Last_line (w.r.t. output_image)']) - \
                             int(burst_res[date][burst]['master'].processes['readfiles']['First_line (w.r.t. output_image)']) + 1

                    rsmp_file = self.burst_path(self.master_date, burst, full_path=True, file_path='slave_rsmp_reramped.raw')
                    ifg_file = self.burst_path(self.master_date, burst, full_path=True, file_path='cint.raw')

                    # Cases where we create output files relevant for further processing
                    if not network and step == 'interfero':
                        # If we do not work with a network the interferogram consists of zeros.
                        ifg = np.memmap(ifg_file, dtype='complex64', mode='w+', shape=(lines, pixels))
                        ifg[:, :] = 0
                        ifg.flush()
                    elif network and step == 'interfero' and not os.path.exists(ifg_file):
                        os.symlink(rsmp_file, ifg_file)

                        # Check the whether we have to create a file and define which file that should be.
                    elif step in file_steps.keys():
                        if step == 'coherence':
                            # Coherence of ifg with itself is 1.
                            coh = np.memmap(ifg_file, dtype='float32', mode='w+', shape=(lines, pixels))
                            coh[:, :] = 1
                            coh.flush()
                        else:
                            # For other steps we only have to link to the interferogram. This does not make sense for the case
                            # we are using a network for unwrapping and filtphase. But in that case it will not be used...
                            step_file = self.burst_path(self.master_date, burst, full_path=True, file_path=file_steps[step])
                            if network and not os.path.exists(step_file) and os.path.exists(rsmp_file):
                                os.symlink(rsmp_file, step_file)
                            elif not os.path.exists(ifg_file) and os.path.exists(ifg_file):
                                os.symlink(ifg_file, step_file)

                    # Then copy the information from the other burst files. This hold for all interferogram steps.
                    res_step = copy.deepcopy(self.stack[date_1][burst]['ifgs'].processes[step])
                    burst_res[date][burst]['ifgs'].insert(res_step, step)   # This is generally the same.

        # And from the main files if needed. Works only if the master resampled slave is also concatenated...
        if full_swath:
            if image_res[date]['ifgs'].process_control[step] != '1':
                # Follow a similar procedure for the full swath
                lines = int(image_res[date]['master'].processes['readfiles']['Last_line (w.r.t. output_image)']) - \
                        int(image_res[date]['master'].processes['readfiles']['First_line (w.r.t. output_image)']) + 1
                pixels = int(image_res[date]['master'].processes['readfiles']['Last_line (w.r.t. output_image)']) - \
                         int(image_res[date]['master'].processes['readfiles']['First_line (w.r.t. output_image)']) + 1

                rsmp_file = self.image_path(self.master_date, file_path='slave_rsmp_reramped.raw')
                ifg_file = self.image_path(self.master_date, file_path='cint.raw')

                if not os.path.exists(rsmp_file) and network:
                    print('Please concatenate the resampled reramped master if you want to do network processing!')

                # Cases where we create output files relevant for further processing
                if network == False and step == 'interfero':
                    # If we do not work with a network the interferogram consists of zeros.
                    ifg = np.memmap(ifg_file, dtype='complex64', mode='w+', shape=(lines, pixels))
                    ifg[:, :] = 0
                    ifg.flush()
                elif network and step == 'interfero' and not os.path.exists(ifg_file) and os.path.exists(rsmp_file):
                    os.symlink(rsmp_file, ifg_file)

                    # Check the whether we have to create a file and define which file that should be.
                elif step in file_steps.keys():
                    if step == 'coherence':
                        # Coherence of ifg with itself is 1.
                        coh = np.memmap(ifg_file, dtype='float32', mode='w+', shape=(lines, pixels))
                        coh[:, :] = 1
                        coh.flush()
                    else:
                        # For other steps we only have to link to the interferogram. This does not make sense for the case
                        # we are using a network for unwrapping and filtphase. But in that case it will not be used...
                        step_file = self.image_path(self.master_date, file_path=file_steps[step])
                        if network and not os.path.exists(step_file) and os.path.exists(rsmp_file):
                            os.symlink(rsmp_file, step_file)
                        elif not os.path.exists(step_file) and os.path.exists(ifg_file):
                            os.symlink(ifg_file, step_file)

                # Then copy the information from the other burst files. This hold for all interferogram steps.
                res_step = copy.deepcopy(self.full_swath[date_1]['ifgs'].processes[step])
                image_res[date]['ifgs'].insert(res_step, step)  # This is generally the same.

        self.update_res(dates=[date], image_stack=image_res, burst_stack=burst_res)

    def fake_interferogram(self):
        # This step fakes the creation of an interferogram by renaming the resampled slave

        if len(self.coreg_dates) == 0:
            return
        self.read_res(dates=self.coreg_dates)

        interfero_dummy = OrderedDict()
        interfero_dummy['Data_output_file'] = 'slave_rsmp_reramped.raw'
        interfero_dummy['Data_output_format'] = 'complex_real4'
        interfero_dummy['First_line (w.r.t. original_master)'] = ''
        interfero_dummy['First_pixel (w.r.t. original_master)'] = ''
        interfero_dummy['Last_line (w.r.t. original_master)'] = ''
        interfero_dummy['Last_pixel (w.r.t. original_master)'] = ''
        interfero_dummy['Multilookfactor_azimuth_direction'] = '1'
        interfero_dummy['Multilookfactor_range_direction'] = '1'
        interfero_dummy['Number of lines (multilooked)'] = ''
        interfero_dummy['Number of pixels (multilooked)'] = ''

        # Fake an interferogram for the different bursts.
        for date in self.coreg_dates:
            for burst in self.stack[date].keys():

                if self.stack[date][burst]['ifgs'].process_control['interfero'] != '1':

                    m_dat = self.stack[date][burst]['master'].processes['crop']

                    interfero = copy.deepcopy(interfero_dummy)
                    interfero['First_line (w.r.t. original_master)'] = m_dat['First_line (w.r.t. original_image)']
                    interfero['First_pixel (w.r.t. original_master)'] = m_dat['First_pixel (w.r.t. original_image)']
                    interfero['Last_line (w.r.t. original_master)'] = m_dat['Last_line (w.r.t. original_image)']
                    interfero['Last_pixel (w.r.t. original_master)'] = m_dat['Last_pixel (w.r.t. original_image)']
                    n_lines = int(m_dat['Last_line (w.r.t. original_image)']) - int(
                        m_dat['First_line (w.r.t. original_image)']) + 1
                    n_pixels = int(m_dat['Last_pixel (w.r.t. original_image)']) - int(
                        m_dat['First_pixel (w.r.t. original_image)']) + 1
                    interfero['Number of lines (multilooked)'] = str(n_lines)
                    interfero['Number of pixels (multilooked)'] = str(n_pixels)

                    if not self.stack[date][burst]['ifgs'].process_control['interfero'] == '1':
                        self.stack[date][burst]['ifgs'].insert(interfero, 'interfero')

        # The master and slave results files are switched to get the right correction of the slave file.
        self.update_res(dates=self.coreg_dates)

        self.fake_master_steps(step='interfero', network=True, full_swath=False)

    def interferogram(self, concatenate=True, overwrite=False, ras=False):

        if len(self.coreg_dates) == 0:
            return
        self.read_res(dates=self.coreg_dates)

        # Make an interferogram for the different bursts. (Not always necessary)
        jobList1 = []
        for date in self.coreg_dates:
            for burst in self.stack[date].keys():

                if self.stack[date][burst]['ifgs'].process_control['interfero'] != '1':
                    path = self.burst_path(date, burst, full_path=True)
                    os.chdir(path)

                    command1 = self.doris_path + ' ' + os.path.join(self.input_files, 'input.interferogram')
                    jobList1.append({"path": path, "command": command1})

                    if (not(self.parallel)):
                        os.chdir(path)
                        os.system(command1)

        if (self.parallel):
            jobs = Jobs(self.nr_of_jobs, self.doris_parameters)
            jobs.run(jobList1)

        self.read_res(dates=self.coreg_dates)

        if concatenate == True:
            cint_name = 'cint.raw'
            self.concatenate(cint_name, cint_name, dt=np.dtype('complex64'), overwrite=overwrite)

            for date in self.coreg_dates:
                if self.full_swath[date]['ifgs'].process_control['interfero'] != '1' or overwrite is True:
                    # Add res file information
                    no_lines = self.full_swath[date]['master'].processes['readfiles']['Number_of_lines_original']
                    no_pixels = self.full_swath[date]['master'].processes['readfiles']['Number_of_pixels_original']
                    line_0 = self.full_swath[date]['master'].processes['readfiles']['First_line (w.r.t. output_image)']
                    line_1 = self.full_swath[date]['master'].processes['readfiles']['Last_line (w.r.t. output_image)']
                    pix_0 = self.full_swath[date]['master'].processes['readfiles']['First_pixel (w.r.t. output_image)']
                    pix_1 = self.full_swath[date]['master'].processes['readfiles']['Last_pixel (w.r.t. output_image)']

                    burst = list(self.stack[date].keys())[0]
                    res = copy.deepcopy(self.stack[date][burst]['ifgs'].processes['interfero'])

                    res['First_line (w.r.t. original_master)'] = line_0
                    res['Last_line (w.r.t. original_master)'] = line_1
                    res['First_pixel (w.r.t. original_master)'] = pix_0
                    res['Last_pixel (w.r.t. original_master)'] = pix_1
                    res['Number of lines (multilooked)'] = no_lines
                    res['Number of pixels (multilooked)'] = no_pixels

                    self.full_swath[date]['ifgs'].insert(res, 'interfero')

                    path = self.image_path(date)
                    os.chdir(path)
                    # Finally show preview based on cpxfiddle

                    if ras:
                        pixels = self.full_swath[date]['master'].processes['readfiles']['Number_of_pixels_original']

                        if not os.path.exists('interferogram_mag.ras') or overwrite:
                            mag = ' -w ' + pixels + ' -e 0.3 -s 1.0 -q mag -o sunraster -b -c gray -M 20/5 -f cr4 -l1 ' \
                                                             '-p1 -P' + pixels + ' ' + cint_name + ' > interferogram_mag.ras'
                            os.system(self.cpxfiddle + mag)
                        if not os.path.exists('interferogram_mix.ras') or overwrite:
                            mix = ' -w ' + pixels + ' -e 0.3 -s 1.2 -q mixed -o sunraster -b -c jet -M 20/5 -f cr4 -l1 ' \
                                                             '-p1 -P' + pixels + ' ' + cint_name + ' > interferogram_mix.ras'
                            os.system(self.cpxfiddle + mix)
                        if not os.path.exists('interferogram_pha.ras') or overwrite:
                            pha = ' -w ' + pixels + ' -q phase -o sunraster -b -c jet -M 20/5 -f cr4 -l1 ' \
                                                             '-p1 -P' + pixels + ' ' + cint_name + ' > interferogram_pha.ras'
                            os.system(self.cpxfiddle + pha)

        self.update_res(dates=self.coreg_dates)

        self.fake_master_steps(step='interfero', network=False, full_swath=concatenate)

    def overlapping(self):
        # This function calculates the overlapping areas between different bursts. This function will give a list of
        # overlapping areas between the different bursts.

        for date in self.stack.keys():
            # First make a list of all min max coordinates of all bursts.

            x0=[]; x1=[]; y0=[]; y1=[]
            bursts = list(self.stack[date].keys())
            for burst in bursts:
                y0.append(int(self.stack[date][burst][type].processes['readfiles']['First_line (w.r.t. output_image)']))
                y1.append(int(self.stack[date][burst][type].processes['readfiles']['Last_line (w.r.t. output_image)']))
                x0.append(int(self.stack[date][burst][type].processes['readfiles']['First_pixel (w.r.t. output_image)']))
                x1.append(int(self.stack[date][burst][type].processes['readfiles']['Last_pixel (w.r.t. output_image)']))

            for b1 in range(len(bursts)):
                print('hello')

    def esd(self, esd_type='ps', max_baseline='200'):

        esd_folder = os.path.join(self.folder, 'esd')
        if not os.path.exists(esd_folder):
            os.mkdir(esd_folder)

        jobList = []
        # First run all the ESD calculations in parallel
        for date in [list(self.stack.keys())[0]]:
            bursts = list(self.stack[date].keys())
            sort_id = [int(dat[6]) * 100 + int(dat[14:]) for dat in bursts]
            bursts = [x for (y, x) in sorted(zip(sort_id, bursts))]

            for burst, id in zip(bursts, range(len(bursts))):

                nBurst = int(burst[14:])
                next_burst = burst[:14] + str(nBurst + 1)
                if next_burst in bursts:
                    stack_folder = self.folder
                    overlap = burst + '_' + next_burst
                    ps_select = '1'
                    master_date = self.master_date
                    command = sys.executable + ' ' + os.path.join(self.function_path, 'ESD_ps_ds.py') + ' ' + stack_folder + ' ' \
                              + overlap + ' ' + esd_type + ' ' + max_baseline + ' ' + master_date + ' ' + ps_select
                    jobList.append({"path": stack_folder, "command": command})

                    if not (self.parallel):
                        os.chdir(stack_folder)
                        os.system(command)
        if self.parallel:
            jobs = Jobs(self.nr_of_jobs, self.doris_parameters)
            jobs.run(jobList)

        # Now load the different matrices again.
        # Find all the overlaps and corresponding results:
        esd_folder = os.path.join(self.stack_folder, 'esd')
        folders = os.listdir(esd_folder)

        self.diff_matrix[esd_type] = np.zeros(shape=(len(folders), len(self.stack.keys()) + 1, len(self.stack.keys()) + 1))
        self.var_matrix[esd_type] = np.zeros(shape=(len(folders), len(self.stack.keys()) + 1, len(self.stack.keys()) + 1))
        self.to_angle_matrix[esd_type] = np.zeros(shape=(len(folders), len(self.stack.keys()) + 1, len(self.stack.keys()) + 1))
        self.weight_matrix[esd_type] = np.zeros(shape=(len(folders), len(self.stack.keys()) + 1, len(self.stack.keys()) + 1))

        for folder, n in zip(folders, range(len(folders))):
            f = os.path.join(esd_folder, folder)
            diff_m = np.load(os.path.join(f, esd_type + '_diff_matrix.npy'))
            var_m = np.load(os.path.join(f, esd_type + '_var_matrix.npy'))
            to_angle_m = np.load(os.path.join(f, esd_type + '_to_angle_matrix.npy'))
            w = np.load(os.path.join(f, esd_type + '_weight_matrix.npy'))

            self.diff_matrix[esd_type][n, :, :] = diff_m
            self.var_matrix[esd_type][n, :, :] = var_m
            self.to_angle_matrix[esd_type][n, :, :] = to_angle_m
            self.weight_matrix[esd_type][n, :, :] = w

    def network_esd(self, esd_type='ps', var_calc=False):
        # This function calculates the ESD values using a network approach

        dates = (list(self.stack.keys()))
        dates.append(self.master_date)
        dates = sorted(dates)

        w_matrix = np.sum(self.weight_matrix[esd_type], 0)
        diff_matrix = np.sum(self.diff_matrix[esd_type] * self.weight_matrix[esd_type], 0)
        diff_matrix[w_matrix > 0] = diff_matrix[w_matrix > 0] / w_matrix[w_matrix > 0]

        angle_pixel = np.sum(self.to_angle_matrix[esd_type] * self.weight_matrix[esd_type], 0)
        angle_pixel[w_matrix > 0] = angle_pixel[w_matrix > 0] / w_matrix[w_matrix > 0]

        # In case we want to use the variances...
        if var_calc == True:
            var_matrix = np.zeros(w_matrix.shape)
            id = np.where(w_matrix != 0)
            for n, m in zip(id[0], id[1]):
                w = self.weight_matrix[esd_type][:, n, m][None, :]
                v = self.var_matrix[esd_type][:, n, m][None, :]
                var_matrix[n,m] = np.sum(np.dot(w.transpose(), w) * np.dot(v.transpose(), v))
            var_matrix[w_matrix != 0] = var_matrix[w_matrix != 0] / w_matrix[w_matrix != 0]
            std_calc = np.sqrt(var_matrix)

        # Finally calculate the network

        # Find the connections in the difference matrix
        m_s = np.where(diff_matrix != 0)
        weight = w_matrix[diff_matrix != 0]

        # Find the master date
        master_num = dates.index(self.master_date)
        slave_nums = list(range(len(dates)))
        slave_nums.remove(master_num)

        # Create the A matrix
        A = np.zeros(shape=(len(m_s[0]), np.max([np.max(m_s[0]), np.max(m_s[1])]) + 1))
        A[list(range(len(m_s[0]))), m_s[0]] = 1
        A[list(range(len(m_s[0]))), m_s[1]] = -1
        A = np.hstack((A[:, :master_num], A[:, master_num + 1:]))

        # Create the weight matrix
        W = np.zeros((len(m_s[0]), len(m_s[0])))
        id = list(range(len(m_s[0])))

        W[id, id] = 1 / weight
        W = np.linalg.inv(W)

        esd_diff = np.dot(np.dot(np.dot(np.linalg.inv(np.dot(np.dot(A.T, W), A)), A.T), W), diff_matrix[diff_matrix != 0])
        esd_residue = np.dot(A, esd_diff) - diff_matrix[diff_matrix != 0]

        print(str(np.nanmean(np.abs(esd_residue))))
        sigma = np.std(esd_residue)

        dates = sorted(self.stack.keys())
        for date, shift, n in zip(dates, esd_diff, slave_nums):
            self.ESD_shift[date] = shift
            self.ESD_angle_pixel[date] = np.max([angle_pixel[n, master_num], angle_pixel[master_num, n]])

    def ESD_correct_ramp(self, filename='cint_srd.raw', network=False):
        # This function correct for ESD using the expected ramp in the resampled slave image

        self.read_res()
        jobList = []

        for date in self.stack.keys():
            for burst in self.stack[date].keys():
                if self.stack[date][burst]['slave'].processes['readfiles']['ESD_correct'] == '0':
                    path = self.burst_path(date, burst, full_path=True)

                    offset = self.ESD_shift[date]
                    angle = self.ESD_angle_pixel[date]
                    if not network and filename.startswith('cint'):
                        # Because after interferogram the slave is subtracted from the master we have to compensate.
                        angle_pixel = str(-offset / angle)
                    else:
                        angle_pixel = str(offset / angle)
                    script = os.path.join(self.function_path, 'correct_ESD.py')
                    command = sys.executable + ' ' + script + ' ' + filename + ' ' + angle_pixel

                    jobList.append({"path": path, "command": command})
                    self.stack[date][burst]['slave'].processes['readfiles']['ESD_correct'] = '1'

                    if not (self.parallel):
                        os.chdir(path)
                        os.system(command)

        if self.parallel:
            jobs = Jobs(self.nr_of_jobs, self.doris_parameters)
            jobs.run(jobList)

        self.update_res()

    def combine_slave(self, overwrite=False, ramped=False, deramped=True, ras=False):
        # This function concatenates the different slave values. Both ramped and deramped.

        # Add the resample step to the .res file
        if len(self.coreg_dates) == 0:
            return
        self.read_res(dates=self.coreg_dates)

        if deramped:
            self.concatenate('slave_rsmp.raw', 'slave_rsmp.raw', dt= np.dtype('complex64'), overwrite=overwrite)
        if ramped:
            self.concatenate('slave_rsmp_reramped.raw', 'slave_rsmp_reramped.raw', dt=np.dtype('complex64'), overwrite=overwrite)

        for date in self.coreg_dates:

            if self.full_swath[date]['slave'].process_control != '1':
                path = self.image_path(date)
                os.chdir(path)

                burst = list(self.stack[date].keys())[0]
                slave_res = copy.deepcopy(self.stack[date][burst]['slave'].processes['resample'])

                # Read number of lines
                lines = int(self.full_swath[date]['master'].processes['readfiles']['Number_of_lines_original'])
                pixels = int(self.full_swath[date]['master'].processes['readfiles']['Number_of_pixels_original'])

                # Add information to interfero step about lines and pixels.
                slave_res['First_line (w.r.t. original_master)'] = str(1)
                slave_res['Last_line (w.r.t. original_master)'] = str(lines)
                slave_res['First_pixel (w.r.t. original_master)'] = str(1)
                slave_res['Last_pixel (w.r.t. original_master)'] = str(pixels)
                slave_res['Data_output_file'] = 'slave_rsmp_reramped.raw'

                # Finally add to result file
                self.full_swath[date]['slave'].insert(slave_res, 'resample')

            pixels = self.full_swath[date]['master'].processes['readfiles']['Number_of_pixels_original']

            if ras:
                if deramped and (not os.path.exists('slave_rsmp.ras') or overwrite):
                    mag = ' -w ' + pixels + ' -e 0.3 -s 1.0 -q mag -o sunraster -b -c gray -M 20/5 -f cr4 -l1 ' \
                                             '-p1 -P' + pixels + ' slave_rsmp.raw > slave_rsmp.ras'
                    os.system(self.cpxfiddle + mag)
                if ramped and (not os.path.exists('slaver_rsmp_reramped.ras') or overwrite):
                    mag = ' -w ' + pixels + ' -e 0.3 -s 1.0 -q mag -o sunraster -b -c gray -M 20/5 -f cr4 -l1 ' \
                                 '-p1 -P' + pixels + ' slave_rsmp_reramped.raw > slave_rsmp_reramped.ras'
                    os.system(self.cpxfiddle + mag)

        self.update_res(dates=self.coreg_dates)

    def combine_master(self, overwrite=False, ramped=False, deramped=True, ras=False):
        # This function concatenates the master files to one image. Afterwards the full master files are linked using
        # symbolic links.

        date = self.master_date
        date_1 = list(self.stack.keys())[0]
        bursts = list(self.stack[date_1].keys())
        burst_res = dict()
        image_res = dict()

        self.read_res(dates=[date], bursts=bursts, burst_stack=burst_res, image_stack=image_res)
        if deramped:
            self.concatenate('slave_rsmp.raw', 'slave_rsmp.raw', dt=np.dtype('complex64'),
                             overwrite=overwrite, dates=[date])
        if ramped:
            self.concatenate('slave_rsmp_reramped.raw', 'slave_rsmp_reramped.raw', dt=np.dtype('complex64'),
                             overwrite=overwrite, dates=[date])

        path = self.image_path(date)
        os.chdir(path)

        if image_res[date]['slave'].process_control != '1':
            burst = list(burst_res[date].keys())[0]
            slave_res = copy.deepcopy(burst_res[date][burst]['slave'].processes['resample'])

            # Read number of lines
            lines = int(image_res[date]['master'].processes['readfiles']['Number_of_lines_original'])
            pixels = int(image_res[date]['master'].processes['readfiles']['Number_of_pixels_original'])

            # Add information to interfero step about lines and pixels.
            slave_res['First_line (w.r.t. original_master)'] = str(1)
            slave_res['Last_line (w.r.t. original_master)'] = str(lines)
            slave_res['First_pixel (w.r.t. original_master)'] = str(1)
            slave_res['Last_pixel (w.r.t. original_master)'] = str(pixels)
            slave_res['Data_output_file'] = 'slave_rsmp_reramped.raw'

            # Finally add to result file
            image_res[date]['slave'].insert(slave_res, 'resample')

        pixels = image_res[date]['master'].processes['readfiles']['Number_of_pixels_original']

        if ras:
            if deramped and (not os.path.exists('slave_rsmp.ras') or overwrite):
                mag = ' -w ' + pixels + ' -e 0.3 -s 1.0 -q mag -o sunraster -b -c gray -M 20/5 -f cr4 -l1 ' \
                                         '-p1 -P' + pixels + ' slave_rsmp.raw > slave_rsmp.ras'
                os.system(self.cpxfiddle + mag)
            if ramped and (not os.path.exists('slaver_rsmp_reramped.ras') or overwrite):
                mag = ' -w ' + pixels + ' -e 0.3 -s 1.0 -q mag -o sunraster -b -c gray -M 20/5 -f cr4 -l1 ' \
                             '-p1 -P' + pixels + ' slave_rsmp_reramped.raw > slave_rsmp_reramped.ras'
                os.system(self.cpxfiddle + mag)

        self.update_res(dates=[date], image_stack=image_res, burst_stack=burst_res)

    def compref_phase(self, network=False):
        # This function performs the final steps in making an interferogram for all full images.

        if len(self.coreg_dates) == 0:
            return
        self.read_res(dates=self.coreg_dates)
        if network:
            self.update_res(dates=self.coreg_dates, switch=True)

        for date in self.coreg_dates:
            job_list1 = []
            for burst in self.stack[date].keys():
                if self.stack[date][burst]['ifgs'].process_control['comp_refphase'] != '1':
                    path = self.burst_path(date, burst, full_path=True)
                    command1 = self.doris_path + ' ' + os.path.join(self.input_files, 'input.comprefpha')
                    job_list1.append({"path": path, "command": command1})
                    if (not(self.parallel)):
                        os.chdir(path)
                        os.system(command1)
            if (self.parallel):
                jobs = Jobs(self.nr_of_jobs, self.doris_parameters)
                jobs.run(job_list1)

        if network:
            self.read_res(dates=self.coreg_dates)
            self.update_res(dates=self.coreg_dates, switch=True)

        self.fake_master_steps(step='comp_refphase', full_swath=False)

    def ref_phase(self,concatenate=True, overwrite=False, network=False, ras=False):
        # This function performs the final steps in making an interferogram for all full images.

        if len(self.coreg_dates) == 0:
            return
        self.read_res(dates=self.coreg_dates)

        job_list2 = []
        for date in self.coreg_dates:
            for burst in self.stack[date].keys():
                if self.stack[date][burst]['ifgs'].process_control['subtr_refphase'] != '1':
                    path = self.burst_path(date, burst, full_path=True)
                    command2 = self.doris_path + ' ' + os.path.join(self.input_files, 'input.subtrrefpha')
                    job_list2.append({"path": path, "command": command2})
                    if (not(self.parallel)):
                        os.chdir(path)
                        os.system(command2)
        if (self.parallel):
            jobs = Jobs(self.nr_of_jobs, self.doris_parameters)
            jobs.run(job_list2)

        self.read_res(dates=self.coreg_dates)

        if concatenate == True:
            self.concatenate('cint_srp.raw', 'cint_srp.raw', dt=np.dtype('complex64'), overwrite=overwrite)
            for date in self.coreg_dates:

                if self.full_swath[date]['ifgs'].process_control['subtr_refphase'] != '1' or overwrite is True:
                    # Add res file information
                    no_lines = self.full_swath[date]['master'].processes['readfiles']['Number_of_lines_original']
                    no_pixels = self.full_swath[date]['master'].processes['readfiles']['Number_of_pixels_original']
                    line_0 = self.full_swath[date]['master'].processes['readfiles']['First_line (w.r.t. output_image)']
                    line_1 = self.full_swath[date]['master'].processes['readfiles']['Last_line (w.r.t. output_image)']
                    pix_0 = self.full_swath[date]['master'].processes['readfiles']['First_pixel (w.r.t. output_image)']
                    pix_1 = self.full_swath[date]['master'].processes['readfiles']['Last_pixel (w.r.t. output_image)']

                    burst = list(self.stack[date].keys())[0]
                    res_1 = copy.deepcopy(self.stack[date][burst]['ifgs'].processes['comp_refphase'])
                    res_2 = copy.deepcopy(self.stack[date][burst]['ifgs'].processes['subtr_refphase'])

                    res_2['First_line (w.r.t. original_master)'] = line_0
                    res_2['Last_line (w.r.t. original_master)'] = line_1
                    res_2['First_pixel (w.r.t. original_master)'] = pix_0
                    res_2['Last_pixel (w.r.t. original_master)'] = pix_1
                    res_2['Number of lines (multilooked)'] = no_lines
                    res_2['Number of pixels (multilooked)'] = no_pixels

                    self.full_swath[date]['ifgs'].insert(res_1, 'comp_refphase')
                    self.full_swath[date]['ifgs'].insert(res_2, 'subtr_refphase')

                    path = self.image_path(date)
                    os.chdir(path)
                    # Finally show preview based on cpxfiddle

                    if ras:
                        pixels = self.full_swath[date]['master'].processes['readfiles']['Number_of_pixels_original']

                        if not os.path.exists('interferogram_srp_mag.ras') or overwrite:
                            mag = ' -w ' + pixels + ' -e 0.3 -s 1.0 -q mag -o sunraster -b -c gray -M 20/5 -f cr4 -l1 ' \
                                                             '-p1 -P' + pixels + ' cint_srp.raw > interferogram_srp_mag.ras'
                            os.system(self.cpxfiddle + mag)
                        if not os.path.exists('interferogram_srp_mix.ras') or overwrite:
                            mix = ' -w ' + pixels + ' -e 0.3 -s 1.2 -q mixed -o sunraster -b -c jet -M 20/5 -f cr4 -l1 ' \
                                                             '-p1 -P' + pixels + ' cint_srp.raw > interferogram_srp_mix.ras'
                            os.system(self.cpxfiddle + mix)
                        if not os.path.exists('interferogram_srp_pha.ras') or overwrite:
                            pha = ' -w ' + pixels + ' -q phase -o sunraster -b -c jet -M 20/5 -f cr4 -l1 ' \
                                                             '-p1 -P' + pixels + ' cint_srp.raw > interferogram_srp_pha.ras'
                            os.system(self.cpxfiddle + pha)

        self.update_res(dates=self.coreg_dates)

        self.fake_master_steps(step='subtr_refphase', full_swath=concatenate)

    def compref_dem(self, network=False):
        # This function performs the final steps in making an interferogram for all full images.
        if len(self.coreg_dates) == 0:
            return
        self.read_res(dates=self.coreg_dates)

        for date in self.coreg_dates:
            job_list1 = []
            for burst in self.stack[date].keys():
                if self.stack[date][burst]['ifgs'].process_control['comp_refdem'] != '1':
                    path = self.burst_path(date, burst, full_path=True)
                    command1 = self.doris_path + ' ' + os.path.join(self.input_files, 'input.comprefdem')
                    job_list1.append({"path": path, "command": command1})
                    if (not(self.parallel)):
                        os.chdir(path)
                        os.system(command1)
            if (self.parallel):
                jobs = Jobs(self.nr_of_jobs, self.doris_parameters)
                jobs.run(job_list1)

        self.fake_master_steps(step='comp_refdem', full_swath=False)

    def ref_dem(self,concatenate=True, overwrite=False, network=False, ras=False):
        # This function performs the final steps in making an interferogram for all full images.

        if len(self.coreg_dates) == 0:
            return
        self.read_res(dates=self.coreg_dates)

        job_list1 = []
        job_list2 = []

        for date in self.coreg_dates:
            for burst in self.stack[date].keys():
                if self.stack[date][burst]['ifgs'].process_control['subtr_refdem'] != '1':
                    path = self.burst_path(date, burst, full_path=True)
                    command1 = self.doris_path + ' ' + os.path.join(self.input_files, 'input.subtrrefdem')
                    job_list1.append({"path": path, "command": command1})
                    if network:
                        command2 = sys.executable + ' ' + os.path.join(self.function_path, 'remove_dem_earth_phase.py') + ' ' + \
                                   self.stack_folder + ' ' + date + ' ' + burst
                        job_list2.append({"path": path, "command": command2})
                    if not self.parallel:
                        os.chdir(path)
                        os.system(command2)
        if self.parallel:
            jobs = Jobs(self.nr_of_jobs, self.doris_parameters)
            jobs.run(job_list1)
        if self.parallel:
            jobs = Jobs(self.nr_of_jobs, self.doris_parameters)
            jobs.run(job_list2)

        self.read_res(dates=self.coreg_dates)

        if concatenate == True:
            self.concatenate('cint_srd.raw', 'cint_srd.raw', dt=np.dtype('complex64'), overwrite=overwrite)

            for date in self.coreg_dates:

                if self.full_swath[date]['ifgs'].process_control['subtr_refdem'] != '1' or overwrite is True:
                    # Add res file information
                    no_lines = self.full_swath[date]['master'].processes['readfiles']['Number_of_lines_original']
                    no_pixels = self.full_swath[date]['master'].processes['readfiles']['Number_of_pixels_original']
                    line_0 = self.full_swath[date]['master'].processes['readfiles']['First_line (w.r.t. output_image)']
                    line_1 = self.full_swath[date]['master'].processes['readfiles']['Last_line (w.r.t. output_image)']
                    pix_0 = self.full_swath[date]['master'].processes['readfiles']['First_pixel (w.r.t. output_image)']
                    pix_1 = self.full_swath[date]['master'].processes['readfiles']['Last_pixel (w.r.t. output_image)']

                    burst = list(self.stack[date].keys())[0]
                    res_1 = copy.deepcopy(self.stack[date][burst]['ifgs'].processes['comp_refdem'])
                    res_2 = copy.deepcopy(self.stack[date][burst]['ifgs'].processes['subtr_refdem'])

                    res_1['First_line (w.r.t. original_master)'] = line_0
                    res_1['Last_line (w.r.t. original_master)'] = line_1
                    res_1['First_pixel (w.r.t. original_master)'] = pix_0
                    res_1['Last_pixel (w.r.t. original_master)'] = pix_1
                    res_1['Number of lines (multilooked)'] = no_lines
                    res_1['Number of pixels (multilooked)'] = no_pixels

                    res_2['First_line (w.r.t. original_master)'] = line_0
                    res_2['Last_line (w.r.t. original_master)'] = line_1
                    res_2['First_pixel (w.r.t. original_master)'] = pix_0
                    res_2['Last_pixel (w.r.t. original_master)'] = pix_1
                    res_2['Number of lines (multilooked)'] = no_lines
                    res_2['Number of pixels (multilooked)'] = no_pixels

                    self.full_swath[date]['ifgs'].insert(res_1, 'comp_refdem')
                    self.full_swath[date]['ifgs'].insert(res_2, 'subtr_refdem')

                    path = self.image_path(date)
                    os.chdir(path)
                    # Finally show preview based on cpxfiddle

                    if ras:
                        pixels = self.full_swath[date]['master'].processes['readfiles']['Number_of_pixels_original']

                        if not os.path.exists('interferogram_srd_mag.ras') or overwrite:
                            mag = ' -w ' + pixels + ' -e 0.3 -s 1.0 -q mag -o sunraster -b -c gray -M 20/5 -f cr4 -l1 ' \
                                                             '-p1 -P' + pixels + ' cint_srd.raw > interferogram_srd_mag.ras'
                            os.system(self.cpxfiddle + mag)
                        if not os.path.exists('interferogram_srd_mix.ras') or overwrite:
                            mix = ' -w ' + pixels + ' -e 0.3 -s 1.2 -q mixed -o sunraster -b -c jet -M 20/5 -f cr4 -l1 ' \
                                                             '-p1 -P' + pixels + ' cint_srd.raw > interferogram_srd_mix.ras'
                            os.system(self.cpxfiddle + mix)
                        if not os.path.exists('interferogram_srd_pha.ras') or overwrite:
                            pha = ' -w ' + pixels + ' -q phase -o sunraster -b -c jet -M 20/5 -f cr4 -l1 ' \
                                                             '-p1 -P' + pixels + ' cint_srd.raw > interferogram_srd_pha.ras'
                            os.system(self.cpxfiddle + pha)

        self.update_res(dates=self.coreg_dates)

        self.fake_master_steps(step='comp_refdem', full_swath=concatenate)
        self.fake_master_steps(step='subtr_refdem', full_swath=concatenate)

    def coherence(self, concatenate=True, overwrite=False, coh_type='single_master', ras=False):
        # This function performs the final steps in making an interferogram for all full images.
        if len(self.coreg_dates) == 0:
            return
        self.read_res(dates=self.coreg_dates)

        for date in self.coreg_dates:
            job_list = []
            for burst in self.stack[date].keys():
                if self.stack[date][burst]['ifgs'].process_control['coherence'] != '1':
                    path = self.burst_path(date, burst, full_path=True)
                    if coh_type == 'single_master':
                        command = self.doris_path + ' ' + os.path.join(self.input_files, 'input.coherence')
                    elif coh_type == 'network':
                        command = self.doris_path + ' ' + os.path.join(self.input_files, 'input.coherence_network')
                    job_list.append({"path": path, "command": command})
                    if (not(self.parallel)):
                        os.chdir(path)
                        os.system(command)
            if (self.parallel):
                jobs = Jobs(self.nr_of_jobs, self.doris_parameters)
                jobs.run(job_list)

        self.read_res(dates=self.coreg_dates)

        if concatenate == True:
            self.concatenate('coherence.raw', 'coherence.raw', dt=np.dtype('float32'), overwrite=overwrite)

            for date in self.coreg_dates:

                if self.full_swath[date]['ifgs'].process_control['coherence'] != '1' or overwrite is True:
                    # Add res file information
                    no_lines = self.full_swath[date]['master'].processes['readfiles']['Number_of_lines_original']
                    no_pixels = self.full_swath[date]['master'].processes['readfiles']['Number_of_pixels_original']
                    line_0 = self.full_swath[date]['master'].processes['readfiles']['First_line (w.r.t. output_image)']
                    line_1 = self.full_swath[date]['master'].processes['readfiles']['Last_line (w.r.t. output_image)']
                    pix_0 = self.full_swath[date]['master'].processes['readfiles']['First_pixel (w.r.t. output_image)']
                    pix_1 = self.full_swath[date]['master'].processes['readfiles']['Last_pixel (w.r.t. output_image)']

                    burst = list(self.stack[date].keys())[0]
                    res = copy.deepcopy(self.stack[date][burst]['ifgs'].processes['coherence'])

                    res['First_line (w.r.t. original_master)'] = line_0
                    res['Last_line (w.r.t. original_master)'] = line_1
                    res['First_pixel (w.r.t. original_master)'] = pix_0
                    res['Last_pixel (w.r.t. original_master)'] = pix_1
                    res['Number of lines (multilooked)'] = no_lines
                    res['Number of pixels (multilooked)'] = no_pixels

                    self.full_swath[date]['ifgs'].insert(res, 'coherence')

                    path = self.image_path(date)
                    os.chdir(path)
                    # Finally show preview based on cpxfiddle

                    if ras:
                        pixels = self.full_swath[date]['master'].processes['readfiles']['Number_of_pixels_original']

                        if not os.path.exists('coherence.ras') or overwrite:
                            mag = ' -w ' + pixels + ' -q normal -o sunraster -b -c gray -M 20/5 -r 0.0/1.0 -f r4 -l1 ' \
                                                             '-p1 -P' + pixels + ' coherence.raw > coherence.ras'
                            os.system(self.cpxfiddle + mag)

        self.update_res(dates=self.coreg_dates)

        self.fake_master_steps(step='coherence', full_swath=concatenate)

    def phasefilt(self,concatenate=True, overwrite=False, ras=False):
        # This function performs the phase filtering of the individual bursts.

        if len(self.coreg_dates) == 0:
            return
        self.read_res(dates=self.coreg_dates)

        for date in self.coreg_dates:
            job_list = []
            for burst in self.stack[date].keys():
                if self.stack[date][burst]['ifgs'].process_control['filtphase'] != '1':
                    path = self.burst_path(date, burst, full_path=True)
                    command = self.doris_path + ' ' + os.path.join(self.input_files, 'input.phasefilt')
                    job_list.append({"path": path, "command": command})
                    if (not(self.parallel)):
                        os.chdir(path)
                        os.system(command)
            if (self.parallel):
                jobs = Jobs(self.nr_of_jobs, self.doris_parameters)
                jobs.run(job_list)

        self.read_res(dates=self.coreg_dates)

        if concatenate == True:

            self.concatenate('cint.0.2filtered', 'cint_filt.raw', dt=np.dtype('complex64'), overwrite=overwrite)
            for date in self.coreg_dates:

                if self.full_swath[date]['ifgs'].process_control['filtphase'] != '1' or overwrite is True:
                    # Add res file information
                    no_lines = self.full_swath[date]['master'].processes['readfiles']['Number_of_lines_original']
                    no_pixels = self.full_swath[date]['master'].processes['readfiles']['Number_of_pixels_original']
                    line_0 = self.full_swath[date]['master'].processes['readfiles']['First_line (w.r.t. output_image)']
                    line_1 = self.full_swath[date]['master'].processes['readfiles']['Last_line (w.r.t. output_image)']
                    pix_0 = self.full_swath[date]['master'].processes['readfiles']['First_pixel (w.r.t. output_image)']
                    pix_1 = self.full_swath[date]['master'].processes['readfiles']['Last_pixel (w.r.t. output_image)']

                    burst = list(self.stack[date].keys())[0]
                    res = copy.deepcopy(self.stack[date][burst]['ifgs'].processes['filtphase'])

                    res['First_line (w.r.t. original_master)'] = line_0
                    res['Last_line (w.r.t. original_master)'] = line_1
                    res['First_pixel (w.r.t. original_master)'] = pix_0
                    res['Last_pixel (w.r.t. original_master)'] = pix_1
                    res['Number of lines (multilooked)'] = no_lines
                    res['Number of pixels (multilooked)'] = no_pixels

                    self.full_swath[date]['ifgs'].insert(res, 'filtphase')

                    path = self.image_path(date)
                    os.chdir(path)
                    # Finally show preview based on cpxfiddle

                    if ras:
                        pixels = self.full_swath[date]['master'].processes['readfiles']['Number_of_pixels_original']

                        if not os.path.exists('interferogram_filt_mag.ras') or overwrite:
                            mag = ' -w ' + pixels + ' -e 0.3 -s 1.0 -q mag -o sunraster -b -c gray -M 20/5 -f cr4 -l1 ' \
                                                             '-p1 -P' + pixels + ' cint.0.2filtered > interferogram_filt_mag.ras'
                            os.system(self.cpxfiddle + mag)
                        if not os.path.exists('interferogram_filt_mix.ras') or overwrite:
                            mix = ' -w ' + pixels + ' -e 0.3 -s 1.2 -q mixed -o sunraster -b -c jet -M 20/5 -f cr4 -l1 ' \
                                                             '-p1 -P' + pixels + ' cint.0.2filtered > interferogram_filt_mix.ras'
                            os.system(self.cpxfiddle + mix)
                        if not os.path.exists('interferogram_filt_pha.ras') or overwrite:
                            pha = ' -w ' + pixels + ' -q phase -o sunraster -b -c jet -M 20/5 -f cr4 -l1 ' \
                                                             '-p1 -P' + pixels + ' cint.0.2filtered > interferogram_filt_pha.ras'
                            os.system(self.cpxfiddle + pha)

        self.update_res(dates=self.coreg_dates)

        self.fake_master_steps(step='filtphase', full_swath=concatenate)

    def unwrap(self, ras=True):
        # This function is used to call the unwrapping program snaphu via doris.

        for date in self.coreg_dates:
            path = self.image_path(date)
            os.chdir(path)

            # First create an phase input file for unwrapping
            pixels = self.full_swath[date]['ifgs'].processes['filtphase']['Number of pixels (multilooked)']
            print(pixels)
            pha = ' -w ' + pixels + ' -q phase -o float -M 1/1 -f cr4 -l1 ' \
                                    '-p1 -P' + pixels + ' cint_filt_ml.raw > unwrap_input.raw'
            os.system(self.cpxfiddle + pha)

            command = self.doris_path + ' ' + os.path.join(self.input_files, 'input.unwrap')
            os.system(command)

            # And create an image using cpxfiddle
            if ras:
                pha = ' -w ' + pixels + ' -q normal -o sunraster -b -c jet -M 1/1 -f r4 -l1 ' \
                                        '-p1 -P' + pixels + ' unwrapped.raw > unwrapped.ras'
                os.system(self.cpxfiddle + pha)

        self.fake_master_steps(step='unwrap', burst_proc=False)

    def calc_coordinates(self, createdem=True):
        # Calculate the coordinates of grid cells

        # choose date closest to master as reference
        date = self.master_date
        date_1 = list(self.stack.keys())[0]
        bursts = list(self.stack[date_1].keys())
        burst_res = dict()
        image_res = dict()

        self.read_res(dates=[self.master_date], bursts=bursts, burst_stack=burst_res, image_stack=image_res)

        print('Coordinates are created for master date ' + date)

        doris_dir = self.doris_path

        for burst in burst_res[date].keys():
            dem_path = self.burst_path(date, burst, 'dem_radar.raw', full_path=True)
            dem_path_s = self.burst_path(date_1, burst, 'dem_radar.raw', full_path=True)

            if not os.path.exists(dem_path) and createdem:
                if os.path.exists(dem_path_s):
                    os.symlink(dem_path_s, dem_path)

                # path = self.burst_path(date, burst, full_path=True)

                # Create grid coordinates and heights
                # dem_inputfile = os.path.join(self.input_files, 'input.createdem')

                # Run if one of the files does not exist...
                # geocode_master(folder, geocode_inputfile, dem_inputfile, doris_dir)
                # this function geocode the bursts of a master file, based on a DEM
                # Run the create DEM command

                # command1 = self.doris_path + ' ' + dem_inputfile
                # job_list1.append({"path": path, "command": command1})
                #if not self.parallel:
                #    os.chdir(path)
                #    os.system(command1)
        #if self.parallel:
        #    jobs = Jobs(self.nr_of_jobs, self.doris_parameters)
        #    jobs.run(job_list1)

        self.read_res(dates=[self.master_date], bursts=bursts, burst_stack=burst_res, image_stack=image_res)

        for burst in burst_res[date].keys():
            if not burst_res[date][burst]['ifgs'].process_control['slant2h'] == 1:
                resultfile = copy.deepcopy(self.stack[date_1][burst]['ifgs'])

                # Add the slant2height information. This is meant to fake the doris script
                sl2h_dat = collections.OrderedDict()
                sl2h_dat['Method'] = 'schwabisch'
                sl2h_dat['Data_output_file'] = 'dem_radar.raw'
                sl2h_dat['Data_output_format'] = 'real4'
                sl2h_dat['First_line (w.r.t. original_master)'] = resultfile.processes['comp_refdem'][
                    'First_line (w.r.t. original_master)']
                sl2h_dat['Last_line (w.r.t. original_master)'] = resultfile.processes['comp_refdem'][
                    'Last_line (w.r.t. original_master)']
                sl2h_dat['First_pixel (w.r.t. original_master)'] = resultfile.processes['comp_refdem'][
                    'First_pixel (w.r.t. original_master)']
                sl2h_dat['Last_pixel (w.r.t. original_master)'] = resultfile.processes['comp_refdem'][
                    'Last_pixel (w.r.t. original_master)']
                sl2h_dat['Multilookfactor_azimuth_direction'] = resultfile.processes['comp_refdem'][
                    'Multilookfactor_azimuth_direction']
                sl2h_dat['Multilookfactor_range_direction'] = resultfile.processes['comp_refdem'][
                    'Multilookfactor_range_direction']
                sl2h_dat['Ellipsoid (name,a,b)'] = 'WGS84 6.37814e+06 6.35675e+06'

                # Add this to all date .res files
                burst_res[date][burst]['ifgs'].insert(sl2h_dat, process='slant2h')
        self.update_res(dates=[date], image_stack=image_res, burst_stack=burst_res)

        job_list1 = []
        for burst in burst_res[date].keys():
            if not burst_res[date][burst]['ifgs'].process_control['geocoding'] == 1:
                path = self.burst_path(date, burst, full_path=True)
                geocode_inputfile = os.path.join(self.input_files, 'input.geocode')
                # Generate lat / lon files
                command1 = doris_dir + ' ' + geocode_inputfile
                job_list1.append({"path": path, "command": command1})

                if not self.parallel:
                    os.system(command1)
        if self.parallel:
            jobs = Jobs(self.nr_of_jobs, self.doris_parameters)
            jobs.run(job_list1)

        self.concatenate('phi.raw', 'phi.raw', dt=np.dtype('float32'), dates=[date])
        self.concatenate('lam.raw', 'lam.raw', dt=np.dtype('float32'), dates=[date])
        self.concatenate('dem_radar.raw', 'dem_radar.raw', dt=np.dtype('float32'), dates=[date])

        dates = self.coreg_dates

        # create links
        for burst in burst_res[date].keys():
            if burst_res[date][burst]['ifgs'].process_control['slant2h'] == 1 and \
                            burst_res[date][burst]['ifgs'].process_control['geocoding'] == 1:

                res_sl2h = copy.deepcopy(burst_res[date][burst]['ifgs'].processes['slant2h'])
                res_geo = copy.deepcopy(burst_res[date][burst]['ifgs'].processes['geocoding'])
                for date_s in dates:
                    if not self.stack[date_s][burst]['ifgs'].process_control['slant2h'] == '1':
                        self.stack[date_s][burst]['ifgs'].insert(res_sl2h, process='slant2h')
                    if not self.stack[date_s][burst]['ifgs'].process_control['geocoding'] == '1':
                        self.stack[date_s][burst]['ifgs'].insert(res_geo, process='geocoding')

        # Create symlinks for bursts
        for burst in burst_res[date].keys():
            dat_lam = self.burst_path(self.master_date, burst, 'lam.raw', full_path=True)
            dat_phi = self.burst_path(self.master_date, burst, 'phi.raw', full_path=True)
            dat_dem = self.burst_path(self.master_date, burst, 'dem_radar.raw', full_path=True)

            for date_s in dates:
                link_lam = self.burst_path(date_s, burst, 'lam.raw', full_path=True)
                link_phi = self.burst_path(date_s, burst, 'phi.raw', full_path=True)
                link_dem = self.burst_path(date_s, burst, 'dem_radar.raw', full_path=True)

                if not os.path.exists(link_lam):
                    os.symlink(dat_lam, link_lam)
                if not os.path.exists(link_phi):
                    os.symlink(dat_phi, link_phi)
                if not os.path.exists(link_dem):
                    os.symlink(dat_dem, link_dem)

        # Create symlinks for images
        for date in dates:
            dat_lam = self.image_path(self.master_date, 'lam.raw')
            dat_phi = self.image_path(self.master_date, 'phi.raw')
            dat_dem = self.image_path(self.master_date, 'dem_radar.raw')

            link_lam = self.image_path(date, 'lam.raw')
            link_phi = self.image_path(date, 'phi.raw')
            link_dem = self.image_path(date, 'dem_radar.raw')

            if not os.path.exists(link_lam):
                os.symlink(dat_lam, link_lam)
            if not os.path.exists(link_phi):
                os.symlink(dat_phi, link_phi)
            if not os.path.exists(link_dem):
                os.symlink(dat_dem, link_dem)

    def concatenate(self, burst_file, master_file, dt=np.dtype(np.float32), overwrite=False, dates=[], multilooked='False', res_type='master'):
        # Concatenate all burst to a single full swath product. If burst_file = 'master' then the input master files are read...
        # This function also accepts cpxint16 datatype

        if not dates:
            dates = list(self.stack.keys())
        job_list1 = []

        for date in dates:
            path = self.image_path(date)
            final_path = os.path.join(path, master_file)

            if not os.path.exists(final_path) or overwrite == True:
                command1 = sys.executable + ' ' + os.path.join(self.function_path, 'concatenate_decatenate.py') + ' ' + path \
                           + ' concatenate ' + burst_file + ' ' + dt.name + ' ' + multilooked + ' ' + res_type
                job_list1.append({"path": path, "command": command1})

            if not self.parallel:
                os.chdir(path)
                os.system(command1)
        if self.parallel:
            jobs = Jobs(self.nr_of_jobs, self.doris_parameters)
            jobs.run(job_list1)

    def multilook(self, ra=40, az=10, step='filtphase'):
        # This function does the multilooking using cpxfiddle and updates the resolution of the step variable. You
        # have to careful that if you want to perform this step to follow on with a smaller data file, for e.g. unwrapping
        # this should be the last mentioned step.

        if step == 'filtphase':
            filename = 'cint.0.2filtered'
            filename2 = 'cint_filt_ml.raw'
            type = 'cr4'
        elif step == 'coherence':
            filename = 'coherence.raw'
            filename2 = 'coherence_ml.raw'
            type = 'r4'
        elif step == 'subtr_refdem':
            filename = 'cint_srd.raw'
            filename2 = 'cint_srd_ml.raw'
            type = 'cr4'
        elif step == 'subtr_refpha':
            filename = 'cint_srp.raw'
            filename2 = 'cint_srp_ml.raw'
            type = 'cr4'
        elif step == 'interfero':
            filename = 'cint.raw'
            filename2 = 'cint_ml.raw'
            type = 'cr4'
        else:
            print('Choose for step between filtphase, coherence, subtrefdem, subtrefpha and interfero')

        self.read_res()

        for date in self.coreg_dates:
            print(date)

            lines = int(self.full_swath[date]['master'].processes['readfiles']['Number_of_lines_original'])
            pixels = int(self.full_swath[date]['master'].processes['readfiles']['Number_of_pixels_original'])

            date_path = self.image_path(date)
            os.chdir(date_path)

            # Create cpxfiddle command
            command = ' -w ' + str(pixels) + ' -o float -M ' + str(ra) + '/'+ str(az) + ' -f ' + type + ' ' \
                                             '-l1 -p1 -P' + str(pixels) + ' -q normal ' + filename + ' > ' + filename2
            os.system(self.cpxfiddle + command)

            # Update res file
            new_lines = str(int(np.floor(lines / az)))
            new_pixels = str(int(np.floor(pixels / ra)))

            res = self.full_swath[date]['ifgs'].processes[step]

            res['Data_output_file'] = filename2
            res['Multilookfactor_azimuth_direction'] = str(az)
            res['Multilookfactor_range_direction'] = str(ra)
            res['Number of lines (multilooked)'] = new_lines
            res['Number of pixels (multilooked)'] = new_pixels

            self.full_swath[date]['ifgs'].processes[step] = res

            # Finally create an image using cpxfiddle (full resolution)
            if type == 'r4':
                # Only show the magnitude
                if step == 'coherence':
                    mag = ' -w ' + new_pixels + ' -q normal -o sunraster -b -c gray -M 20/5 -r 0.0/1.0 -f r4 -l1 ' \
                                            '-p1 -P' + new_pixels + ' ' + filename2 + ' > ' + filename2[:-4] + '.ras'
                else:
                    mag = ' -w ' + new_pixels + ' -e 0.3 -s 1.0 -q normal -o sunraster -b -c gray -M 1/1 -f r4 -l1 ' \
                                            '-p1 -P' + new_pixels + ' ' + filename2 + ' > ' + filename2[:-4] + '.ras'
                os.system(self.cpxfiddle + mag)
            elif type == 'cr4':
                # Show the 3 images
                mag = ' -w ' + new_pixels + ' -e 0.3 -s 1.0 -q mag -o sunraster -b -c gray -M 1/1 -f cr4 -l1 ' \
                                                 '-p1 -P' + new_pixels + ' ' + filename2 + ' > ' + filename2[:-4] + '_mag.ras'
                os.system(self.cpxfiddle + mag)
                mix = ' -w ' + new_pixels + ' -e 0.3 -s 1.2 -q mixed -o sunraster -b -c jet -M 1/1 -f cr4 -l1 ' \
                                                 '-p1 -P' + new_pixels + ' ' + filename2 + ' > ' + filename[:-4] + '_mix.ras'
                os.system(self.cpxfiddle + mix)
                pha = ' -w ' + new_pixels + ' -q phase -o sunraster -b -c jet -M 1/1 -f cr4 -l1 ' \
                                                 '-p1 -P' + new_pixels + ' ' + filename2 + ' > ' + filename2[:-4] + '_pha.ras'
                os.system(self.cpxfiddle + pha)

        self.update_res()

    def decatenate(self, burst_file, master_file, dt=np.dtype(np.float32), dates=[], multilooked=False, res_type='master'):
        # Split full swath into different burst products. (to be used for DEM result splitting)
        
        if not dates:
            dates = list(self.stack.keys())
        job_list1 = []

        for date in dates:
            path = self.image_path(date)

            command1 = os.path.join(self.function_path, 'concatenate_decatenate.py') + ' ' + path + ' decatenate ' + burst_file + ' ' + dt.str
            job_list1.append({"path": path, "command": command1})
            if not(self.parallel):
                os.chdir(path)
                os.system(command1)
        if (self.parallel):
            jobs = Jobs(self.nr_of_jobs, self.doris_parameters)
            jobs.run(job_list1)

    # Following functions are helper function which help acces the correct folders and files more easily:

    def burst_path(self, date='', key='', file_path='', stack_folder=False, dat_type='', full_path=False):
        if stack_folder == False:
            stack_folder = self.stack_folder

        if dat_type:
            file_path = '_iw_' + key[6] + '_burst_' + key[14:]
            if dat_type == 'master' or dat_type == 'slave':
                file_path = dat_type + file_path + '.raw'
            elif dat_type == 'master_deramped' or dat_type == 'slave_deramped':
                file_path = dat_type[:-9] + file_path + '_deramped.raw'

        if full_path is True:
            if len(date) == 10:
                date_folder = date[:4] + date[5:7] + date[8:10]
            else:
                date_folder = date
            swath_folder = key[:7]
            burst_folder = key[8:]
            if file_path:
                file_path = os.path.join(stack_folder, date_folder, swath_folder, burst_folder, file_path)
            else:
                file_path = os.path.join(stack_folder, date_folder, swath_folder, burst_folder)

        return file_path

    def burst_paths(self, stack, dates=[], stack_folder=False, dat_type='', file_path='', full_path=False):
        # Creates a list of all burst paths for this file, given the dates.
        if stack_folder == False:
            stack_folder = self.stack_folder
        if not dates:
            dates = list(stack.keys())

        paths = []
        for date in dates:
            for key in stack[date].keys():
                burst_path = self.burst_path(date, key, stack_folder=stack_folder, dat_type=dat_type, file_path=file_path, full_path=full_path)
                paths.append(burst_path)

        return paths

    def swath_path(self, date, key, stack_folder=False):
        if stack_folder == False:
            stack_folder = self.stack_folder

        if len(date) == 10:
            date_folder = date[:4] + date[5:7] + date[8:10]
        else:
            date_folder = date
        swath_burst = key.split('_')
        file_path = os.path.join(stack_folder, date_folder, swath_burst[0] + '_' + swath_burst[1])

        return file_path

    def swath_paths(self, stack, dates=[], stack_folder=False):
        # Creates a list of all swath paths given the dates.
        if stack_folder == False:
            stack_folder = self.stack_folder
        if not dates:
            dates = list(stack.keys())

        paths = []
        for date in dates:
            keys = set([key[:7] for key in stack[date].keys()])
            for key in keys:
                swath_path = self.swath_path(date, key, stack_folder)
                paths.append(swath_path)
        return paths

    def image_path(self, date, file_path='', stack_folder=False):
        if stack_folder == False:
            stack_folder = self.stack_folder

        if len(date) == 10:
            date_folder = date[:4] + date[5:7] + date[8:10]
        else:
            date_folder = date
        if file_path:
            file_path = os.path.join(stack_folder, date_folder, file_path)
        else:
            file_path = os.path.join(stack_folder, date_folder)

        return file_path

    def image_paths(self, stack, dates=[], file_path='', stack_folder=False):
        # Creates a list of all image paths
        if stack_folder == False:
            stack_folder = self.stack_folder
        if not dates:
            dates = list(stack.keys())

        paths = []
        for date in dates:
            image_path = self.image_path(date, file_path, stack_folder)
            paths.append(image_path)

        return paths

    @staticmethod
    def read_image_paths(master_key, stack_folder):
        # This functions reads all the current folders from a stack.

        # Select different categories of directories.
        stack_dirs = next(os.walk(stack_folder))[1]
        master = os.path.join(stack_folder, master_key)
        esd = os.path.join(stack_folder, 'esd')
        slaves = [os.path.join(stack_folder, dir_s) for dir_s in stack_dirs if dir_s != master_key and len(dir_s) == 8]
        ifg = [os.path.join(stack_folder, dir_s) for dir_s in stack_dirs if len(dir_s) == 21]

        return master, slaves, ifg, esd

    def read_burst_paths(self, master_key, stack_folder, dat_type='list'):
        # The output can be in 2 formats. A list with all
        # folders, including image and burst folders, or a dict structure.
        if dat_type == 'list':
            folders = [[], [], []]
        elif dat_type == 'dict':
            folders = [dict(), dict(), dict()]
        else:
            print('dat_type should either be a list or dict')
            return

        master, slaves, ifg = self.read_image_paths(master_key, stack_folder)[:3]

        # And find the corresponding folders:
        for fold, dirs in zip(folders, [[master], slaves, ifg]):
            for direc in dirs:
                if dat_type == 'list':
                    fold.append(os.path.basename(direc))
                elif dat_type == 'dict':
                    fold[os.path.basename(direc)] = dict()
                swaths = next(os.walk(direc))[1]
                for swath in swaths:
                    if dat_type == 'dict':
                        fold[os.path.basename(direc)][swath] = dict()
                    bursts = next(os.walk(os.path.join(direc, swath)))[1]
                    for burst in bursts:
                        if dat_type == 'dict':
                            fold[os.path.basename(direc)][swath][burst] = []
                        elif dat_type == 'list':
                            fold.append(os.path.join(os.path.basename(direc), swath, burst))

        # Now return master, slave and ifg setup
        return folders[0], folders[1], folders[2]

    def update_res(self, dates='default', stack_folder='', burst_stack=list(), image_stack=list()):
        # Save to .res file based on the burst objects.
        if not burst_stack:
            burst_stack = self.stack
        if not image_stack:
            image_stack = self.full_swath
        if dates == 'default':
            dates = list(self.stack.keys())
        if not stack_folder:
            stack_folder = self.stack_folder

        for date in dates:
            for burst in burst_stack[date].keys():

                files = list(burst_stack[date][burst].keys())
                if 'slave' in files:
                    slave_res = self.burst_path(date, burst, 'slave.res', stack_folder=stack_folder, full_path=True)
                    burst_stack[date][burst]['slave'].write(new_filename=slave_res)
                if 'master' in files:
                    master_res = self.burst_path(date, burst, 'master.res', stack_folder=stack_folder, full_path=True)
                    burst_stack[date][burst]['master'].write(new_filename=master_res)
                if 'ifgs' in files:
                    ifgs_res = self.burst_path(date,burst,'ifgs.res', stack_folder=stack_folder, full_path=True)
                    burst_stack[date][burst]['ifgs'].write(new_filename=ifgs_res)

            files = list(image_stack[date].keys())
            if 'slave' in files:
                slave_res = self.image_path(date, 'slave.res', stack_folder=stack_folder)
                image_stack[date]['slave'].write(new_filename=slave_res)
            if 'master' in files:
                master_res = self.image_path(date, 'master.res', stack_folder=stack_folder)
                image_stack[date]['master'].write(new_filename=master_res)
            if 'ifgs' in files:
                ifgs_res = self.image_path(date, 'ifgs.res', stack_folder=stack_folder)
                image_stack[date]['ifgs'].write(new_filename=ifgs_res)

    def read_res(self, coreg_dates=False, dates='default', stack_folder='', burst_stack='', image_stack='', bursts=[]):
        # Read .res data to the burst objects. Generally done after a processing step.
        if burst_stack == '':
            burst_stack = self.stack
        if image_stack == '':
            image_stack = self.full_swath
        if dates == 'default':
            dates = list(self.stack.keys())
        if not stack_folder:
            stack_folder = self.stack_folder
        if not bursts:
            if dates[0] in burst_stack.keys():
                bursts = list(burst_stack[dates[0]].keys())
            else:
                date_1 = list(self.stack.keys())[0]
                bursts = list(self.stack[date_1].keys())
        # TODO Maybe add search for folders and bursts if no specific date or burst is specified?

        for date in dates:
            if not date in burst_stack.keys():
                burst_stack[date] = dict()
            for burst in bursts:
                if not burst in burst_stack[date].keys():
                    burst_stack[date][burst] = dict()

                slave_res = self.burst_path(date, burst, 'slave.res', stack_folder=stack_folder, full_path=True)
                master_res = self.burst_path(date, burst, 'master.res', stack_folder=stack_folder, full_path=True)
                ifgs_res = self.burst_path(date, burst,'ifgs.res', stack_folder=stack_folder, full_path=True)

                if os.path.exists(slave_res):
                    burst_stack[date][burst]['slave'] = ResData(filename=slave_res)
                if os.path.exists(master_res):
                    burst_stack[date][burst]['master'] = ResData(filename=master_res)
                if os.path.exists(ifgs_res):
                    burst_stack[date][burst]['ifgs'] = ResData(filename=ifgs_res)

            if not date in image_stack.keys():
                image_stack[date] = dict()

            slave_res = self.image_path(date, 'slave.res', stack_folder=stack_folder)
            master_res = self.image_path(date, 'master.res', stack_folder=stack_folder)
            ifgs_res = self.image_path(date, 'ifgs.res', stack_folder=stack_folder)

            if os.path.exists(slave_res):
                image_stack[date]['slave'] = ResData(filename=slave_res)
            if os.path.exists(master_res):
                image_stack[date]['master'] = ResData(filename=master_res)
            if os.path.exists(ifgs_res):
                image_stack[date]['ifgs'] = ResData(filename=ifgs_res)

    def del_res(self, type='ifgs', images=False, bursts=True, dates='default', stack_folder=''):

        if dates == 'default':
            dates = list(self.stack.keys())

        for date in dates:
            for burst in self.stack[date].keys():
                if bursts:
                    res = self.burst_path(date,burst, type + '.res', stack_folder=stack_folder)
                    if os.path.exists(res):
                        os.remove(res)

            if images:
                res = self.image_path(date, type + '.res', stack_folder=stack_folder)
                if os.path.exists(res):
                    os.remove(res)

    def del_process(self, process, type='ifgs', images=False, bursts=True, dates='default', burst_stack=list(), image_stack=list()):
        # Delete a process from the .res files.

        if not burst_stack:
            burst_stack = self.stack
        if not image_stack:
            image_stack = self.full_swath
        if dates == 'default':
            dates = list(self.stack.keys())

        self.read_res(dates=dates) # Read data

        for date in dates:
            for burst in burst_stack[date].keys():
                if bursts and burst_stack[date][burst][type].process_control[process] == '1':
                    burst_stack[date][burst][type].delete(process)
            if images and image_stack[date][type].process_control[process] == '1':
                image_stack[date][type].delete(process)
        self.update_res(dates=dates)