#!/Users/boydb1/anaconda/bin/python
# -*- coding: utf-8 -*-

""" Executables to manage project with REDCap
    by generating the settings file automatically

    dax_manager SHOULD run as a crontab job every hour
"""

import os
import sys
import redcap
import smtplib
import traceback
import subprocess
import multiprocessing
from string import Template
from datetime import datetime
from email.mime.text import MIMEText
from subprocess import CalledProcessError

import dax
from dax import ADMIN_EMAIL, RESULTS_DIR, DEFAULT_GATEWAY,\
                REDCAP_VAR, API_URL, API_KEY_DAX, SMTP_FROM, SMTP_HOST,\
                SMTP_PASS, DEFAULT_EMAIL_OPTS, DEFAULT_MAX_AGE
from dax.launcher import BUILD_SUFFIX, UPDATE_SUFFIX, LAUNCH_SUFFIX

#Information on gateway/user
USER = os.environ['USER']
try:
    GATEWAY = subprocess.check_output('hostname', stderr=subprocess.STDOUT, shell=True).strip()
except (CalledProcessError, ValueError):
    GATEWAY = DEFAULT_GATEWAY

#Template for Module:
MOD_TEMPLATE = Template("""${PROJECT}_${MODNAME}=${MODNAME}(${ARGUMENTS})""")
#Template for Process:
PROC_TEMPLATE = Template("""${PROJECT}_${PROCNAME}=${PROCNAME}(${ARGUMENTS})""")
#Template for settings files:
SETTINGS_TEMPLATE = Template("""#import packages:
import os
from dax import Launcher
from Xnat_process_importer import *
from Xnat_module_importer import *

#Email & tmp folder:
email = "${EMAIL}"
tmpdir = "${TMPDIR}"

#Create TEMP if doesn't exist
if not os.path.exists(tmpdir):
    os.mkdir(tmpdir)

## Init proc and mod ##
#Mod
${MOD}
#Proc
${PROC}

#Set up modules for projects
proj_mod = ${PROJ_MOD}
#Set up processors for projects
proj_proc = ${PROJ_PROC}

#Order project:
p_order = ${PROJ_ORDER}

#Launch jobs:
myLauncher = Launcher(proj_proc, proj_mod, priority_project=p_order, job_email=email, job_email_options="${EMAIL_OPTS}", queue_limit=${QUEUE_LIMIT}, max_age=${MAX_AGE})
""")

EMAIL_HEADER = """Long updates running on <{gateway}> for the user <{user}>:
----------------------------------------------------------------------------------
"""

#Class for settings:
class Settings_File:
    """ Class to generate settings file """
    def __init__(self, filepath, tmpdir='/tmp/', email=None,
                 email_opts=DEFAULT_EMAIL_OPTS, dax_logs_path='/tmp/',
                 queue_limit=200, max_age=DEFAULT_MAX_AGE, admin_email=ADMIN_EMAIL):
        """ init function """
        self.filepath = filepath
        #email
        if isinstance(email, list):
            self.email = email
        else:
            self.email = email.split(',')
        #email_opts
        self.email_opts = email_opts
        #tmpdir
        self.tmpdir = tmpdir
        #Project order
        self.proj_order = list()
        #String with all modules definitions
        self.mod_str = ''
        #dict with project: [] list of modules
        self.pm_dict = dict()
        #String with all processes definitions
        self.proc_str = ''
        #dict with project: [] list of processes
        self.pp_dict = dict()
        #logs for dax path
        self.dax_logs_path = dax_logs_path
        #Maximum Number of job that can be submitted
        if queue_limit.isdigit():
            self.queue_limit = int(queue_limit)
        else:
            self.queue_limit = 400
        #Admin email to receve error from dax
        if isinstance(email, list):
            self.admin_email = admin_email
        else:
            self.admin_email = admin_email.split(',')
        #max age for sessions:
        if not max_age:
            self.max_age = DEFAULT_MAX_AGE
        else:
            self.max_age = max_age

    def add_mod_str(self, project, mod_str):
        """ add a module string for a new module """
        self.mod_str += mod_str+'\n'
        if project in self.pm_dict.keys():
            self.pm_dict[project].append(mod_str.split('=')[0])
        else:
            self.pm_dict[project] = [mod_str.split('=')[0]]

    def add_proc_str(self, project, proc_str):
        """ add a processor string for a new proc """
        self.proc_str += proc_str+'\n'
        if project in self.pp_dict.keys():
            self.pp_dict[project].append(proc_str.split('=')[0])
        else:
            self.pp_dict[project] = [proc_str.split('=')[0]]

    def get_project_list(self):
        """ return the list of project """
        return set(self.pm_dict.keys()+self.pp_dict.keys())

    def merge_settings(self, settings):
        """ merge the settings for different projects """
        #Email:
        if list(set(self.email)-set(settings.email)):
            self.email += list(set(self.email)-set(settings.email))
        if list(set(self.admin_email)-set(settings.admin_email)):
            self.admin_email += list(set(self.admin_email)-set(settings.admin_email))
        #queue_limit (difference divide by 2):
        self.queue_limit = (self.queue_limit+settings.queue_limit)/2
        #Add mod_str and proc_str
        self.mod_str += settings.mod_str
        self.proc_str += settings.proc_str
        #Add project to dict:
        self.pm_dict.update(settings.pm_dict)
        self.pp_dict.update(settings.pp_dict)

    def set_order(self, proj_order):
        """ set the order for the project """
        self.proj_order = proj_order

    def get_order(self):
        """ get the order for the project """
        order_str = '['
        for project in self.proj_order:
            order_str += """'{project}',""".format(project=project)
        if order_str[-1] == ',':
            order_str = order_str[:-1]
        order_str += ']'
        return order_str

    def get_admin_email(self):
        """ return the list of the admin emails """
        email_list = list()
        if '' in self.email:
            self.email.remove('')

        if not self.admin_email:
            return email_list
        else:
            for email in self.admin_email:
                if email.find('@') == -1:
                    e_addr = email+'@vanderbilt.edu'
                else:
                    e_addr = email
                email_list.append(e_addr)
            return email_list

    def get_email_list(self):
        """ return the list of the emails """
        email_list = list()
        if '' in self.email:
            self.email.remove('')

        if not self.email:
            return email_list
        else:
            for email in self.email:
                if email.find('@') == -1:
                    e_addr = email+'@vanderbilt.edu'
                else:
                    e_addr = email
                email_list.append(e_addr)
            return email_list

    def get_email_opts(self):
        """ return the email options """
        if self.email_opts == '':
            return DEFAULT_EMAIL_OPTS
        else:
            return self.email_opts

    def get_pm_dict(self):
        """ return the dict for the project modules """
        pm_dict_str = '{'
        first_elem = True
        spacing = 12*" "
        for project in sorted(self.pm_dict):
            #Generate the string:
            dict_elem = """'{project}':[""".format(project=project)
            for mod in self.pm_dict[project]:
                dict_elem += mod+','
            if dict_elem[-1] == ',':
                dict_elem = dict_elem[:-1]
            dict_elem += '],\n'
            #Keep a nice format:
            if not first_elem:
                pm_dict_str += spacing+dict_elem
            else:
                pm_dict_str += dict_elem
            first_elem = False
        #Remove the last comma
        if pm_dict_str.strip().endswith(','):
            pm_dict_str = pm_dict_str[:-2] #for the \n
        pm_dict_str += '}'
        return pm_dict_str

    def get_pp_dict(self):
        """ return the dict for the project processors """
        pp_dict_str = '{'
        first_elem = True
        spacing = 13*" "
        for project in sorted(self.pp_dict):
            #Generate the string:
            dict_elem = """'{project}':[""".format(project=project)
            for proc in self.pp_dict[project]:
                dict_elem += proc+','
            if dict_elem[-1] == ',':
                dict_elem = dict_elem[:-1]
            dict_elem += '],\n'
            #Keep a nice format:
            if not first_elem:
                pp_dict_str += spacing+dict_elem
            else:
                pp_dict_str += dict_elem
            first_elem = False
        #Remove the last comma
        if pp_dict_str.strip().endswith(','):
            pp_dict_str = pp_dict_str[:-2] #for the \n
        pp_dict_str += '}'
        return pp_dict_str

def get_list_modproc(redcap_project):
    """ return the list of modules and processors """
    #Getting modules/processors
    all_forms = list()
    fnames, _ = redcap_project.names_labels()
    for k in fnames:
        try:
            field = filter(lambda x: x['field_name'] == k, redcap_project.metadata)[0]
            all_forms.append(field['form_name'])
        except IndexError as _:
            LOGGER.error('IndexError when checking the libraries.', exc_info=True)
            sys.exit()
    #split module from process:
    list_forms = set(all_forms)
    mod = [name for name in list_forms if name[:6] == 'module']
    proc = [name for name in list_forms if name[:7] == 'process']
    return mod, proc

def get_field_for_modproc(redcap_project, form_name):
    """ return the list of the variables for the module """
    avoid_labels = [form_name[7:]+'_inputs',
                    form_name[7:]+'_on',
                    form_name[8:]+'_inputs',
                    form_name[8:]+'_on']
    fields_list = list()
    fnames, _ = redcap_project.names_labels()
    for k in fnames:
        try:
            field = filter(lambda x: x['field_name'] == k, redcap_project.metadata)[0]
            if field['form_name'] == form_name and k not in avoid_labels:
                fields_list.append(field)
        except IndexError as _:
            LOGGER.error('IndexError when checking the libraries.', exc_info=True)
            sys.exit()
    return fields_list

def read_redcap_db(redcap_project):
    """ read the project on redcap """
    #Extract all attributes for the records that belong to the gateway/user we are on:
    fields_search = [redcap_project.def_field, REDCAP_VAR['user'], REDCAP_VAR['gateway']]
    record_list = redcap_project.export_records(fields=fields_search)
    records = [record[redcap_project.def_field] for record in record_list \
               if record[REDCAP_VAR['user']] == USER and record[REDCAP_VAR['gateway']] == GATEWAY]
    if records:
        return redcap_project.export_records(records=records)
    else:
        return list()

def generate_settings(gateway, user, redcap_project, settings_info):
    """ create the settings file """
    #Variable:
    settings_dict = dict()
    modules_list, processes_list = get_list_modproc(redcap_project)
    LOGGER.info('Projects found on Redcap: ')
    if not settings_info:
        LOGGER.info('  - No project found')

    update_email_dict = {'dax_build still running after 2 days':[],
                         'dax_build still running after 7 days':[],
                         'dax_launch still running after 2 days':[],
                         'dax_launch still running after 7 days':[],
                         'dax_update_tasks still running after 2 days':[],
                         'dax_update_tasks still running after 7 days':[]}

    #Organize the data to in complex dictionaries:
    for project_settings in settings_info:
        user = project_settings[REDCAP_VAR['user']]
        gateway = project_settings[REDCAP_VAR['gateway']]
        mess = """  - {project} for settings {settings}"""
        message = mess.format(project=project_settings[REDCAP_VAR['project']],
                              settings=project_settings[REDCAP_VAR['settingsfile']])
        LOGGER.info(message)
        if project_settings['general_complete'] == '2':
            #Check the process:
            update_email_dict = check_executables(project_settings, update_email_dict)
            #New PSettings
            psettings = Settings_File(filepath=project_settings[REDCAP_VAR['settingsfile']],
                                      tmpdir=project_settings[REDCAP_VAR['tmp']],
                                      email=project_settings[REDCAP_VAR['email']],
                                      email_opts=project_settings[REDCAP_VAR['email_opts']],
                                      dax_logs_path=project_settings[REDCAP_VAR['logsdir']],
                                      queue_limit=project_settings[REDCAP_VAR['queue']],
                                      max_age=project_settings[REDCAP_VAR['max_age']],
                                      admin_email=project_settings[REDCAP_VAR['admin_email']])

            #Look for modules
            for mod_name in modules_list:
                args_mod_dict = dict()
                #If module on:
                if project_settings[mod_name[7:]+'_on'] == '1' and \
                   project_settings[mod_name+'_complete'] == '2':
                    #for variables from the module:
                    for field in get_field_for_modproc(redcap_project, mod_name):
                        field_name = field['field_name'].split(mod_name[7:]+'_')[1]
                        args_mod_dict[field_name] = project_settings[field['field_name']]
                        #Get ModName:
                        if field_name == 'mod_name':
                            if project_settings[field['field_name']] == '':
                                modname = field['field_note'].strip().split('Default: ')[1]
                            else:
                                modname = project_settings[field['field_name']]
                    #Add tmpdir for modules:
                    folder_name = project_settings[REDCAP_VAR['project']]+'_'+mod_name[7:]
                    args_mod_dict['directory'] = os.path.join(project_settings[REDCAP_VAR['tmp']],
                                                              folder_name)
                    module = generate_mod(project_settings[REDCAP_VAR['project']],
                                          modname,
                                          args_mod_dict)
                    psettings.add_mod_str(project_settings[REDCAP_VAR['project']], module)

            #Look for processes
            for proc_name in processes_list:
                args_proc_dict = dict()
                #Number of process of this type: if ';' in version or processName or spiderpath
                nb_proc = 1
                #If process on:
                if project_settings[proc_name[8:]+'_on'] == '1' and \
                   project_settings[proc_name+'_complete'] == '2':
                    #Check the number of process (fiels version, processname,spiderpath
                    nb_proc = check_number_process(project_settings, proc_name)
                    #For each proc define by the tag (version or processName)
                    for index in range(nb_proc):
                        #for variables from the process:
                        for field in get_field_for_modproc(redcap_project, proc_name):
                            #Get ProcName:
                            if field['field_name'] == proc_name[8:]+'_proc_name':
                                procname = read_procname(project_settings,
                                                         index,
                                                         field['field_name'],
                                                         field['field_note'])
                            else:
                                field_name = field['field_name'].split(proc_name[8:]+'_')[1]
                                args_proc_dict[field_name] = read_value(project_settings,
                                                                        index,
                                                                        field['field_name'],
                                                                        field['field_note'])
                        processors = generate_proc(project_settings[REDCAP_VAR['project']],
                                                   procname, args_proc_dict, str(index))
                        psettings.add_proc_str(project_settings[REDCAP_VAR['project']], processors)

            if psettings.filepath in settings_dict.keys():
                settings_dict[psettings.filepath].merge_settings(psettings)
            else:
                settings_dict[psettings.filepath] = psettings
    LOGGER.info('\n')

    email_text = EMAIL_HEADER.format(gateway=gateway, user=user)
    for header, proj_list in update_email_dict.items():
        if proj_list:
            email_text += '\t'+header+' -->  '+' , '.join(proj_list)+'\n'
            email_text += '-----------------------------------------'\
                         +'-----------------------------------------\n'

    if email_text != EMAIL_HEADER:
        subject = 'Dax_manager ADMIN: updates running for 2/7days '
        if ADMIN_EMAIL:
            email(email_text, ADMIN_EMAIL, subject)

    return settings_dict

def get_status(settings_info):
    """ get the status of the dax executables that are running """
    LOGGER.info('Status on each settings updates:')
    separator = '-'*160
    LOGGER.info(' '+separator)
    header = '| %*s | %*s | %*s | %*s | %*s | %*s | %*s | %*s |' % (-23, 'Project',
                                                                    -3, 'R',
                                                                    -17, 'Last Build',
                                                                    -20, 'Build Info',
                                                                    -17, 'Last Launch',
                                                                    -20, 'Launch Info',
                                                                    -17, 'Last Update Tasks',
                                                                    -20, 'Update Tasks Info')
    LOGGER.info(header)
    LOGGER.info(' '+separator)
    for pset in sorted(settings_info, key=lambda k: k['general_complete']):
        if pset:
            projectstr = smaller_str(str(pset[REDCAP_VAR['project']]), size=23)
            status = get_status_string(pset['general_complete'])
            build_start = str(pset[REDCAP_VAR['dax_build_start_date']][:-3])
            launch_start = str(pset[REDCAP_VAR['dax_launch_start_date']][:-3])
            up_tasks_start = str(pset[REDCAP_VAR['dax_update_tasks_start_date']][:-3])
            build_info = get_update_info(pset[REDCAP_VAR['dax_build_start_date']], 
                                         pset[REDCAP_VAR['dax_build_end_date']],
                                         pset[REDCAP_VAR['dax_build_pid']])
            launch_info = get_update_info(pset[REDCAP_VAR['dax_launch_start_date']], 
                                          pset[REDCAP_VAR['dax_launch_end_date']],
                                          pset[REDCAP_VAR['dax_launch_pid']])
            up_tasks_info = get_update_info(pset[REDCAP_VAR['dax_update_tasks_start_date']], 
                                            pset[REDCAP_VAR['dax_update_tasks_end_date']],
                                            pset[REDCAP_VAR['dax_update_tasks_pid']])
            val = '| %*s | %*s | %*s | %*s | %*s | %*s | %*s | %*s |' % (-23, projectstr,
                                                                         -3, status,
                                                                         -17, build_start,
                                                                         -20, build_info,
                                                                         -17, launch_start,
                                                                         -20, launch_info,
                                                                         -17, up_tasks_start,
                                                                         -20, up_tasks_info)
            LOGGER.info(val)
    LOGGER.info(' '+separator)

def get_status_string(status):
    """ return ON if project is ON on REDCap, OFF else """
    if str(status) == '2':
        return 'Y'
    else:
        return 'N'

def get_update_info(update_start, update_end, update_pid):
    """ return string update information """
    if update_end == 'In Process':
        return update_end+' PID:'+update_pid
    else : # both are datetimes
        date_format = '%Y-%m-%d %H:%M:%S'
        try:
            time_delta = datetime.strptime(update_end, date_format) - datetime.strptime(update_start, date_format)
            secs = time_delta.total_seconds()
            hours = int(secs // 3600)
            mins = int((secs % 3600) // 60)
            if hours > 0:
                elapsed_time = '{} hrs {} mins'.format(hours, mins)
            else:
                elapsed_time = '{} mins'.format(mins)

        except ValueError:
            elapsed_time = update_end[:-3]

        return elapsed_time

def smaller_str(str_option, size, end=False):
    """ return a smaller string """
    if len(str_option) > size:
        if end:
            return '...'+str_option[-size-3:]
        else:
            return str_option[:size-3]+'...'
    else:
        return str_option

def check_number_process(project_settings, proc_name):
    """ return the number of processes if more than just one set """
    proc_name_tags = project_settings[proc_name[8:]+'_proc_name'].split(';')
    spider_tags = project_settings[proc_name[8:]+'_spider_path'].split(';')
    suffix_tags = project_settings[proc_name[8:]+'_suffix_proc'].split(';')
    version_tags = project_settings[proc_name[8:]+'_version'].split(';')
    return max(len(l) for l in [proc_name_tags, spider_tags, suffix_tags, version_tags])

def read_procname(project_settings, index, field_name, field_note):
    """ return procname from REDCap"""
    if ';' in project_settings[field_name]:
        value = project_settings[field_name].split(';')
        if len(value) <= index:
            mess = """wrong procname with ";" in {record_id} and field {field}: using default"""
            message = mess.format(record_id=project_settings[REDCAP_VAR['project']],
                                  field=field_name)
            LOGGER.warn(message)
            return field_note.strip().split('Default: ')[1]
        else:
            if value[index] == '':
                return field_note.strip().split('Default: ')[1]
            else:
                return value[index]
    else:
        if project_settings[field_name] == '':
            return field_note.strip().split('Default: ')[1]
        else:
            return project_settings[field_name]

def read_value(project_settings, index, field_name, field_note):
    """ return the value for a field """
    if ';' in project_settings[field_name]:
        value = project_settings[field_name].split(';')
        if len(value) <= index:
            mess = """wrong procname with ";" in {record_id} and field {field}: using default"""
            message = mess.format(record_id=project_settings[REDCAP_VAR['project']],
                                  field=field_name)
            LOGGER.warn(message)
            return ''
        else:
            if value[index] == '':
                return ''
            else:
                return value[index]
    else:
        return project_settings[field_name]

def generate_proc(project, procname, args_dict, tag):
    """ return the processor call to write down in the settings """
    #Get the arguments
    arguments = dict2str(args_dict)
    #Get the string for a processor
    proc_data = {'PROJECT':project.replace('-', '_')+'_'+tag,
                 'PROCNAME':procname,
                 'ARGUMENTS':arguments}

    return PROC_TEMPLATE.safe_substitute(**proc_data)

def generate_mod(project, modname, args_dict):
    """ return the module call to write down in the settings """
    #Get the arguments
    arguments = dict2str(args_dict)
    #Get the string for a processor
    mod_data = {'PROJECT':project.replace('-', '_'),
                'MODNAME':modname,
                'ARGUMENTS':arguments}

    return MOD_TEMPLATE.safe_substitute(**mod_data)

def dict2str(arg_dict):
    """ convert a dict into a string """
    arg_str = ''
    for key, value in arg_dict.items():
        if value != '':
            arg_str += key+'="'+value+'",'
    if arg_str:
        return arg_str[:-1]  #remove last comma
    else:
        return ''

def ordering_project(setting, settings_info):
    """ return the order for the projects """
    list_priority = dict()
    #Organize the data to in complex dictionaries:
    for project in setting.get_project_list():
        for project_settings in settings_info:
            if project_settings[REDCAP_VAR['project']] == project and \
               project_settings[REDCAP_VAR['priority']] != '':
                list_priority[project] = int(project_settings[REDCAP_VAR['priority']])
    return sorted(list_priority, key=list_priority.get)

def write_settings(settings):
    """ write down the settings """
    # Check the Settings path:
    if '.py' not in os.path.basename(settings.filepath):
        LOGGER.error('Settings filepath is not a proper file. It needs the extension .py .')
    else:
        if not os.path.exists(os.path.dirname(settings.filepath)):
            os.makedirs(os.path.dirname(settings.filepath))
        # Write the Settings file
        settings_data = {'EMAIL':','.join(settings.get_email_list()),
                         'EMAIL_OPTS':settings.get_email_opts(),
                         'MAX_AGE':settings.max_age,
                         'TMPDIR':settings.tmpdir,
                         'PROJ_ORDER':settings.get_order(),
                         'QUEUE_LIMIT':settings.queue_limit,
                         'MOD':settings.mod_str,
                         'PROC':settings.proc_str,
                         'PROJ_MOD':settings.get_pm_dict(),
                         'PROJ_PROC':settings.get_pp_dict()}

        with open(settings.filepath, 'w') as f:
            f.write(SETTINGS_TEMPLATE.safe_substitute(**settings_data))

def email(content, to_addr, subject):
    """ send an email """
    if SMTP_FROM and SMTP_HOST and SMTP_PASS:
        # Create the container (outer) email message.
        msg = MIMEText(content)
        msg['Subject'] = subject
        # me == the sender's email address
        # family = the list of all recipients' email addresses
        msg['From'] = SMTP_FROM
        msg['To'] = ','.join(to_addr)
        # Send the email via our own SMTP server.
        smtp = smtplib.SMTP(SMTP_HOST)
        smtp.starttls()
        smtp.login(SMTP_FROM, SMTP_PASS)
        smtp.sendmail(SMTP_FROM, to_addr, msg.as_string())
        smtp.quit()

def unlock_executables(flagfile, suffix):
    """ unlock the executables by removing the flagfile """
    lockfile_prefix = os.path.splitext(os.path.basename(filepath))[0]
    flagfile = os.path.join(RESULTS_DIR, 'FlagFiles',
                            lockfile_prefix+'_'+suffix)
    if os.path.exists(flagfile):
        os.remove(flagfile)

def check_executables(project_settings, update_email_dict):
    """ check for how long the executables are running """
    today_date = int('{:%Y%m%d%H%M%S}'.format(datetime.now()))
    #dax_build:
    if 'In Process' in project_settings['dax_build_end_date']:
        update_email_dict = check_date(update_email_dict,
                                       project_settings[REDCAP_VAR['project']],
                                       project_settings['dax_build_start_date'],
                                       today_date,
                                       'dax_build')

    #dax_launch
    if 'In Process' in project_settings['dax_launch_end_date']:
        update_email_dict = check_date(update_email_dict,
                                       project_settings[REDCAP_VAR['project']],
                                       project_settings['dax_launch_start_date'],
                                       today_date,
                                       'dax_launch')

    #dax_update_tasks
    if 'In Process' in project_settings['dax_launch_end_date']:
        update_email_dict = check_date(update_email_dict,
                                       project_settings[REDCAP_VAR['project']],
                                       project_settings['dax_update_tasks_start_date'],
                                       today_date,
                                       'dax_update_tasks')

    return update_email_dict

def check_date(update_email_dict, project, start_date, now, exec_name):
    """ check the date for the executable """
    if start_date:
        start_date = int(start_date.replace('-', '').replace(':', '').replace(' ', ''))
        start_date_2days = start_date+2000000 #adding 2 days
        start_date_2days2hours = start_date+2020000 #adding 2 days - 2 hours
        start_date_7days = start_date+7000000 #adding 7 days
        if start_date_2days <= now <= start_date_2days2hours:
            update_email_dict[exec_name+' still running after 2 days'].append(project)
        if start_date_7days <= now:
            update_email_dict[exec_name+' still running after 7 days'].append(project)
    return update_email_dict

def get_logfile_name(logdir, prefix, filepath):
    """ return the name of the logfile """
    if not os.path.exists(logdir):
        os.makedirs(logdir)
    project = os.path.basename(filepath).split('.')[0]
    now = '{:%Y%m%d_%H:%M:%S}'.format(datetime.now())
    filename = """{prefix}_{project}_{date}.log""".format(prefix=prefix,
                                                          project=project,
                                                          date=now)
    return os.path.join(logdir, filename)

def email_error(exec_name, email_add, filepath):
    """ email if error in dax executables """
    temp = """Caught exception in worker "{exec_name}" {file} : \n{trace}"""
    content = temp.format(exec_name=exec_name,
                          file=filepath,
                          trace=traceback.format_exc())
    subject = """Error in {exec_name} for {file}""".format(exec_name=exec_name,
                                                           file=filepath)
    email(content, email_add, subject)
####################################################################################
#####################           Principal functions            #####################
####################################################################################
def generate_settings_REDCap(gateway, user, api_key, status=False):
    """Extract information from REDCap and generate the settings file"""
    settings_path_list = list()
    daxlogsdir_dict = dict()
    email_dict = dict()
    #get the information for xnat from redcap for the projects:
    LOGGER.info('Loading RedCap to extract information to run dax_updates...')
    try:
        redcap_project = redcap.Project(API_URL, api_key)
    except:
        LOGGER.error('Could not access redcap. Either wrong API_URL/API_KEY or redcap down.',
                     exc_info=True)
        sys.exit()

    #Extract the information about the settings from redcap
    settings_info = read_redcap_db(redcap_project)
    #Status:
    if status:
        get_status(settings_info)
    else:
        #Generate information for the settings
        settings_dict = generate_settings(gateway,
                                          user,
                                          redcap_project,
                                          settings_info)
        #For each settings in the dict:
        for filepath, setting in settings_dict.items():
            #Ordering project:
            setting.set_order(ordering_project(setting, settings_info))
            #Write the settings:
            LOGGER.debug(' -> Writing settings : '+filepath)
            write_settings(setting)
            settings_path_list.append(setting.filepath)
            daxlogsdir_dict[filepath] = setting.dax_logs_path
            email_dict[filepath] = setting.get_admin_email()

    return settings_path_list, daxlogsdir_dict, email_dict

def run_dax_build(filepath, logdir, debug, email_add):
    """Run dax_build with the settings file and redirect output to the logfile"""
    try:
        logfile = get_logfile_name(logdir, "buildlog", filepath)
        dax.bin.build(filepath, logfile, debug)
    except Exception as _:
        LOGGER.error('Caught exception in worker "dax_build '+filepath+'" : \n'\
                     +traceback.format_exc())
        #Remove the flagfile:
        unlock_executables(filepath, BUILD_SUFFIX)
        if email_add:
            email_error('dax_build', email_add, filepath)

def run_dax_launch(filepath, logdir, debug, email_add):
    """Run dax_launch with the settings file and redirect output to the logfile"""
    try:
        logfile = get_logfile_name(logdir, "launchlog", filepath)
        dax.bin.launch_jobs(filepath, logfile, debug)
    except Exception as _:
        LOGGER.error('Caught exception in worker "dax_launch '+filepath+'" : \n'\
                     +traceback.format_exc())
        #Remove the flagfile:
        unlock_executables(filepath, LAUNCH_SUFFIX)
        if email_add:
            email_error('dax_launch', email_add, filepath)

def run_dax_update_tasks(filepath, logdir, debug, email_add):
    """Run dax_update_task with the settings file and redirect output to the logfile"""
    try:
        logfile = get_logfile_name(logdir, "update_taskslog", filepath)
        dax.bin.update_tasks(filepath, logfile, debug)
    except Exception as _:
        LOGGER.error('Caught exception in worker "dax_update_open_tasks '+filepath\
                     +'" : \n'+traceback.format_exc())
        #Remove the flagfile:
        unlock_executables(filepath, UPDATE_SUFFIX)
        if email_add:
            email_error('dax_update_tasks', email_add, filepath)

def submit_exec(target, arguments, filepath, exec_name):
    """ submit executables on the gateway: return the process """
    LOGGER.debug("""new process {execn} on {file}""".format(execn=exec_name, file=filepath))
    proc = multiprocessing.Process(target=target, args=arguments)
    proc.start()
    return proc

def parse_args():
    """ options parser for the script """
    from argparse import ArgumentParser
    des = "dax_manager to manage project from redcap database or lists of settings files"
    argp = ArgumentParser(prog='dax_manager', description=des)
    argp.add_argument('--key', dest='key',
                      help='API Key for redcap project containing dax informations.', default=None)
    argp.add_argument('--settings', dest='settings', help='List of Settings Path', default=None)
    argp.add_argument('--daxlogsdir', dest='daxlogsdir',
                      help='Directory for dax logs file.Default: /tmp', default='/tmp')
    argp.add_argument('--writeonly', dest='writeonly', action='store_true',
                      help='Write the settings from REDCap only')
    argp.add_argument('--status', dest='status', action='store_true', help='Status on the updates')
    argp.add_argument('--email', dest='email', help='Email address to send errors.', default=None)
    argp.add_argument('--logfile', dest='logfile', help='Logs file path if needed.', default=None)
    argp.add_argument('--nodebug', dest='debug', action='store_false',
                      help='Avoid printing DEBUG information.')
    return argp.parse_args()

if __name__ == '__main__':
    # Arguments:
    PARGS = parse_args()
    if PARGS.email:
        EMAIL_LIST = PARGS.email.split(',')
    else:
        EMAIL_LIST = list()
    API_KEY = PARGS.key
    if not API_KEY and not PARGS.settings and API_KEY_DAX:
        API_KEY = API_KEY_DAX
    if PARGS.settings:
        SETTINGS_PATHS = PARGS.settings.split(',')
    else:
        SETTINGS_PATHS = list()
    #hour right now:
    NOW_HOUR = datetime.now().hour

    #Logger for logs
    if PARGS.debug and not PARGS.status:
        LOGGER = dax.log.setup_debug_logger('manager', PARGS.logfile)
    else:
        LOGGER = dax.log.setup_info_logger('manager', PARGS.logfile)

    DAX_LOGS_DIR = PARGS.daxlogsdir
    DAX_LOGS_DIRS_DICT = dict()
    EMAIL_DICT = dict()
    LOGGER.info('Time at the beginning of the DAX Manager: '+ str(datetime.now()))
    LOGGER.info('Current Process ID: '+str(os.getpid()))
    LOGGER.info('Current Process Name: '+os.path.basename(__file__))
    LOGGER.info('Current User: '+USER)
    if GATEWAY:
        LOGGER.info('Current Gateway/Computer: '+GATEWAY+'\n')
    else:
        LOGGER.debug('Current Gateway/Computer: not found...\n')

    #If api_key given, use the redcap db:
    if API_KEY:
        SETTINGS_PATHS, DAX_LOGS_DIRS_DICT, EMAIL_DICT = generate_settings_REDCap(GATEWAY,
                                                                                  USER,
                                                                                  API_KEY,
                                                                                  PARGS.status)

    if PARGS.writeonly or PARGS.status:
        pass
    else:
        if PARGS.debug:
            LOGGER.debug('Settings file found: ')
            for filepath in SETTINGS_PATHS:
                LOGGER.debug('  -'+os.path.basename(filepath))
            LOGGER.debug('\n')

        # Workers:
        WORKERS = []
        #When to run dax_build / dax_update_tasks / dax_launch
        # dax_build -> 6am / 8pm
        # dax_update_tasks -> every odd hours (1-3-5...)
        # dax_launch -> every hour
        #Submit executables
        LOGGER.info('dax_build will run at 6am and 8pm...')
        LOGGER.info('dax_update_tasks will run on odd hours...')
        LOGGER.info('dax_launcher will run every hour...')
        LOGGER.info('Submitting dax executables for each settings file...')
        for filepath in SETTINGS_PATHS:
            if filepath in DAX_LOGS_DIRS_DICT.keys():
                DAX_LOGS_DIR = DAX_LOGS_DIRS_DICT[filepath]
            if filepath in EMAIL_DICT.keys():
                EMAIL_LIST = EMAIL_DICT[filepath]

            # start processes
            arguments = (filepath, DAX_LOGS_DIR, PARGS.debug, EMAIL_LIST)
            if NOW_HOUR == 6 or NOW_HOUR == 20: #6am-8pm
                proc = submit_exec(run_dax_build, arguments, filepath, 'dax_build')
                WORKERS.append(proc)

            if NOW_HOUR%2 == 1: # odd hours
                proc = submit_exec(run_dax_update_tasks, arguments, filepath, 'dax_update_tasks')
                WORKERS.append(proc)

            # Run every time that dax_manager run
            proc = submit_exec(run_dax_launch, arguments, filepath, 'dax_launch')
            WORKERS.append(proc)

        #If ctrl-c, kill all children
        try:
            for worker in WORKERS:
                worker.join()
        except KeyboardInterrupt:
            LOGGER.warn('Dax_manager received ctrc-c/kill. All workers are going to be terminated.')
            worker.terminate()
            worker.join()

    LOGGER.info('Time at the end of the DAX Manager: '+ str(datetime.now()))
