#!python

'''
Usage:
    beekeeper [-d] --config <config_file> --target <api_target> [--r-params=<n:v>,...] [--t-params=<n:v>,...] [--macro-args=<n:v>,...]
    beekeeper [-d] -b --config <config_file> --target <api_target> --outfile <filename> [--r-params=<n:v>,...] [--t-params=<n:v>,...] [--macro-args=<n:v>,...]
    beekeeper --config <config_file> --list

Options:
    -d --debug      execute in debug mode (dump headers and template values)
    -b --bytes      return request data as bytes (as opposed to string)    
'''

#
#  beekeeper: command-line utility for interacting with HTTP API's
#

import os, sys
import re
import json
from collections import namedtuple
import requests
import docopt
from snap import snap, common
import yaml
from mercury.utils import split_unescape

APICallStatus = namedtuple('APICallStatus', 'ok url status_code data')

ACCEPTED_METHODS = [
    'POST',
    'GET',
    'PUT',
    'PATCH'
]

MACRO_RX = re.compile(r'^~macro\[.+\]$')
MACRO_PREFIX = '~macro['
MACRO_SUFFIX = ']'

TEMPLATE_RX = re.compile(r'^~template\[.+\]$')
TEMPLATE_PREFIX = '~template['
TEMPLATE_SUFFIX = ']'

TEMPLATE_ENV_VAR_RX = re.compile(r'\$\(.+?\)')
TEMPLATE_ENV_VAR_PREFIX = '$('
TEMPLATE_ENV_VAR_SUFFIX = ')'


process_output_as_bytes = False

class MacroDefinition(object):
    def __init__(self, name, *arg_names):
        self.name = name
        self.arg_names = arg_names


class UnregisteredMacroError(Exception):
    def __init__(self, macro_name):
        super().__init__(f'An unregistered macro "{macro_name}" is being invoked in an API target. Please check your configuration.')


class MissingMacroArgError(Exception):
    def __init__(self, argname, macro_definition):
        super().__init__(f'''
            The macro "{macro_definition.name}" is being invoked, but the argument "{argname}" is missing.
            Required arguments are: {macro_definition.arg_names}.''')


debug_mode = False


class ParameterSetTable(object):
    def __init__(self):
        self.parameter_sets = {}

    def add(self, name, **kwargs):
        self.parameter_sets[name] = kwargs

    def get_parameter_set(self, name):
        if not name in self.parameter_sets.keys():
            raise Exception(f'no parameter set "{name}" found.')
        
        return self.parameter_sets[name]


class APITarget(object):
    def __init__(self, target_url, http_method, headers, processor=None, **kwargs):
        self.base_url = target_url
        self.processor = processor
        self.method = http_method.upper()
        if self.method not in ACCEPTED_METHODS:
            raise Exception(f'HTTP method "{self.method}" is not supported.')

        self.headers = headers
        self.parameters = kwargs
        self.requests_lib_keyword_args = dict()
        self.file_refs = []


    def add_request_setting(self, name, value):
        self.requests_lib_keyword_args[name] = value


    def add_file_ref(self, name, file_object):
        self.file_refs.append((name, file_object))


    def call(self, **kwargs):
        
        raw_params = {}
        raw_params.update(self.parameters)
        raw_params.update(kwargs)

        if debug_mode:
            print('### raw call parameters: %s' % raw_params, file=sys.stderr)

        # Because we are SOMETIMES required to pass JSON to an endpoint, and because
        # macros and templates ALWAYS return strings, here we "normalize" all incoming
        # call parameters to proper JSON. This way, even if we are passing those parameters
        # as JSON, the request will be formatted correctly (and our ability to pass ordinary 
        # name-value pairs is unaffected).
        #
        # If we didn't do this, we could encounter a scenario where a macro which returns
        # 
        # json.dumps({'name': 'value', ...}) 
        # 
        # results in the quoted dictionary 
        # 
        # "{'name': 'value', ...}"
        #
        # being sent to an endpoint instead of actual JSON  -- causing a nonintuitive 
        # server-side error. (Ask me how I know.)
        #
        call_params = dict()
        request_kwargs = self.requests_lib_keyword_args

        for key, value in raw_params.items():
            if debug_mode:
                print(f'### Normalizing the value of request param "{key}" (value is {value})...', file=sys.stderr)
            try:
                call_params[key] = json.loads(value)
            except:
                call_params[key] = value
        
        if debug_mode:
            print('### Call parameters: %s' % call_params, file=sys.stderr)
            print(f'### File refs:', file=sys.stderr)
            print("\n".join(self.file_refs), file=sys.stderr)

        if self.method == 'GET':
            response = requests.get(self.base_url, headers=self.headers, params=call_params, **request_kwargs)
            
        elif self.method == 'POST':

            if self.headers.get('Accept') == 'application/json':
                response = requests.post(self.base_url, headers=self.headers, json=call_params, files=self.file_refs, **request_kwargs)

            elif self.headers.get('Content-Type') == 'application/json':
                response = requests.post(self.base_url, headers=self.headers, json=call_params, files=self.file_refs, **request_kwargs)

            else:                
                response = requests.post(self.base_url,
                                         data=call_params,
                                         headers=self.headers,                                         
                                         files=self.file_refs,
                                         **request_kwargs)

        if debug_mode:
            print(f'#--- Response headers from target (URL {self.base_url}):', file=sys.stderr)  
            print(common.jsonpretty(dict(response.headers)), file=sys.stderr)            

        if response.status_code == requests.codes.ok:
            if process_output_as_bytes:
                return APICallStatus(ok=True,
                                     url=response.url,
                                     status_code=response.status_code,
                                     data=response.content) # the "content" property gives us back a byte array, not a string
            else:
                return APICallStatus(ok=True,
                                     url=response.url,
                                     status_code=response.status_code,
                                     data=response.text)
        else:
            return APICallStatus(ok=False,
                                 url=response.url,
                                 status_code=response.status_code,
                                 data=response.text)


def read_target_params(target_yaml_config: dict, global_yaml_config: dict, macro_defs: dict, macro_args: dict, **kwargs):
    # kwargs should contain any name value pairs passed to us on the command line as --r-params
    # (that is, HTTP request params). Any command-line params will override parameter values
    # specified in the config file.
    #
    params = {}
    config_params = target_yaml_config.get('request_params')

    if not config_params:
        return params

    for p in config_params:
        raw_value = p['value']

        macro_rx_match = MACRO_RX.match(str(raw_value))
        template_rx_match = TEMPLATE_RX.match(str(raw_value))

        if macro_rx_match:
            macro_function_name = str(raw_value).lstrip(MACRO_PREFIX).rstrip(MACRO_SUFFIX)

            # is this macro registered?
            macro_def = macro_defs.get(macro_function_name)
            if not macro_def:
                raise UnregisteredMacroError(macro_function_name)

            # if the macro has been registered, did we receive the required params?            
            for argname in macro_def.arg_names:
                if macro_args.get(argname) is None:
                    raise MissingMacroArgError(argname, macro_def)

            value = eval_macro(macro_function_name, global_yaml_config, **macro_args)
            
        else:
            value = common.load_config_var(raw_value)
        params[p['name']] = value

    params.update(kwargs)
    return params


def read_target_files(target_yaml_config: dict, macro_defs: dict, macro_args: dict, **kwargs)->list:
    #
    # This allows us to refer directly to files so that we can use the Requests library
    # to pass local file objects (intended as email attachments) to, say, the Mailgun API.
    #

    output_tuples = []
    file_refs = target_yaml_config.get('files')

    if not file_refs:
        return output_tuples

    for ref in file_refs:
        ref_name = ref['name']
        ref_file = open(ref['path'], 'rb')
        output_tuples.append((ref_name, ref_file))

    return output_tuples


def eval_macro(macro_funcname, yaml_config, **kwargs):

    macro_module = yaml_config['globals'].get('macro_module')
    if not macro_module:
        raise Exception('you have defined a macro, but there is no macro_module in the globals section. Please check your config file.')

    macro_func = None
    try:
        macro_func = common.load_class(macro_funcname, macro_module)        
    except AttributeError:
        raise Exception(f'The code macro "{macro_funcname}" was not found in the specified macro_module {macro_module}.')

    try:
        return macro_func(**kwargs)
    except KeyError as err:
        # The reason I am wording the error message in this way is that since beekeeper macro functions are user-defined, I can't be sure that
        # a KeyError is being thrown specifically as a result of a missing macro-arg; that is merely the most 
        # likely explanation. --DT
        raise Exception(f'The macro function "{macro_funcname}" may be referencing a missing macro-arg value {str(err)}. Are you passing it?')



def resolve_env_vars_in_template(template_string):
    # now find refs to environment variables. WITHIN a ~template[] block, they
    # are formatted like this:
    #
    # ${ENV_VAR}
    #
    result = template_string
    for match_obj in re.finditer(TEMPLATE_ENV_VAR_RX, template_string):
        str_match = template_string[match_obj.start():match_obj.end()]
        var_name = str_match.lstrip(TEMPLATE_ENV_VAR_PREFIX).rstrip(TEMPLATE_ENV_VAR_SUFFIX)
        var_value = os.getenv(var_name)
        if not var_value:
            raise Exception(f'The environment variable {var_name} was referenced in a template expression, but has not been set.')

        result = result.replace(str_match, var_value)

    return result


def load_config_dictionary(config_dict:dict, yaml_config:dict, macro_defs:dict, macro_args:dict, **kwargs):
    # kwargs will contain any name-value pairs passed to us as template params on the command line
    #

    data = {}

    for key, value in config_dict.items():

        macro_rx_match = MACRO_RX.match(str(value))
        template_rx_match = TEMPLATE_RX.match(str(value))

        # if we have a macro as the value, load and execute it
        if macro_rx_match:
            macro_function_name = value.split(MACRO_PREFIX)[1].rstrip(MACRO_SUFFIX)
            #
            # this was yielding odd behavior:
            #
            # str(value).lstrip(MACRO_PREFIX).rstrip(MACRO_SUFFIX)
            #
            # It was stripping off one character too many -- intermittently.
            # The problem was an odd behavior in lstrip(). It would strip
            # an additional letter from the string if that letter matched
            # one of the letters in the MACRO_PREFIX string -- so instead of
            # stripping off "~macro[", it would strip off "~macro[c" if the
            # target string began with "c".
            # Keeping this note because it might be a Python bug.
            # --DT

            # is this macro registered?
            macro_def = macro_defs.get(macro_function_name)
            if not macro_def:
                raise UnregisteredMacroError(macro_function_name)

            # if the macro has been registered, did we receive the required params?            
            for argname in macro_def.arg_names:
                if macro_args.get(argname) is None:
                    raise MissingMacroArgError(argname, macro_def)

            data[key] = eval_macro(macro_function_name, yaml_config, **macro_args)

        # if we have a template as the value, populate it
        elif template_rx_match:
            template_str = str(value).lstrip(TEMPLATE_PREFIX).rstrip(TEMPLATE_SUFFIX)
            new_value = resolve_env_vars_in_template(template_str)            
            data[key] = new_value

        # if the value is not a macro or a template, see if it's an env var
        else:
            data[key] = common.load_config_var(value)

    
    # if the user has passed us any template variables on the command line, add those to the mix
    # (override any variables already specified)
    data.update(kwargs)
    return data


def load_api_target(target_name:str, yaml_config:dict, macro_defs:dict, macro_args:dict, paramsets: ParameterSetTable):
    # kwargs should contain any name-value pairs passed to us on the command line as --tparams (template parameters)
    #

    cli_request_parameters = paramsets.get_parameter_set('cli_request_parameters')
    cli_template_parameters = paramsets.get_parameter_set('cli_template_parameters')

    if target_name not in yaml_config['targets'].keys():
        raise Exception(f'No API target registered under the name {target_name}.')

    target_config = yaml_config['targets'][target_name]

    processor_func = None
    processor_func_name = target_config.get('processor')

    if processor_func_name:
        processor_module = yaml_config['globals'].get('processor_module')
        try:
            processor_func = common.load_class(processor_func_name, processor_module)
        except AttributeError:
            raise Exception(f'The processor function "{processor_func_name}" was not found in the specified processor_module {processor_module}.')

    raw_headers = target_config.get('headers', {})
    headers = load_config_dictionary(raw_headers, yaml_config, macro_defs, macro_args, **cli_template_parameters)
    
    # retrieve any default template values supplied in the configuration file
    #
    raw_template_params = target_config.get('template_params', {})
    template_params = load_config_dictionary(raw_template_params, yaml_config, macro_defs, macro_args, **cli_template_parameters)
    raw_url = target_config['url']
    url = None

    try:
        url = raw_url.format(**template_params)        
    except KeyError as err:
        raise Exception('The specified URL %s contains a template variable %s, but no data was found to populate it. Please check your config.'
                            % (raw_url, str(err)))

    try:
        http_request_params = read_target_params(target_config, yaml_config, macro_defs, macro_args, **cli_request_parameters)
        target = APITarget(url,
                        target_config['method'],
                        headers,
                        processor_func,
                        **http_request_params)

        #
        # read any name-value pairs specified in the (optional) "settings" field of the target config.
        # These n-v pairs will be passed through to our underlying call to the requests HTTP library as keyword args.
        # For example: if you wished to turn off SSL verification on a call to an endpoint, you could do so by specifying
        #
        # settings:
        #    verify: False
        #
        # and in our callout to the requests lib, we would pass 
        #
        # requests.get(<url>, <other_params>, verify=False)
        #
        #
        request_lib_settings = target_config.get('settings', {})
        for name, value in request_lib_settings.items():
            target.add_request_setting(name, value)
                        
        filerefs = read_target_files(target_config, macro_defs, macro_args)
        for ref in filerefs:
            target.add_file_ref(ref[0], ref[1])
        
        return target

    except Exception as err:
        raise Exception(f'An exception of type {err.__class__.__name__} was thrown while resolving beekeeper config target "{target_name}": \n{str(err)}')


def load_macro_defs(yaml_config):

    mdefs = {}
    macro_config_section = yaml_config.get('macros') or {}
    for name, mdef in macro_config_section.items():
        macro_arg_names = mdef.get('arg_names') or []
        
        mdefs[name] = MacroDefinition(name, *macro_arg_names)
    
    return mdefs


def read_cli_request_params(args) -> dict:
    '''Read name/value pairs passed as the value of --r-params. These will be passed to the target
    as HTTP request parameters. If the same parameters are specified in the YAML config file,
    the values passed on the command line will override them.
    '''

    cli_request_parameters = {}
    if args.get('--r-params'):
        raw_params = args['--r-params'][0]
        
        nvpairs = split_unescape(raw_params,',')

        for nvpair in nvpairs:
            tokens = split_unescape(nvpair,':')
            name = tokens[0]
            value = tokens[1]
            cli_request_parameters[name] = value
    
    return cli_request_parameters


def read_cli_template_params(args) -> dict:
    '''Read name/value pairs passed as the value of --t-params. These will override
    any template parameters specified in the YAML config file for the active target.
    '''

    cli_template_parameters = {}
    if args.get('--t-params'):
        raw_params = args['--t-params'][0]
        
        nvpairs = split_unescape(raw_params, ',')

        for nvpair in nvpairs:
            tokens = split_unescape(nvpair,':')
            name = tokens[0]
            value = tokens[1]
            cli_template_parameters[name] = value
    
    return cli_template_parameters


def read_macro_args(args):
    macro_args = {}
    if args.get('--macro-args'):
        raw_args = args['--macro-args'][0]
        nvpairs = raw_args.split(',')

        for nvpair in nvpairs:
            tokens = split_unescape(nvpair,':')
            name = tokens[0]
            value = tokens[1]
            macro_args[name] = value
    
    return macro_args


def main(args):

    # we may run in a mode that simply lists API targets
    list_mode = False
    if args['--list']:
        list_mode = True

    global debug_mode
    debug_mode = args['--debug'] # this is a boolean

    configfile_name = args['<config_file>']
    yaml_config = common.read_config_file(configfile_name)
    project_home = common.load_config_var(yaml_config['globals']['project_home'])

    # add the project home to our PYTHONPATH
    sys.path.append(os.path.join(os.getcwd(), project_home))

    configured_targets = yaml_config['targets']
    if list_mode:
        print(common.jsonpretty([t for t in configured_targets]))
        return

    target_name = args['<api_target>']
    if not configured_targets.get(target_name):
        raise Exception(f'No API target registered under the name "{target_name}". Please check your config.')

    # if we know the specified target is valid, go ahead and spin up service wrappers
    service_registry = common.ServiceObjectRegistry(snap.initialize_services(yaml_config))

    # users can specify additional request parameters on the command line.
    # Parameters so specified, if their names collide with the params in the config file,
    # will override the ones in the config.
    #
    cli_request_params = read_cli_request_params(args)

    # users can specify additional TEMPLATE parameters on the command line.
    # As with HTTP request parameters, if their names collide with the template params 
    # specified in the config file, these will take precedence.
    #
    cli_template_params = read_cli_template_params(args)
    
    # users can also specify macros for supplying data at runtime.
    # each macro invoked in the context of an api target must have a corresponding
    # macro definition in the [macros] section of the YAML config.
    macro_args = read_macro_args(args)
    macro_defs = load_macro_defs(yaml_config)

    parameter_sets = ParameterSetTable()

    parameter_sets.add('cli_request_parameters', **cli_request_params)
    parameter_sets.add('cli_template_parameters', **cli_template_params)
    

    if debug_mode:
        print('#--- Macros defined in config:', file=sys.stderr)
        for _, macro_def in macro_defs.items():
            print(f'{macro_def.name} : {macro_def.arg_names}', file=sys.stderr)

        print(f'#---Macro arguments:', file=sys.stderr)
        print(common.jsonpretty(macro_args), file=sys.stderr)

        print(f'#--- URL template values for API target {target_name}:', file=sys.stderr)
        print(common.jsonpretty(yaml_config['targets'][target_name].get('template_values', {})), file=sys.stderr)

        print(f'#--- CLI template parameters:', file=sys.stderr)
        print(common.jsonpretty(cli_template_params), file=sys.stderr)

        print(f'#--- CLI HTTP request parameters:', file=sys.stderr)
        print(common.jsonpretty(cli_request_params), file=sys.stderr)

    
    # TODO: find a less subtle way of passing both the user-supplied request params AND user_supplied template params
    #
    api_target = load_api_target(target_name, yaml_config, macro_defs, macro_args, parameter_sets)    


    if debug_mode:
        print(f'#---API target {target_name} loaded.', file=sys.stderr)

        print(f'#--- Requests library callthrough parameters:', file=sys.stderr)
        print(common.jsonpretty(api_target.requests_lib_keyword_args), file=sys.stderr)

        print(f'#--- Request headers for API target {target_name}:', file=sys.stderr)
        print(common.jsonpretty(api_target.headers), file=sys.stderr)


    global process_output_as_bytes
    process_output_as_bytes = args['--bytes']

    call_status = api_target.call(**cli_request_params)

    if call_status.ok:

        if api_target.processor:            
            print(api_target.processor.process(call_status, service_registry))

        elif process_output_as_bytes:
            output_dict = call_status._asdict()
            decoded_bindata = output_dict['data'].decode('utf-8', 'replace')
            output_dict.update({'data': decoded_bindata})

            print(json.dumps(output_dict))

            # write the binary output data to the specified file
            output_binfile = args['<filename>']
            with open(output_binfile, 'wb+') as f:
                f.write(call_status.data)
            
        else:
            print(json.dumps(call_status._asdict()))       
    else:
        print(json.dumps(call_status._asdict()))
        

if __name__ == '__main__':
    args = docopt.docopt(__doc__)
    main(args)


