#!/usr/bin/python
"""Pythonic command-line interface to mandoc-manpage converter/parser that will make you smile.
 * based on:
 * http://docopt.org
 * Repository and issue-tracker: https://github.com/docopt/docopt
 * Licensed under terms of MIT license (see LICENSE-MIT)
 * Copyright (c) 2013 Vladimir Keleshev, vladimir@keleshev.com
 * Copyright (c) 2014 Tobias Glaesser

"""
import os
import sys
import re
import datetime
import subprocess
import tempfile
import pprint
import gzip

from docopt import docopt

__all__ = ['cli2man']
__version__ = '0.2.3'


class Cli2ManLanguageError(Exception):

    """Error in construction of usage-message by developer."""


class Cli2ManExit(SystemExit):

    """Exit in case user invoked program with incorrect arguments."""

    usage = ''

    def __init__(self, message=''):
        SystemExit.__init__(self, (message + '\n' + self.usage).strip())


class Pattern(object):

    def __eq__(self, other):
        return repr(self) == repr(other)

    def __hash__(self):
        return hash(repr(self))

    def fix(self):
        self.fix_identities()
        self.fix_repeating_arguments()
        return self

    def fix_identities(self, uniq=None):
        """Make pattern-tree tips point to same object if they are equal."""
        if not hasattr(self, 'children'):
            return self
        uniq = list(set(self.flat())) if uniq is None else uniq
        for i, child in enumerate(self.children):
            if not hasattr(child, 'children'):
                assert child in uniq
                self.children[i] = uniq[uniq.index(child)]
            else:
                child.fix_identities(uniq)

    def fix_repeating_arguments(self):
        """Fix elements that should accumulate/increment values."""
        either = [list(child.children) for child in transform(self).children]
        for case in either:
            for e in [child for child in case if case.count(child) > 1]:
                if type(e) is Argument or type(e) is Option and e.argcount:
                    if e.value is None:
                        e.value = []
                    elif type(e.value) is not list:
                        e.value = e.value.split()
                if type(e) is Command or type(e) is Option and e.argcount == 0:
                    e.value = 0
        return self


def transform(pattern):
    """Expand pattern into an (almost) equivalent one, but with single Either.

    Example: ((-a | -b) (-c | -d)) => (-a -c | -a -d | -b -c | -b -d)
    Quirks: [-a] => (-a), (-a...) => (-a -a)

    """
    result = []
    groups = [[pattern]]
    while groups:
        children = groups.pop(0)
        parents = [Required, Optional, OptionsShortcut, Either, OneOrMore]
        if any(t in map(type, children) for t in parents):
            child = [c for c in children if type(c) in parents][0]
            children.remove(child)
            if type(child) is Either:
                for c in child.children:
                    groups.append([c] + children)
            elif type(child) is OneOrMore:
                groups.append(child.children * 2 + children)
            else:
                groups.append(child.children + children)
        else:
            result.append(children)
    return Either(*[Required(*e) for e in result])


class LeafPattern(Pattern):

    """Leaf/terminal node of a pattern tree."""

    def __init__(self, name, value=None):
        self.name, self.value = name, value

    def __repr__(self):
        return '%s(%r, %r)' % (self.__class__.__name__, self.name, self.value)

    def flat(self, *types):
        return [self] if not types or type(self) in types else []

    def match(self, left, collected=None):
        collected = [] if collected is None else collected
        pos, match = self.single_match(left)
        if match is None:
            return False, left, collected
        left_ = left[:pos] + left[pos + 1:]
        same_name = [a for a in collected if a.name == self.name]
        if type(self.value) in (int, list):
            if type(self.value) is int:
                increment = 1
            else:
                increment = ([match.value] if type(match.value) is str
                             else match.value)
            if not same_name:
                match.value = increment
                return True, left_, collected + [match]
            same_name[0].value += increment
            return True, left_, collected
        return True, left_, collected + [match]


class BranchPattern(Pattern):

    """Branch/inner node of a pattern tree."""

    def __init__(self, *children):
        self.children = list(children)

    def __repr__(self):
        return '%s(%s)' % (self.__class__.__name__,
                           ', '.join(repr(a) for a in self.children))

    def flat(self, *types):
        if type(self) in types:
            return [self]
        return sum([child.flat(*types) for child in self.children], [])


class Argument(LeafPattern):

    def single_match(self, left):
        for n, pattern in enumerate(left):
            if type(pattern) is Argument:
                return n, Argument(self.name, pattern.value)
        return None, None

    @classmethod
    def parse(class_, source):
        name = re.findall('(<\S*?>)', source)[0]
        value = re.findall('\[default: (.*)\]', source, flags=re.I)
        return class_(name, value[0] if value else None)


class Command(Argument):

    def __init__(self, name, value=False):
        self.name, self.value = name, value

    def single_match(self, left):
        for n, pattern in enumerate(left):
            if type(pattern) is Argument:
                if pattern.value == self.name:
                    return n, Command(self.name, True)
                else:
                    break
        return None, None


class Option(LeafPattern):

    def __init__(self, short=None, long=None, argcount=0, value=False):
        assert argcount in (0, 1)
        self.short, self.long, self.argcount = short, long, argcount
        self.value = None if value is False and argcount else value

    @classmethod
    def parse(class_, option_description):
        short, long, argcount, value = None, None, 0, False
        options, _, description = option_description.strip().partition('  ')
        options = options.replace(',', ' ').replace('=', ' ')
        global manual

        for s in options.split():
            if s.startswith('--'):
                long = s
            elif s.startswith('-'):
                short = s
            else:
                argcount = 1
        if argcount:
            matched = re.findall('\[default: (.*)\]', description, flags=re.I)
            value = matched[0] if matched else None

        #mdoc
        manual.format.add_option(options,argcount,short,long,description)

        return class_(short, long, argcount, value)

    def single_match(self, left):
        for n, pattern in enumerate(left):
            if self.name == pattern.name:
                return n, pattern
        return None, None

    @property
    def name(self):
        return self.long or self.short

    def __repr__(self):
        return 'Option(%r, %r, %r, %r)' % (self.short, self.long,
                                           self.argcount, self.value)


class Required(BranchPattern):

    def match(self, left, collected=None):
        collected = [] if collected is None else collected
        l = left
        c = collected
        for pattern in self.children:
            matched, l, c = pattern.match(l, c)
            if not matched:
                return False, left, collected
        return True, l, c


class Optional(BranchPattern):

    def match(self, left, collected=None):
        collected = [] if collected is None else collected
        for pattern in self.children:
            m, left, collected = pattern.match(left, collected)
        return True, left, collected


class OptionsShortcut(Optional):

    """Marker/placeholder for [options] shortcut."""


class OneOrMore(BranchPattern):

    def match(self, left, collected=None):
        assert len(self.children) == 1
        collected = [] if collected is None else collected
        l = left
        c = collected
        l_ = None
        matched = True
        times = 0
        while matched:
            # could it be that something didn't match but changed l or c?
            matched, l, c = self.children[0].match(l, c)
            times += 1 if matched else 0
            if l_ == l:
                break
            l_ = l
        if times >= 1:
            return True, l, c
        return False, left, collected


class Either(BranchPattern):

    def match(self, left, collected=None):
        collected = [] if collected is None else collected
        outcomes = []
        for pattern in self.children:
            matched, _, _ = outcome = pattern.match(left, collected)
            if matched:
                outcomes.append(outcome)
        if outcomes:
            return min(outcomes, key=lambda outcome: len(outcome[1]))
        return False, left, collected


class Tokens(list):

    def __init__(self, source, error=Cli2ManExit):
        self += source.split() if hasattr(source, 'split') else source
        self.error = error

    @staticmethod
    def from_pattern(source):
        source = re.sub(r'([\[\]\(\)\|]|\.\.\.)', r' \1 ', source)
        source = [s for s in re.split('\s+|(\S*<.*?>)', source) if s]
        return Tokens(source, error=Cli2ManLanguageError)

    def move(self):
        return self.pop(0) if len(self) else None

    def current(self):
        return self[0] if len(self) else None


def parse_long(tokens, options):
    """long ::= '--' chars [ ( ' ' | '=' ) chars ] ;"""
    long, eq, value = tokens.move().partition('=')
    assert long.startswith('--')
    value = None if eq == value == '' else value
    similar = [o for o in options if o.long == long]
    if tokens.error is Cli2ManExit and similar == []:  # if no exact match
        similar = [o for o in options if o.long and o.long.startswith(long)]
    if len(similar) > 1:  # might be simply specified ambiguously 2+ times?
        raise tokens.error('%s is not a unique prefix: %s?' %
                           (long, ', '.join(o.long for o in similar)))
    elif len(similar) < 1:
        argcount = 1 if eq == '=' else 0
        o = Option(None, long, argcount)
        options.append(o)
        if tokens.error is Cli2ManExit:
            o = Option(None, long, argcount, value if argcount else True)
    else:
        o = Option(similar[0].short, similar[0].long,
                   similar[0].argcount, similar[0].value)
        if o.argcount == 0:
            if value is not None:
                raise tokens.error('%s must not have an argument' % o.long)
        else:
            if value is None:
                if tokens.current() in [None, '--']:
                    raise tokens.error('%s requires argument' % o.long)
                value = tokens.move()
        if tokens.error is Cli2ManExit:
            o.value = value if value is not None else True
    return [o]


def parse_shorts(tokens, options):
    """shorts ::= '-' ( chars )* [ [ ' ' ] chars ] ;"""
    token = tokens.move()
    assert token.startswith('-') and not token.startswith('--')
    left = token.lstrip('-')
    parsed = []
    while left != '':
        short, left = '-' + left[0], left[1:]
        similar = [o for o in options if o.short == short]
        if len(similar) > 1:
            raise tokens.error('%s is specified ambiguously %d times' %
                               (short, len(similar)))
        elif len(similar) < 1:
            o = Option(short, None, 0)
            options.append(o)
            if tokens.error is Cli2ManExit:
                o = Option(short, None, 0, True)
        else:  # why copying is necessary here?
            o = Option(short, similar[0].long,
                       similar[0].argcount, similar[0].value)
            value = None
            if o.argcount != 0:
                if left == '':
                    if tokens.current() in [None, '--']:
                        raise tokens.error('%s requires argument' % short)
                    value = tokens.move()
                else:
                    value = left
                    left = ''
            if tokens.error is Cli2ManExit:
                o.value = value if value is not None else True
        parsed.append(o)
    return parsed


def parse_pattern(source, options):
    tokens = Tokens.from_pattern(source)
    result = parse_expr(tokens, options)

    if tokens.current() is not None:
        raise tokens.error('unexpected ending: %r' % ' '.join(tokens))

    return Required(*result)


def parse_expr(tokens, options):
    """expr ::= seq ( '|' seq )* ;"""
    seq = parse_seq(tokens, options)
    if tokens.current() != '|':
        return seq
    result = [Required(*seq)] if len(seq) > 1 else seq
    while tokens.current() == '|':
        tokens.move()
        seq = parse_seq(tokens, options)
        result += [Required(*seq)] if len(seq) > 1 else seq
    return [Either(*result)] if len(result) > 1 else result


def parse_seq(tokens, options):
    """seq ::= ( atom [ '...' ] )* ;"""
    result = []
    while tokens.current() not in [None, ']', ')', '|']:
        atom = parse_atom(tokens, options)
        if tokens.current() == '...':
            atom = [OneOrMore(*atom)]
            tokens.move()
        result += atom
    return result


def parse_atom(tokens, options):
    """atom ::= '(' expr ')' | '[' expr ']' | 'options'
             | long | shorts | argument | command ;
    """
    token = tokens.current()
    result = []
    if token in '([':
        tokens.move()
        matching, pattern = {'(': [')', Required], '[': [']', Optional]}[token]
        result = pattern(*parse_expr(tokens, options))
        if tokens.move() != matching:
            raise tokens.error("unmatched '%s'" % token)
        return [result]
    elif token == 'options':
        tokens.move()
        return [OptionsShortcut()]
    elif token.startswith('--') and token != '--':
        return parse_long(tokens, options)
    elif token.startswith('-') and token not in ('-', '--'):
        return parse_shorts(tokens, options)
    elif token.startswith('<') and token.endswith('>') or token.isupper():
        return [Argument(tokens.move())]
    else:
        return [Command(tokens.move())]


def parse_argv(tokens, options, options_first=False):
    """Parse command-line argument vector.

    If options_first:
        argv ::= [ long | shorts ]* [ argument ]* [ '--' [ argument ]* ] ;
    else:
        argv ::= [ long | shorts | argument ]* [ '--' [ argument ]* ] ;

    """
    parsed = []
    while tokens.current() is not None:
        if tokens.current() == '--':
            return parsed + [Argument(None, v) for v in tokens]
        elif tokens.current().startswith('--'):
            parsed += parse_long(tokens, options)
        elif tokens.current().startswith('-') and tokens.current() != '-':
            parsed += parse_shorts(tokens, options)
        elif options_first:
            return parsed + [Argument(None, v) for v in tokens]
        else:
            parsed.append(Argument(None, tokens.move()))
    return parsed

section_non_option_additional = ''

def parse_defaults(doc):
    global section_non_option_additional;
    global manual;

    defaults = []
    for s in parse_section('options:', doc):
        # FIXME corner case "bla: options: --foo"
        _, _, s = s.partition(':')  # get rid of "options:"
        split = re.split('^  (-\S+?)', '\n' + s,flags=re.MULTILINE)[1:]
        split = [s1 + s2 for s1, s2 in zip(split[::2], split[1::2])]
        #for p in split:
        #    print "['"+p+"']"

        #mdoc
        manual.format.begin_list()
        options = [Option.parse(s) for s in split if s.startswith('-')]
        defaults += options
        manual.format.end_list()
        manual.format.text(section_non_option_additional)
    return defaults

def my_print(toprint): #just for debugging
    print "--------------------"
    print toprint
    print "____________________"

def parse_section(name, source):
    global section_non_option_additional;
    global manual

    if not "usage" in name:
        manual.section_start(name.replace(':','').upper())
    pattern = re.compile('^([^\n]*' + name + '[^\n]*\n?(?:[ \t].*?(?:\n|$))*)',
                         re.IGNORECASE | re.MULTILINE)

    res = [s.strip() for s in pattern.findall(source)]

    try:
        line = res[0].split("\n")[-1]

        info_start = source.find(line) + len(line)
        section_non_option_additional = source[info_start:].strip()

        info_start = re.search("\n\n^[\w+| |\t]*:[\s]*$",section_non_option_additional,re.MULTILINE)
        if info_start != None:
            info_start = info_start.start()
            section_non_option_additional = section_non_option_additional[:info_start].strip()
        else:
            section_non_option_additional = ''
    except:
        section_non_option_additional = ''

    return res

def formal_usage(section):
    _, _, section = section.partition(':')  # drop "usage:"
    global manual
    pu = section.split()

    command = pu[0]
    manual.set_name(command)

    return '( ' + ' '.join(') | (' if s == pu[0] else s for s in pu[1:]) + ' )'


def extras(help, version, options, doc):
    if help and any((o.name in ('-h', '--help')) and o.value for o in options):
        print(doc.strip("\n"))
        sys.exit()
    if version and any(o.name == '--version' and o.value for o in options):
        print(version)
        sys.exit()


class Dict(dict):
    def __repr__(self):
        return '{%s}' % ',\n '.join('%r: %r' % i for i in sorted(self.items()))

class ManualFormatter():
    def __init__(self,manual):
        self.m = manual

    def begin_section(self,name):
        print "begin section stub"
    def begin_list(self):
        print "begin list stub"
    def end_list(self):
        print "end list stub"
    def date(self):
        print "date stub"
    def document(self):
        print "document stub"
    def os(self):
        print "os stub"
    def name(self):
        print "name stub"
    def description(self):
        print "description stub"
    def synopsis(self):
        print "synopsis stub"
    def text(self,text):
        if text:
            self.add_line(text.strip())
    def paragraph(self):
        print "paragraph stub"

    def add_line(self,code): #adds a linebreak at the end
        self.m.add_line(code)

    def add_code(self,code): #inserts code as is
        self.m.add_code(code)

    def newline(self): #inserts newline
        self.m.add_code("\n")

    def month(self):
        return self.m.now.strftime("%B")
    def day(self):
        return str(self.m.now.day)
    def year(self):
        return str(self.m.now.year)
    def command_name(self):
        return self.m.name_string
    def arguments(self):
        return self.m.arguments
    def options(self):
        return self.m.options

class MarkdownFormatter(ManualFormatter):
    def begin_section(self,name):
        self.add_line("# "+name)
        self.newline()

    def begin_list(self):
        return
        #self.add_line(".Bl -tag -width Ds")

    def end_list(self):
        return
        self.add_line(".El")

    def date(self):
        self.add_line("Creation Date: "+self.month()+" "+self.day()+", "+self.year())

    def document(self):
        self.add_line("HERE_GOES_NAME_UPPER HERE_GOES_SECTION HERE_GOES_ARCH")

    def os(self):
        self.add_line("Operating System - HERE_GOES_OS")

    def name(self):
        self.add_line("**HERE_GOES_NAME**")

    def description(self):
        self.add_line(" -- HERE_GOES_SHORT_DESCRIPTION")

    def synopsis(self):
        self.add_line("HERE_GOES_SYNOPSIS")

    def paragraph(self):
        self.add_line("\n\n")

    def long_flag(self,f):
        return self.flag(f)

    def flag(self,f):
        return "**"+f+"**"

    def short_flag(self,f):
        return self.flag(f)

    def argument(self,a):
        return "*"+a+"*"

    def blockquote(self,text):
        self.newline()
        self.add_line("> "+text)
        self.newline()

    def codeblock(self,code):
        self.add_line("```")
        for line in code:
            self.add_line("    "+line)
        self.add_line("```")
        self.newline()

    def add_option(self,options,argcount,short,long,description):
        if argcount > 0:
            a = options.split()[1]
        if short != None:
            s = ''
            s += self.short_flag(short)
            if argcount > 0:
                s += " "+self.argument(a)+" "
            if long != None:
                s += " **,** "+self.long_flag(long)
                if argcount > 0:
                    s += " "+self.argument(a)+" "
            self.add_line(s)
        elif long != None:
            if argcount > 0:
                self.add_line(self.long_flag(long)+" "+self.argument(a))
            else:
                self.add_line(self.long_flag(long))
        #print class_(short, long, argcount, value)
        #self.newline()
        self.blockquote(' '.join(description.strip().split()))
        #self.newline()

    def item(self,head,content):
        self.add_line(" - "+head)
        self.add_line("   "+content)

class MDocFormatter(ManualFormatter):
    def begin_section(self,name):
        self.add_line(".Sh "+name)

    def begin_list(self):
        self.add_line(".Bl -tag -width Ds")

    def end_list(self):
        self.add_line(".El")

    def date(self):
        self.add_code(".Dd "+self.month()+" "+self.day()+", "+self.year())

    def document(self):
        self.add_line(".Dt HERE_GOES_NAME_UPPER HERE_GOES_SECTIONHERE_GOES_ARCH") #avoid whitespace at end of input line warning

    def os(self):
        self.add_line(".OsHERE_GOES_OS") #avoid whitespace at end of input line warning

    def name(self):
        self.add_line(".Nm HERE_GOES_NAME")

    def description(self):
        self.add_line(".Nd HERE_GOES_SHORT_DESCRIPTION")

    def remove_empty_lines(self,string):
        l = []
        for line in string.split("\n"):
            if not re.match(r'^\s*$', line) and len(line)>0:
                l.append(line.rstrip())
        return '\n'.join(l)

    def replace_empty_lines(self,string,replace):
        l = []
        for line in string.split("\n"):
            if not re.match(r'^\s*$', line) and len(line)>0:
                l.append(line.rstrip())
            else:
                l.append(replace)
        return '\n'.join(l)

    def manpage(self,page,number="1"):
        if number == '':
            number = "1"
        self.add_line(".Xr "+page+" "+number)

    def see_also(self,pages):
        self.begin_section("SEE ALSO")
        i = 0
        for page in pages:
            if i>0:
                self.add_code(" , ")
            self.manpage(page,pages[page])
            i+=1

    def has_child(self,node):
        if hasattr(node,"children"):
            for child in node.children:
                return True
        return False

    def print_synopsis_tree(self,pattern,parent,l): #the main purpose of this function is to understand the tree and debug code
        c='----'
        for child in pattern.children:
            c=str().rjust((4*l+4),'-')
            t = type(child)
            print c+str(type(child))
            if hasattr(child,"children"):
                self.print_synopsis_tree(child,pattern,l+1)



    def iterate_synopsis_tree2(self,pattern,parent,l):
        s = ''
        i = 0
        grandparent_type = type(parent)
        parent_type = type(pattern)
        for child in pattern.children:
            child_type = type(child)
            if child_type is Command:
                if child == Command(self.m.name_string,False):
                    s+= "\n.Nm HERE_GOES_NAME\n"
                    continue
            if parent_type is Either and i>0:
                s += "| "

            if self.has_child(child):
                if child_type is Either:
                    s+= ".Pq "
                s += self.iterate_synopsis_tree2(child,pattern,l+1)
                if child_type:
                    if l>1:
                        if child_type is Either:
                            s+="\n"
            else:
                if child_type is Option:
                    if (parent_type is Optional and (parent_type is not Either)) or (parent_type is OneOrMore and grandparent_type is Optional):
                        s+="\n.Op "
                    elif parent_type is Required:
                        s+="\n."
                    if child.name.startswith('--'):
                        s+=self.long_flag(child.name)+" "
                    else:
                        s+=self.short_flag(child.name)+" "
                elif child_type is OptionsShortcut:
                    if parent_type is Optional:
                        s+="\n.Op"
                    s+= " Ar options"
                elif child_type is Argument:
                    if parent_type is not Either and parent_type is not Optional and parent_type is not OneOrMore:
                        s+="\n."
                    elif (parent_type == Optional) or (parent_type is OneOrMore and grandparent_type is Optional):
                        s+="\n.Op "
                    s+=self.argument(child.name)+" "
                elif child_type is Command:
                    if parent_type is not Optional:
                        s+="\n."
                    else:
                        s+="\n.Op "
                    s+=self.argument(child.name)+" "
                if parent_type is OneOrMore:
                    s+="Ar ... "
            i+=1
        return s

    def make_synopsis(self,pattern,synopsis):
        #print "tree:___"
        #self.print_synopsis_tree(pattern,None,0)
        synopsis = self.remove_empty_lines(self.iterate_synopsis_tree2(pattern,None,0))
        return synopsis

    def synopsis(self):
        self.add_line("HERE_GOES_SYNOPSIS")

    def paragraph(self):
        self.add_line(".Pp")

    def short_flag(self,f):
        return "Fl "+f.lstrip("-")

    def long_flag(self,f):
        return "Fl -"+f.lstrip("-")

    def argument(self,a):
        return "Ar "+a

    def text_raw(self,s):
        if not s:
            return ""
        #find arguments,flags,command name
        i=0
        for arg in self.arguments():
            s=re.sub(arg.replace('[','').replace(']','')+"[ ]{0,1}","\n.Ar "+arg+"\n",s)
            i+=1
        for opt in self.options():
            if opt:
                s2=s
                s=re.sub(opt+"[ |\n]","\n.Fl "+opt[1:].strip()+"\n",s)
                if s2 == s: #FIXME: this is bad regex... should use a capture group and match for / and , and . and probably more
                    s=re.sub(opt+"/","\n.Fl "+opt[1:].strip()+"\n/",s)
        if self.command_name():
            s=re.sub(self.command_name()+"[ ]{0,1}","\n.Cm "+self.command_name()+"\n",s)

        l2=[]
        for l in s.split('\n'):
            if l.startswith("'"):
                l=l[1:]
                l="\&'"+l
            l2.append(l)
        s='\n'.join(l2)
        s=self.replace_empty_lines(s,".Pp") #FIXME: NOT WELL TESTED
        return s

    def text(self,s):
        if not s:
            return
        s = self.text_raw(s)
        s=self.remove_empty_lines(s)
        self.add_line(s.strip())

    def add_option(self,options,argcount,short,long,description):
        if argcount > 0:
            a = options.split()[1]
            self.m.arguments[a]=''
        if short != None:
            s = ''
            s += ".It "+self.short_flag(short)
            if argcount > 0:
                s += " "+self.argument(a)
            if long != None:
                s += " , "+self.long_flag(long)
                if argcount > 0:
                    s += " "+self.argument(a)
            self.add_line(s)
        elif long != None:
            if argcount > 0:
                self.add_line(".It "+self.long_flag(long)+" "+self.argument(a))
            else:
                self.add_line(".It "+self.long_flag(long))

        self.m.options[short]=''
        self.m.options[long]=''
        self.text(' '.join(description.strip().split()))

    def item(self,head,content):
        self.add_line(".It "+head.strip())
        self.add_line(".sp 2")
        self.add_line(content.strip())
        self.add_line(".sp 1")

class Manual():
    def __init__(self,frmt):
        self.string = ''
        self.synopsis_string = ''
        self.name_string = ''
        self.section = "1"
        self.arch = ""
        self.os = ""
        self.actively_parsed_section = ''
        self.section_order = ["MDOC INIT","NAME","SYNOPSIS","DESCRIPTION","OPTIONS","RETURN VALUES","ENVIRONMENT","USAGE","FILES","EXIT STATUS","EXAMPLES","DIAGNOSTICS","ERRORS","SEE ALSO","STANDARDS","HISTORY","AUTHORS","CAVEATS","BUGS","SECURITY CONSIDERATIONS","COPYRIGHT"]
        self.sections = {}
        self.subsections = {}
        self.set_formatter(frmt)
        self.now = datetime.datetime.now()
        self.options = {}
        self.arguments = {}

    def set_formatter(self,frmt):
        if not frmt or frmt == "mdoc":
            self.format = MDocFormatter(self)
        elif frmt == "markdown":
            self.format = MarkdownFormatter(self)
        else:
            self.format = ManualFormatter(self)

    def begin(self):
        self.section_start("MDOC INIT",False)

        self.format.date()
        self.format.document()
        self.format.os()

        self.section_start("NAME")

        self.format.name()
        self.format.description()

        self.section_start("SYNOPSIS")

        self.format.synopsis()

        self.section_start("DESCRIPTION")

        self.format.text("HERE_GOES_DESCRIPTION")

    def section_start(self,section, NonVirtual=True):
        self.push_to_section()

        self.actively_parsed_section = section

        if NonVirtual:
            self.format.begin_section(section)

    def set_name(self,manual_str):
        self.name_string = manual_str

    def set_description(self,manual_str):
        self.desc_string = manual_str

    def merge(self):
        std_sections = self.section_order

        self.push_to_section()

        for section in std_sections:
            if section.lower() in [s.lower() for s in self.sections.keys()]:
                self.string += self.sections[section.upper()]
                if section.upper() in self.subsections: #does the section have at least one subsection?
                    for subsection in self.subsections[section.upper()]:
                        self.string += subsection

        short_description = self.desc_string.strip()
        short_description = short_description.lower()
        short_description = short_description[0:short_description.find('.')]
        if short_description.endswith('.'):
            short_description = short_description[:-1]

        #self.synopsis_string = self.format.make_synopsis(self.pattern,self.synopsis_string);
        self.string = self.string.replace('HERE_GOES_SYNOPSIS',self.synopsis_string)
        self.string = self.string.replace('HERE_GOES_NAME_UPPER',self.name_string.upper())
        self.string = self.string.replace('HERE_GOES_NAME',self.name_string)
        self.string = self.string.replace('HERE_GOES_SHORT_DESCRIPTION',short_description)
        self.string = self.string.replace('HERE_GOES_DESCRIPTION',self.format.text_raw(self.desc_string))
        self.string = self.string.replace('HERE_GOES_SECTION',self.section)
        if self.arch:
            self.string = self.string.replace('HERE_GOES_ARCH'," "+self.arch)
        else:
            self.string = self.string.replace('HERE_GOES_ARCH','')
        if self.os:
            self.string = self.string.replace('HERE_GOES_OS'," "+self.os)
        else:
            self.string = self.string.replace('HERE_GOES_OS','')

    def push_to_section(self):
        if not self.string or not self.actively_parsed_section:
            return
        self.sections[self.actively_parsed_section] = self.string
        self.reset()

    def reset(self):
        self.string = ''

    def add_code(self,manual_str):
        self.string += manual_str

    def add_line(self,manual_str):
        self.string += "\n" + manual_str

    def parse_option_section(self,doc,section):
        defaults = []
        for s in parse_section(section+':', doc):
            # FIXME corner case "bla: options: --foo"

            _, _, s = s.partition(':')  # get rid of "options:"

            split = re.split('^  (-\S+?)', '\n' + s,flags=re.MULTILINE)[1:]
            split = [s1 + s2 for s1, s2 in zip(split[::2], split[1::2])]

            #for p in split:
            #    print "['"+p+"']"

            #mdoc
            self.format.begin_list()
            options = [Option.parse(s) for s in split if s.startswith('-')]
            defaults += options
            self.format.end_list()
        return defaults

    def parse_info_section(self,doc,section):
        defaults = []
        for s in parse_section(section+':', doc):
            # FIXME corner case "bla: options: --foo"

            _, _, s = s.partition(':')  # get rid of "options:"

            split = re.split('^  (-\S+?)', '\n' + s,flags=re.MULTILINE)[1:]
            split = [s1 + s2 for s1, s2 in zip(split[::2], split[1::2])]

            #for p in split:
            #    print "['"+p+"']"

            #mdoc
            self.format.begin_list()
            #manual_add(s)
            options = [Info.parse(s) for s in split if s.startswith('-')] #in info mode '-' is more like a bullet point
            #defaults += options
            self.format.end_list()
        return defaults

    #def sub_section_of(self,subsection)
    #    for section in self.subsections:
    #        for ss in section:
    #            if ss == subsection:
    #                return section
    #    return ""

    def include_parse(self,text):
        name = ''
        for line in text.split("\n"):
            if line == '': #mdoc doesn't support having empty lines
                continue
            if line.startswith(".Sh "): #Section
                #a section begins
                [macro,name] = line.split(' ')
                self.section_start(name)
                continue
            #I started this code, then noticed it's most likely never needed
            #if line.startswith(".Ss "): #Section
            #    #a section begins
            #    [macro,subname] = line.split(' ')
            #
            #    if name: #we previously parsed a section, of which this is logically the subsection
            #        self.sub_section_start(subname,name)
            #    elif self.sub_section_of(
            #        self.
            #
            #    continue
            self.format.text(line)

    def append_section_if_not_exists(self,section):
        if section.upper() not in self.sections:
            self.sections[section.upper()] = ''

    def set_pattern(self,pattern):
        self.pattern = pattern

    def set_options_list(self,options):
        self.options = options

class Info(LeafPattern):

    def __init__(self, short=None, long=None, argcount=0, value=False):
        assert argcount in (0, 1)
        self.short, self.long, self.argcount = short, long, argcount
        self.value = None if value is False and argcount else value

    @classmethod
    def parse(class_, option_description):
        global manual

        short, long, argcount, value = None, None, 0, False
        options, _, description = option_description.strip().partition('  ')

        options = options.replace('"','\(dq')

        manual.format.item(options,(' '.join(description.strip().split())))

        return class_(short, long, argcount, value)

    def single_match(self, left):
        for n, pattern in enumerate(left):
            if self.name == pattern.name:
                return n, pattern
        return None, None

    @property
    def name(self):
        return self.long or self.short

    def __repr__(self):
        return 'Option(%r, %r, %r, %r)' % (self.short, self.long,
                                           self.argcount, self.value)

def cli2man(doc, argv=None, help=True, version=None, options_first=False):
    """Parse `argv` based on command-line interface described in `doc`.

    """
    global section_non_option_additional;
    global manual;

    manual.begin()

    argv = sys.argv[1:] if argv is None else argv

    usage_sections = parse_section('usage:', doc)

    manual.set_description(section_non_option_additional.strip())

    if len(usage_sections) == 0:
        raise Cli2ManLanguageError('"usage:" (case-insensitive) not found.')
    if len(usage_sections) > 1:
        raise Cli2ManLanguageError('More than one "usage:" (case-insensitive).')
    Cli2ManExit.usage = usage_sections[0]

    options = parse_defaults(doc)

    section = Cli2ManExit.usage
    _, _, section = section.partition(':')  # drop "usage:"

    pattern = parse_pattern(formal_usage(Cli2ManExit.usage), options)

    s = []
    for line in section.split("\n"):
        l = line.lstrip()
        if not l.startswith(manual.name_string) and l != '':
            s[-1]+=l
            continue
        s.append(l)

    a = ''
    for l in s:
        pat = parse_pattern(l,options)
        a+= manual.format.make_synopsis(pat,l)+"\n" #FIXME: creating to many newlines here

    manual.synopsis_string=a

    # [default] syntax for argument is disabled
    #for a in pattern.flat(Argument):
    #    same_name = [d for d in arguments if d.name == a.name]
    #    if same_name:
    #        a.value = same_name[0].value
    argv = parse_argv(Tokens(argv), list(options), options_first)
    pattern_options = set(pattern.flat(Option))

    #for options_shortcut in pattern.flat(OptionsShortcut):
     #   doc_options = parse_defaults(doc)
      #  options_shortcut.children = list(set(doc_options) - pattern_options)
        #if any_options:
        #    options_shortcut.children += [Option(o.short, o.long, o.argcount)
        #                    for o in argv if type(o) is Option]
    extras(help, version, argv, doc)
    matched, left, collected = pattern.fix().match(argv)

    #USELESS... works.. but too late... we need that list much sooner
    l={}
    for a in (pattern.flat() + collected):
        if type(a) is Option:
            l[a.name]=''
            l[a.short]=''

    manual.set_options_list(l)

    return pattern

    #if matched and left == []:  # better error message if left?
    #    return Dict((a.name, a.value) for a in (pattern.flat() + collected))


    #raise Cli2ManExit()



doc ='''
usage: cli2man ( <command> | -i FILE | --stdin ) [options]
               [--option-section NAME ...] [--info-section NAME ...]
               [--set-order SECTIONS] [--gzip]
       cli2man --print-order [--set-order SECTIONS]
       cli2man --version

Use the help message of a command to create a manpage.

Options:
  -h, --help                   show this help message and exit
  -m, --open-in-man            open the output in man
  -z, --gzip                   compress file output
  -o FILE, --output FILE       write to file instead of stdout.
                               when FILE is set to "auto" the
                               format is: command.section(.gz)
  -i FILE, --input FILE        read CLI-help input from file
  --stdin                      read CLI-help input from stdin
  --info-section NAME ...      parse non-option sections
  --option-section NAME ...    parse option sections other than "Options:"
  -I FILE, --include FILE      include material from FILE
  --print-order                prints section order
                               default order if non is set by user
  --set-order SECTIONS         comma separated list of sections
  -s NUM, --section NUM        section number for manual page (default: 1)
  --arch ARCH                  set architecture for manual page
  --os OS                      operating system name
  --see-also PAGES             comma separated list of manpages
                               i.e. --see-also mandoc,mdoc7
                               without number at the end, section 1 is assumed
  -T FORMAT                    set output format (default: mdoc)
                               -Tmarkdown EXPERIMENTAL
                               -Tmdoc
  --create-script FILE         creates manpage generation shell script
                               based on current CLI-settings
  -v, --version                display version information


'''

opt = docopt(doc,None,True,__version__)


if opt['--set-order']:
    manual.section_order = ["MDOC INIT"]
    manual.section_order += opt['--set-order'].split(',') #overwrites default order

if opt['--print-order']:
    print "Section Order:"
    print "____________________"
    for section in manual.section_order:
        if section == "MDOC INIT":
            continue
        print "  "+section
    print "____________________"
    print
    print "  --set-order \""+(','.join(manual.section_order[1:])+"\"")
    print "____________________"
    print
    print "NOTE: You can use this string as basis for modifying the section order."
    print
    print "Copy it and keep the quotation marks, so that whitespace won't cause trouble."
    print
    print "Sections added with --info-section and --option-section are automatically appended"
    print "to the end of the manpage, if they aren't otherwise defined in your order settings."
    print
    print "Sections defined in an --include mdoc file that aren't defined in the section order,"
    print "will NOT appear in the manpage AT ALL."

    exit(0)

out = ''

if opt['--input']:
   out = "".join(open(opt['--input']).readlines())
elif opt['--stdin']:
   out = "".join(sys.stdin.readlines())
else:
   os.environ['LC_ALL'] = 'en_US' #we don't intend to parse localized help messages
   p = subprocess.Popen([opt['<command>'],"-h"], stdout=subprocess.PIPE)
   out, err = p.communicate()
   if p.returncode != 0:
        p = subprocess.Popen([opt['<command>'],"--help"], stdout=subprocess.PIPE)
        out, err = p.communicate()

if opt['--section']:
    manual.section = opt['--section']
if opt['--arch']:
    manual.arch = opt['--arch']
if opt['--os']:
    manual.os = opt['--os']

#prepare the input for docopt's parser, it expects strict conventions and i.e. python's ArgumentParser doesn't produce perfect --help pages

def lil_parser(s):
    r = ''
    count_open = 0
    for c in s:
        if c == '[':
            if count_open < 1:
                r+=c
            count_open += 1
        elif c == ']':
            if count_open < 2:
                r+=c
            count_open -= 1
        else:
            r += c
    return r

out = lil_parser(out)

out = out.replace('optional arguments:','Options:')

empty_argv = []

manual = Manual(opt['-T'])

#here we do most of the real work
manual.set_pattern(cli2man(out,empty_argv))

include = ''
if opt['--include']:
    include = "".join(open(opt['--include']).readlines())



if include:
    manual.include_parse(include)


if opt['--info-section']:
    for info_section in opt['--info-section'] if not isinstance(opt['--info-section'],basestring) else [opt['--info-section']]:
        manual.append_section_if_not_exists(info_section)
        manual.parse_info_section(out,info_section)

if opt['--option-section']:
    for option_section in opt['--option-section'] if not isinstance(opt['--option-section'],basestring) else [opt['--option-section']]:
        manual.append_section_if_not_exists(option_section)
        manual.parse_option_section(out,option_section)

if opt['--see-also']:
    pages = {}
    for page in opt['--see-also'].split(','):
        num = re.sub("\D", "", page)
        page = re.sub("[0-9]", "", page)
        pages[page] = num
    manual.format.see_also(pages)


manual.merge()

if opt['--output']:
    if opt['--output'] == 'auto':
        if not opt['<command>']:
            print "Usage error: when --output=auto is set you have to specify the <command>"
            exit(1)
        opt['--output'] = os.path.basename(opt['<command>'])+"."+manual.section

    if opt['--gzip']:
        opt['--output'] += ".gz"

    if opt['--output'].endswith('.gz'):
        f = gzip.open(opt['--output'],'w+')
    else:
        f = open(opt['--output'],'w+')
    f.write(manual.string)
    f.close()
else:
    print manual.string

if opt['--create-script']:
    args = sys.argv[1:]
    args.remove('--create-script')
    args.remove(opt['--create-script'])
    cmd = 'cli2man '+(' '.join(args))

    f = open(opt['--create-script'],'w+')
    f.write("#!/bin/sh\n")
    f.write("#This script was autogenerated with the --create-script option of cli2man\n\n")
    f.write(cmd+"\n")
    f.close()
    st = os.stat(opt['--create-script'])
    os.chmod(opt['--create-script'], st.st_mode | 0111 )

if opt['--open-in-man']:
    if not opt['--output']:
        f = tempfile.NamedTemporaryFile(delete=False)
        f.write(manual.string)
        f.close()
        subprocess.call('man '+f.name,shell=True)
        os.unlink(f.name)
    else:
        subprocess.call('man ./'+opt['--output'],shell=True)

