#!/usr/bin/python
#
# XenRT Control Suite client. Interacts with HTTP server.
#
# (C) XenSource UK Ltd. 2005
# James Bulpin, November-December 2005

import sys, string, getopt, urllib, os, pwd, tempfile, xmlrpclib, shutil, json
import os.path, re, glob, random, time, datetime, getpass, ConfigParser
import stat

try:
    import xenrtapi
    import requests
except:
    sys.path.insert(0, "/usr/groups/xenrt/lib")
    import requests
    import xenrtapi

PROFILER_ENABLED = ("XENRT_PROFILE" in os.environ)
XRTPROF = None
XRTPROF_FILENAME = os.path.join(("TMP" in os.environ and os.environ["TMP"] or "/tmp"), "xenrt.prof")

_console = "/usr/groups/xen/xenuse/bin/console"
_conserver = "10.220.254.109"
_conskey = """-----BEGIN RSA PRIVATE KEY----- MIIEowIBAAKCAQEAqsXo9q6HdjxNzyFsu92JB8UG5Nbb8eUk4KlsbycNacXO4m8I fBD1ScM79y+UZk0J51V+EjRZP7s+syc1fH2tHsNsJjjPaXmw65RYzPxN7vZTRXEl rwE7qzG/j3O2AmfC3N12s/JS0hm30UjBBrlBLyTvoLA+OLD2oJtmRfFO4GPgTTdU t3CuotfI+NvPj06aGGESe1ssNUn27PRsMReaUgDqkHc/VTLwPXxtoB/tMXDasTp0 tt5K/EpYrI1y7xFsxv/fcOTrRnM4Chw4xgyV95mQ/mYEo8E68b50hgba3iw/N0hv qS/fdJmFXWSd6fnNH6lEj+uJ3lv+zFY4Zlf6EwIDAQABAoIBAC3Yd9RRwngRdGoY 9RHyRWlakaE5tU5ZT+8oTL4CY+1zdzj+ye5UtyUYTq7rjRFxuSjroK7Ocu9TG2AJ NKi5LJLHr+8K7VPJoZ988eIGEf4HDw/jclRrJlOtRVoDnGdE5+FXaxigA2/4C4sZ 1wgxD7jSV6px5iJeD7fKQEiioaRHe5nYEqev6iZMQLAdyda4Jz9weuH81nrM/gNg 9K5oSHdJun2MbHBiCKNcSYITkgcm/wIeaws1mnDBHyJ5u6bnFtC3t9DFqNFXXY5D JEkopNDGt+70MaLtJ41XeMLhrcQDJ4Zo3pVXN29bQ+CNLLBqF9IrVPDfSd/nQU2e nvVsKqECgYEA0hHIEB/JADvHXf/xrcrKblqxY6CCC8l9yUembOAdkqVizvs5HOY9 XOi2CVGrldI8gz/Ebp8WNzRFkrJ7KY5bmMB5igT/eqf28KafV9sVhbowuodF7aEF ldAHgaM/kE51oRPNxKdifR6/dnfN/KFrx1SS2C+HHWfmhI8UVH6yoekCgYEA0ByX roXduU/zxK9PyCEz2mGHJBDzXmk+j9OXCh86tVqyKVQ/THjvGatdlkvgvYjNwkGR xOIjqaU4bc6j7zbSyaOpVY9R0T+VwhVIonaMWd+YYNDBCblyEjk2lWE3FlRzX1ho aEaZfG4y8XqC2j1eYZLRpFWfI2b9a0TLrJzXIpsCgYEAnHQV1rx1jTD4eS/oJZHF Vo9BUnv2nyTABCrS7n+0RsXaY2GD1dw8k3fWK/ahnoHoA7B4chMWDqNlta4sQHUA DzI60uqKBCYYqn7OajaHPwO+yw14rVShvkQjcw/MhYOJ5B/FvtMTbTB7VJZB2ge3 J8v7Wft73BDIBK4zbAK4tBECgYBfpHRVQbJnowXt43/wAspY16+K9LtULVwSL3Tf swCBLkBo3A4HMCD6lBX7p72OfZvKZeX/LMqlAYiQdoJCwthW0P9WkfpRBz7lJnDu MTtJZogaHnZI6NAy7dZXComNmpw3OmMpYSDS3bZN4jbW45NHlnkAZFUxqPhYXxKX rrD0dwKBgFQg3WsKooaR+eIOkF06gTxE0GZcvIXI0cuFzPKMIEUAl/wzX9mnuwOH g4TpLvkRyc6KBDuozePGUxrF2jh9Q5wJR0rCR8k6VnqTOpI3S0kBL7zAZix/gkfv hZzt29VZ7NntOok/uXAmrm5/DtnFmGw1ulmTCFM5vKXoGeiUODeR -----END RSA PRIVATE KEY-----"""
_authrealm = """citrite"""

class XenRTCommand:

    name = "<default>"
    summary = "<--- no summary --->"
    usage = ""
    mandatory = ""
    group = "Other"

    commands = None
    hide = False
    usesAPI = True

    def __init__(self, commands):
        self.commands = commands

    def getConfigFile(self):
        path = "%s/.xenrtrc" % os.path.expanduser("~")
        try:
            config = ConfigParser.ConfigParser()
            config.read(path)
            return config
        except:
            return None

    def getAPIKey(self):
        if os.getenv("XENRT_APIKEY"):
            return os.getenv("XENRT_APIKEY")
        else:
            try:
                return self.getConfigFile().get("xenrt", "apikey").strip()
            except:
                return None

    def getXenRTServer(self):
        if os.getenv("XENRT_SERVER"):
            return os.getenv("XENRT_SERVER")
        try:
            return self.getConfigFile().get("xenrt", "server").strip()
        except:
            return None

    def retrieveAPIKeyFromServer(self):
        if not sys.stdin.isatty():
            raise Exception("Running non-interactively and no API key specified in ~/.xenrtrc")
        global _authrealm
        defaultuser = getpass.getuser()
        user = raw_input("Enter your %s username [%s]: " % (_authrealm, defaultuser))
        if not user.strip():
            user = defaultuser
        password = getpass.getpass()
        xrt = xenrtapi.XenRT(user=user, password=password, server=self.getXenRTServer())
        apikey = xrt.get_apikey()
        path = "%s/.xenrtrc" % os.path.expanduser("~")
        print "Writing %s as API key in ~/.xenrtrc" % apikey
        config = ConfigParser.ConfigParser()
        config.add_section("xenrt")
        config.set("xenrt", "apikey", apikey)
        with open(path, "wb") as f:
            config.write(f)
        os.chmod(path, stat.S_IRUSR|stat.S_IWUSR)

    def dispatch(self, args):
        if self.usesAPI:
            apikey = self.getAPIKey()
            if not apikey:
                self.retrieveAPIKeyFromServer()
                apikey = self.getAPIKey()
            self.xenrt = xenrtapi.XenRT(apikey=self.getAPIKey(), server=self.getXenRTServer())

        if len(args) > 0 and args[0] in ("--help", "-h"):
            print "%s\n" % (self. summary)
            if self.usage == "":
                opts = ""
            else:
                opts = " [options]"
            print "Usage: %s %s %s%s" % \
                  (sys.argv[0], self.name, self.mandatory, opts)
            print self.usage
            return 0
        self.run(args)

    def boolToYN(self, val):
        return "yes" if val else "no"

    def printout(self, str):
        sys.stdout.write(str)

    def uploadWithCurl(self, url, localFn, remoteFn):
        cmd = ["curl"]
        cmd.append("--fail")
        cmd.append("-H \"Expect:\"")
        if remoteFn:
            cmd.append("-F 'filename=%s'" % (remoteFn))
        cmd.append("-F file=@%s" % (localFn))
        cmd.append(url)
        rc = os.system("%s > /dev/null 2>&1" % (string.join(cmd)))
        if rc != 0:
            raise Exception("curl returned non-zero exit status")

    def upload(self, url, localFn=None, remoteFn=None):
        if os.path.exists("/usr/bin/curl"):
            return self.uploadWithCurl(url, localFn, remoteFn)
        if localFn:
            f = file(localFn, 'r')
        else:
            f = sys.stdin
        r = requests.post(url, files={"file": (remoteFn or localFn or "stdin", f)})
        r.raise_for_status()

    def parse_options(self, args):
        secondary = {}
        details = {}
        argmap = {
            '-v': 'VERSION',
            '-r': 'REVISION',
            '-o': 'OPTIONS',
            '-n': 'DEPS',
            '-R': 'REPO',
            '-t': 'TARGET',
            '-F': 'FLAGS'
            }
        argmapbool = {
            '-P': 'PUBLISH',
            '-d': 'DEBUGCMD',
            }
        try:
            optlist, optx = getopt.getopt(args, 'm:v:r:o:n:R:t:F:PU:dU:D:',
                                          ['rpm', 'optrpm', 'nfs', 'gnbd',
                                           'gnbdserver=', 'gnbdclient=',
                                           'domu', 'patch=', 'xenargs=',
                                           'distro=', 'sep', 'smpdom0',
                                           'nobuildopts', 'filevbd', 'swap',
                                           'filevbdnfs', 'carboncd=',
                                           'carbonpatch=', 'xenbuild=',
                                           'ring0', 'perftag=', 'email=',
                                           'tarball=', 'noxenargs',
                                           'nodom0args', 'dom0args=',
                                           'hold=', 'holdfail=', 'arch=',
                                           'sepkern=', 'sepkrev=',
                                           'failkeep', 'res=', 'pool=',
                                           'xgt=', 'xgtnfs=', 'qcow',
                                           'removepassed', 'rpmpath=',
                                           'guestrpms=', 'pw=', 'guestaddr=',
                                           'pq=', 'pqpatch=', 'skip=',
                                           'skipgroup=', 'skiptype=',
                                           'run=', 'rungroup=',
                                           'pause-on-fail=',
                                           'pause-on-pass=',
                                           'hvarch=', 'no-finally',
                                           'debug', 'verbose', 'number=',
                                           'priority=', 'customsequence=',
                                           'customupdates=',
                                           'perf-data=', 'perf-regress=',
                                           'testcasefiles=', 'res1=',
                                           'inputs=',
                                           'flags=','guard=',
                                           'xenrtpqname='])
            for argpair in optlist:
                (flag, value) = argpair
                if argmap.has_key(flag):
                    details[argmap[flag]] = value
                elif argmapbool.has_key(flag):
                    details[argmapbool[flag]] = "yes"
                elif flag == "-m":
                    details['MACHINE'] = value
                    details['MACHINES_SPECIFIED'] = "yes"
                elif flag == "--rpm":
                    details['OPTION_XEN_PACKAGE'] = "rpm"
                elif flag == "--rpmpath":
                    details['OPTION_XEN_PACKAGE'] = "rpm"
                    details['OPTION_RPM_PATH'] = value
                elif flag == "--guestrpms":
                    details['OPTION_GUEST_RPMS'] = value
                    details['OPTION_XEN_PACKAGE_G'] = "rpm"
                elif flag == "--optrpm":
                    details['OPTION_XEN_PACKAGE'] = "rpm"
                    details['RPMARG'] = "2"
                elif flag == "--nfs":
                    details['OPTION_GUEST_ROOT'] = "nfs"
                elif flag == "--filevbd":
                    details['OPTION_GUEST_ROOT'] = "file"
                elif flag == "--filevbdnfs":
                    details['OPTION_GUEST_ROOT'] = "file"
                    details['OPTION_GUEST_BACKING'] = "nfs"
                elif flag == "--qcow":
                    details['OPTION_GUEST_ROOT'] = "qcow"
                elif flag == "--gnbd":
                    details['OPTION_USE_GNBD'] = "yes"
                elif flag == "--gnbdserver":
                    details['OPTION_GNBD_SERVER'] = value
                elif flag == "--gnbdclient":
                    details['OPTION_GNBD_SERVE_FOR'] = value
                elif flag == "--domu":
                    details['OPTION_ONLY_DOMU'] = "yes"
                    details['OPTION_SEP_DOM0U'] = "yes"
                    details['OPTION_BUILD_COMPONENTS'] = "kernels"
                    details['XEN_BUILD_OPTIONS2'] = "KERNELS=linux-2.6-xen0"
                    details['EXTRA_BUILD_OPTIONS'] = "KERNELS=linux-2.6-xenU"
                elif flag == "--patch":
                    details['PATCH_XEN'] = os.path.basename(value)
                    secondary['patch'] = value
                elif flag == "--testcasefiles":
                    tl = [ os.path.basename(x) for x in 
                           string.split(value, ",") ]
                    details['TESTCASEFILES'] = string.join(tl, ",")
                    secondary['testcasefiles'] = value
                    for f in string.split(value, ","):
                        if not os.path.exists(f):
                            raise Exception("Cannot find testcase file %s" % f)
                elif flag == "--customsequence":
                    details['CUSTOM_SEQUENCE'] = "yes"
                    details['DEPS'] = os.path.basename(string.split(value, ",")[0])
                    secondary['seqfiles'] = value
                    for f in string.split(value, ","):
                        if not os.path.exists(f):
                            raise Exception("Cannot find sequence file %s" % f)
                elif flag == "--customupdates":
                    updates = []                
                    for u in string.split(value, ","):
                        if not os.path.exists(u):
                            raise Exception("Update %s doesn't exist on this machine." % u)
                        known_formats = ["rpm", "tar", "tar.gz", "tgz", "tar.bz", "tbz"]                    
                        if len(filter(lambda ext: u.endswith("." + ext), known_formats)) == 0:
                            raise Exception("Update %s has an unknown format ."
                                             "Only the following formats are supported: %s"
                                             % (u, known_formats))
                        updates.append(os.path.basename(u))
                    details['CUSTOM_UPDATES'] = string.join(updates, ",")
                    secondary['customupdates'] = value
                elif flag == "--perf-data":
                    details['PERFDATAFILE'] = "yes"
                    secondary['perfdata'] = value
                elif flag == "--perf-regress":
                    details['PERFREGRESSFILE'] = "yes"
                    secondary['perfregress'] = value
                elif flag == "--xenargs":
                    details['XEN_EXTRA_ARGS_USER'] = value
                elif flag == "--dom0args":
                    details['DOM0_EXTRA_ARGS_USER'] = value
                elif flag == "--noxenargs":
                    details['OPTION_NO_XEN_ARGS'] = "yes"
                elif flag == "--nodom0args":
                    details['OPTION_NO_DOM0_ARGS'] = "yes"
                elif flag == "-U":
                    details['USERID'] = value
                elif flag == "--distro":
                    try:
                        (pref, rest) = string.split(value, ":", 1)
                        guest = string.atoi(pref)
                        if rest[-1] == "+":
                            rest = rest[:-1]
                            details['OPTION_XEN_PACKAGE[%u]' % (guest)] = \
                                                             "existing"
                        details['ROOT_DISTRO_DOM[%u]' % (guest)] = rest
                        details['OPTION_GUEST_XGT[%u]' % (guest)] = rest
                    except:
                        details['ROOT_DISTRO'] = value
                        details['OPTION_GUEST_XGT'] = value
                elif flag == "--sep":
                    details['OPTION_SEP_DOM0U'] = "yes"
                elif flag == "--smpdom0":
                    details['OPTION_SMP_DOM0'] = "yes"
                elif flag == "--nobuildopts":
                    details['OPTION_CLEAR_BUILDOPTS'] = "yes"
                elif flag == "--debug":
                    details['OPTION_DEBUG'] = "yes"
                elif flag == "--verbose":
                    details['OPTION_VERBOSE'] = "yes"
                elif flag == "--swap":
                    details['OPTION_SWAP_DOMU'] = "yes"
                elif flag == "--carboncd":
                    details['CARBON_CD_IMAGE'] = value
                elif flag == "--carbonpatch":
                    details['CARBON_INSTALLER_PATCH'] = value
                elif flag == "--xenbuild":
                    details['USER_BUILD_OPTIONS'] = value
                elif flag == "--ring0":
                    details['USER_BUILD_OPTIONS'] = "supervisor_mode_kernel=y"
                    details['OPTION_DEFAULT_GLIBC'] = "yes"
                elif flag == "--perftag":
                    details['PERFTAG'] = value
                elif flag == "--email":
                    details['EMAIL'] = value
                elif flag == "--tarball":
                    try:
                        (pref, rest) = string.split(value, ":", 1)
                        guest = string.atoi(pref)
                        details['OPTION_XEN_PACKAGE[%u]' % (guest)] = "tarball"
                        details['INSTALLATION_TARBALL[%u]' % (guest)] = rest
                    except:
                        details['INSTALLATION_TARBALL[0]'] = value
                        details['OPTION_XEN_PACKAGE'] = "tarball"
                elif flag == "--hold":
                    details['MACHINE_HOLD_FOR'] = value
                elif flag == "--holdfail":
                    details['MACHINE_HOLD_FOR_FAIL'] = value
                elif flag == "--arch":
                    (pref, rest) = string.split(value, ":", 1)
                    guest = string.atoi(pref)
                    details['GUEST_ARCH[%u]' % (guest)] = rest
                elif flag == "--sepkern":
                    details['SEPARATE_KERNEL_TREE'] = value
                elif flag == "--sepkrev":
                    details['OPTION_SEP_KREV'] = value
                elif flag == "--failkeep":
                    details['CLEANUP'] = "onsuccess"
                elif flag == "--res":
                    details['RESOURCES_REQUIRED'] = value
                elif flag == "--res1":
                    details['RESOURCES_REQUIRED_1'] = value
                elif flag == "--pool":
                    details['POOL'] = value
                elif flag == "--xgt":
                    if details.has_key("EXTRA_XGTS"):
                        details['EXTRA_XGTS'] = details['EXTRA_XGTS'] + " " \
                                                + value
                    else:
                        details['EXTRA_XGTS'] = value
                elif flag == "--xgtnfs":
                    details['OPTION_XGT_NFS'] = value
                elif flag == "--pw":
                    details['ROOT_PASSWORDS'] = value
                elif flag == "--guestaddr":
                    details['GUEST_HOSTNAME[1]'] = value
                elif flag == "-D":
                    try:
                        var, varval = string.split(value, "=", 1)
                        if len(var) > 24:
                            raise Exception("-D parameter too long (24 chars max): %s" % (var))
                        if len(varval) > 255:
                            raise Exception("-D value too long (255 chars max): %s=%s" %
                                (var, varval))
                        details[var] = varval
                    except:
                        raise Exception("Error parsing -D variable '%s'\n" %
                                         (value))
                elif flag == "--removepassed":
                    details['OPTION_REMOVE_PASSED'] = "yes"
                elif flag == "--pq":
                    details['OPTION_PATCHQUEUE'] = value
                elif flag == "--pqpatch":
                    details['OPTION_PQ_PATCH'] = value
                elif flag == "--skip":
                    details['SKIP_%s' % (value)] = "yes"
                elif flag == "--skipgroup":
                    details['SKIPG_%s' % (value)] = "yes"
                elif flag == "--skiptype":
                    details['SKIPT_%s' % (value)] = "yes"
                elif flag == "--run":
                    details['RUN_%s' % (value)] = "yes"
                elif flag == "--rungroup":
                    details['RUNG_%s' % (value)] = "yes"
                elif flag == "--priority":
                    details['PRIORITY'] = value
                elif flag == "--pause-on-fail":
                    details['POF_%s' % (value)] = "yes"
                elif flag == "--pause-on-pass":
                    details['POP_%s' % (value)] = "yes"
                elif flag == "--hvarch":
                    details['HVARCH'] = value
                elif flag == "--no-finally":
                    details['NOFINALLY'] = "yes"
                elif flag == "--number":
                    details['MACHINES_REQUIRED'] = value
                elif flag == "--inputs":
                    details['INPUTDIR'] = value
                elif flag == "--flags":
                    details['FLAGS'] = value
                elif flag == "--guard":
                    details['GUARD'] = value
                elif flag == "--xenrtpqname":
                    details['XENRT_PQ_NAME'] = value
                else:
                    raise Exception("Unknown argument '%s'\n" % (flag))
        except getopt.GetoptError, e:
            raise Exception("Unknown argument: %s\n" % (str(e)))

        if details.has_key('USER_BUILD_OPTIONS') and \
               details['USER_BUILD_OPTIONS'] == "supervisor_mode_kernel=y":
            if not details.has_key('TARGET'):
                details['TARGET'] = "dom0"
            if not details.has_key('OPTION_CLEAR_BUILDOPTS'):
                details['OPTION_CLEAR_BUILDOPTS'] = "yes"

        # For vendor kernel testing set a default repo
        if details.has_key('OPTION_ONLY_DOMU') and \
               details['OPTION_ONLY_DOMU'] == "yes":
            if not details.has_key('SECONDARY_REPO_VERSION'):
                details['SECONDARY_REPO_VERSION'] = "xen-3.0-testing"

        # And not let OPTION_ONLY_DOMU through otherwise run-all will override
        if details.has_key('OPTION_ONLY_DOMU'):
            del details['OPTION_ONLY_DOMU']

        return details, secondary

    job_usage = """
        [ --inputs <directory> ]    directory containing product
        [ -o <architecture> ]       x86-32 (default), x86-32p or x86-64
        [ -v <version> ]            xen-unstable (default), linux, etc.
        [ -r <revision> ]           as datestamp of changeset hash (default is tip)
        [ -m <machine> ]            optional specification of required machine
        [ -R <repository URL> ]     path, ssh or http. (default is local cache)
        [ -n <sequence> ]           test sequence (default is full run)
        [ -t <target> ]             sequence target (default 'all')
        [ -F <flags> ]              miscellaneous for machine selection, e.g. 4GB
        [ -P ]                      enable publishing of results
        [ -d ]                      debug command (show config info, no run)
        [ --res <string> ]          resource requirements
        [ --pool <poolname> ]       the machine pool to use
        [ --number <n> ]            number of machines to acquire (default 1)
        [ --arch <guest>:<arch> ]   x86-32, x86-32p or x86-64
        [ --hvarch <arch> ]         Hypervisor arch (v2 jobs only)
        [ --rpm ]                   test community RPMs
        [ --optrpm ]                test Optimizer RPMs
        [ --rpmpath <path> ]        path to RPMs for dom0/general use
        [ --guestrpms <path> ]      path to RPMs to use for guests
        [ --tarball <URL> ]         use a dist tarball from the specified URL
        [ --tarball <guest>:<URL> ] use a dist tarball from the specified URL
        [ --distro <distro> ]       the Linux distribution to use
        [ --distro <guest>:<distro>[+] ] the Linux distribution to use
        [ --nfs ]                   use NFS root for guests (where supported)
        [ --filevbd ]               use file-backed VBD for guests (local storage)
        [ --filevbdnfs ]            use file-backed VBD for guests (NFS storage)
        [ --qcow ]                  use blktap QCOW storage (if supported)
        [ --gnbd ]                  use GNBD for guest roots
        [ --gnbdclient <machine> ]  be a GNBD server for <machine>
        [ --gnbdserver <machine> ]  use <machine> as GNBD server instead of local
        [ --swap ]                  give swap partitions to guests
        [ --domu ]                  only build guest kernel from this repository,
                                    hypervisor, dom0 kernel and tools come from
                                    xen-3.0-testing
        [ --sep ]                   build -xenU/0 (rather than -xen) kernels
        [ --smpdom0 ]               build and/or boot dom0 on multiple CPUs
        [ --noxenargs ]             clear any default Xen command line arguments
        [ --nodom0args ]            clear any default dom0 command line arguments
        [ --xenargs "<args>" ]      additional arguments for the Xen command line
        [ --dom0args "<args>" ]     additional arguments for the dom0 command line
        [ --debug ]                 perform a debug build (if applicable)
        [ --verbose ]               perform a verbose build (if applicable)
        [ --xenbuild "<opts>" ]     extra build options for the Xen "make world"
        [ --ring0 ]                 build dom0 to run in ring0 (no domUs allowed)
        [ --carboncd "<iso>" ]      path to Carbon installer ISO image
        [ --perftag "<string>" ]    add a performance tag for graph grouping
        [ --email <address> ]       email a summary of results to this address
        [ --hold <minutes> ]        hold a machine after job completes
        [ --holdfail <minutes> ]    hold a machine after job fails
        [ --sepkern <URL> ]         use the kernel from this kernel-only repo
        [ --sepkrev <revision> ]    the revision for the above (default is tip)
        [ --xgtnfs <NFS URL> ]      NFS path for XGT imports
        [ --xgt <xgt file/url> ]    Add an extra XGT to a Carbon installation
        [ -D <variable>=<value> ]   set a general variable
        [ --removepassed ]          remove the job from the joblist if it passes
        [ --pw <password> ]         root password for existing guests
        [ --guestaddr <ipaddr> ]    guest IP address for existing guests
        [ --skip <test> ]           skip test <test> (all phases)
        [ --skipgroup <group> ]     skip group <group>
        [ --skiptype <type> ]       skip test type <type>
        [ --run <test> ]            do not skip test <test> (all phases)
        [ --rungroup <group> ]      do not skip group <group>
        [ --priority <n> ]          run tests up to and including P<n>
        [ --pause-on-fail <test> ]  ask for user intervention for <test> or "ALL"
        [ --pause-on-pass <test> ]  ask for user intervention for <test> or "ALL"
        [ --no-finally ]            no run run any "finally" actions
        [ --customsequence <file> ] use a customised sequence file       
        [ --perf-data <file> ]      upload performance limits data
        [ --perf-regress <file> ]   upload performance regression data
        [ --testcasefiles <files> ] upload extra testcases
        [ --customupdates <files> ] upload customized host installation updates (rpm/tarball)
        [ --guard <guard> ]         patchqueue guard to apply
        [ --xenrtpqname <pqname> ]  patchqueue repository, default "xenrt.pq.hg" (use with --guard)
        """
        
class XenRTStatus(XenRTCommand):

    name = "status"
    summary = "Get the status of a submitted job. Any follow-up arguments" \
            " are considered variable names that we wish status to return.\n" \
            "(Analogous to piping xenrt status through grep.)"
    mandatory = "<jobid> [VarName1] [VarName#] [...]"
    group = "Job"

    def printVar(self, var):
        return not self.printVars or var in self.printVars

    def run(self, args):
        results = self.xenrt.get_job(int(args[0]))
        # Print CHECK first (XRT-303)
        self.printVars = args[1:]
        if results['result'] and self.printVar("CHECK"):
            self.printout("CHECK='%s'\n" % (results["result"]))
        if self.printVar("JOBSTATUS"):
            self.printout("JOBSTATUS='%s'\n" % (results["rawstatus"]))
        if self.printVar("USERID"):
            self.printout("USERID='%s'\n" % (results["user"]))
        if self.printVar("JOBID"):
            self.printout("JOBID='%s'\n" % (results["id"]))
        if self.printVar("PAUSED"):
            self.printout("PAUSED='%s'\n" % self.boolToYN("paused" in [x['result'] for x in results['results'].values()]))
        for key in results['params'].keys():
            if key != "CHECK" and self.printVar(key):
                self.printout("%s='%s'\n" % (key, results['params'][key]))

class XenRTMachine(XenRTCommand):

    name = "machine"
    summary = "Get data for the specified machine"
    mandatory = "<machine>"
    group = "Machine"

    def run(self, args):
        results = self.xenrt.get_machine(args[0])
        self.printout("STATUS='%s'\n" % results['rawstatus'])
        for i in ('jobid', 'leasereason', 'pool', 'leaseuser', 'cluster', 'site', 'prio', 'leasepolicy', 'aclid'):
            if results[i]:
                self.printout("%s='%s'\n" % (i.upper(), results[i]))
        for i in ('leaseto', 'leasefrom'):
            if results[i]:
                t = datetime.datetime.utcfromtimestamp(results[i]).isoformat(" ")
                self.printout("%s='%s'\n" % (i.upper(), t))
        if results['leaseuser']:
            self.printout("COMMENT='%s'\n" % results['leaseuser'])
        if results['resources']:
            self.printout("RESOURCES='%s'\n" % "/".join(["=".join(x) for x in results['resources'].items()]))
        if results['description']:
            self.printout("DESCRIPTION='%s'\n" % results['description'])
        for key in results['params'].keys():
            self.printout("%s='%s'\n" % (key, results['params'][key]))

class XenRTShowLog(XenRTCommand):

    name = "showlog"
    summary = "Show the test progress log"
    mandatory = "<jobid>"
    usage = """
    [ -v ]      verbose    
    [ -w ]      wide format
    [ -t ]      show testcase durations
    """
    group = "Job"

    def run(self, args):
        if len(args) < 1:
            raise Exception("Invalid usage\n")
        verbose=False
        wide=False
        times=False
        try:
            optlist, optx = getopt.getopt(args[1:], 'vwt')
            for argpair in optlist:
                (flag, value) = argpair
                if flag == "-v":
                    verbose = True
                elif flag == "-w":
                    wide = True
                elif flag == "-t":
                    times = True
                else:
                    raise Exception("Unknown argument '%s'\n" % (flag))
        except getopt.GetoptError:
            raise Exception("Unknown argument\n")

        
        job = self.xenrt.get_job(string.atoi(args[0]), logitems=(verbose or times))

        pref = ""
        if wide:
            pref = job['params'].get("OPTIONS", "")
        if pref:
            pref += " "

        for r in sorted(job['results'].keys(), key=int):
            timestr = ""
            if times:
                start = [x['ts'] for x in job['results'][r]['log'] if x['log'] == "started"]
                finish = [x['ts'] for x in job['results'][r]['log'] if x['log'] in("pass", "fail", "error", "partial")]
                if start and finish:
                    timestr = " (Duration %6us)" % (finish[0] - start[0])
            self.printout("%s%-10s %-12s %-10s%s\n" % (
                    pref,
                    job['results'][r]['phase'],
                    job['results'][r]['test'],
                    job['results'][r]['result'],
                    timestr))
            if verbose:
                for l in job['results'][r]['log']:
                    dt = datetime.datetime.utcfromtimestamp(l['ts']).isoformat(" ")
                    self.printout("...[%-19s] %-10s %s\n" % (dt, l['type'], l['log']))
            

class XenRTMList2(XenRTCommand):

    name = "mlist2"
    summary = "Get a list of machines"
    usage = """
    [-s] <site>      Filter on site
    [-c] <cluster>   Filter on cluster
    [-o] <pool>      Filter on pool
    [-R] <resources> Filter on a resource string (e.g. "memory>3G/disks=1")
    [-P] <props>     Filter on machines matching the props string
    [-f] <flags>     Filter on machine flags
    [-b] <user>      Filter on machines borrowed by user
    [-n]             Filter on machines that are not borrowed
    [-m]             Filter on machines borrowed by me
    [-a]             Filter on machines borrowed by anyone
    [-r]             Show machine resources
    [-d]             Show machine descriptions
    [-p]             Show machine properties
    [-v]             Show pseudohosts
    """

    def run(self, args):
        self.controller = False
        details = {}
        quiet = False
        show=None
        filterLeased = None
        filterBroken = False
        controllerFormat = False
        try:
            optlist, optx = getopt.getopt(args, 's:c:o:qrCdR:pP:vf:mb:na', "broken")
            for argpair in optlist:
                (flag, value) = argpair
                if flag == "-s":
                    details["site"] = [value]
                elif flag == "-C":
                    controllerFormat = True
                    details["pseudohosts"] = True
                elif flag == "-c":
                    details["cluster"] = [value]
                elif flag == "-o":
                    details["pool"] = [value]
                elif flag == "-q":
                    quiet = True
                elif flag == "-r":
                    show = "Resources"
                elif flag == "-d":
                    show = "Description"
                elif flag == "-R":
                    details["resource"] = value.split("/")
                elif flag == "-p":
                    show = "Properties"
                elif flag == "-P":
                    if not "flag" in details:
                        details['flag'] = []
                    details["flag"].extend(value.split(","))
                elif flag == "-f":
                    if not "flag" in details:
                        details['flag'] = []
                    details["flag"].extend(value.split(","))
                elif flag == "-b":
                    details["user"] = [value]
                elif flag == "-m":
                    details["user"] = ["${user}"]
                elif flag == "-n":
                    filterLeased = False
                elif flag == "-a":
                    filterLeased = True
                elif flag == "-v":
                    details["pseudohosts"] = True
                elif flag == "--broken":
                    filterBroken = True
                else:
                    raise Exception("Unknown argument '%s'\n" % (flag))
        except getopt.GetoptError:
            raise Exception("Unknown argument\n")
        machines = self.xenrt.get_machines(**details)

        jobids = [x['jobid'] for x in machines.values() if x['status'] == "running"]

        jobs = self.xenrt.get_jobs(jobid=jobids, params=True, limit=0)

        if show:
            chead = show
        else:
            chead = "Comment/Leased to"

        fmt = "%-15s %-12s %-8s %-9s %-8s %s\n"
        if not quiet and not controllerFormat:
            self.printout(fmt % ("Machine", "Site", "Cluster", "Status", "Pool", chead))
            self.printout("============================================================================\n")
            
        for m in sorted(machines.keys()):
            if filterBroken and not machines[m]['broken']:
                continue
            if filterLeased and not machines[m]['leaseuser']:
                continue
            if filterLeased == False and machines[m]['leaseuser']:
                continue
            if controllerFormat:
                self.printout("%s,%s,%s" % (m,machines[m]['rawstatus'],str(machines[m]['jobid']) if machines[m]['jobid'] else ""))
            else:
                status = machines[m]['rawstatus']
                if status == "scheduled":
                    status = "%d (S)" % machines[m]['jobid']
                elif status == "running":
                    status = "%d" % machines[m]['jobid']
                elif status == "slaved":
                    status = "(%d)" % machines[m]['jobid']

                if show == "Resources":
                    comment = "/".join([k + "=" + v for k, v in machines[m]['resources'].iteritems()])
                elif show == "Properties":
                    comment = ",".join(machines[m]['flags'])
                elif show == "Description":
                    comment = machines[m]['description'] or ""
                else:
                    if machines[m]['broken']:
                        comment = "Broken"
                        brokenInfo = machines[m]['params'].get("BROKEN_INFO")
                        brokenTicket = machines[m]['params'].get("BROKEN_TICKET")
                        if brokenInfo or brokenTicket:
                            comment += " -"
                        if brokenInfo and brokenTicket and brokenTicket in brokenInfo:
                            brokenTicket = None
                        if brokenTicket:
                            comment += " " + brokenTicket
                        if brokenInfo:
                            comment += " " + brokenInfo
                    elif machines[m]['leaseuser']:
                        dt = datetime.datetime.utcfromtimestamp(machines[m]['leaseto']).isoformat(" ")
                        if machines[m]['leasereason']:
                            userstr = "%s - %s" % (machines[m]['leaseuser'], machines[m]['leasereason'])
                        else:
                            userstr = machines[m]['leaseuser']
                        comment = "%s (%s)" % (dt, userstr)
                    elif machines[m]['status'] == "running" and str(machines[m]['jobid']) in jobs:
                        job = jobs[str(machines[m]['jobid'])]
                        comment = "%s - %s" % (job["description"], job['user'])
                    else:
                        comment = ""
                
                self.printout(fmt % (m, machines[m]['site'], machines[m]['cluster'] or "DEFAULT", status, machines[m]['pool'] or "DEFAULT", comment))
                    

class XenRTMList(XenRTMList2):
    # Alias for xenrt mlist2
    name = "mlist"

class XenRTList(XenRTCommand):

    name = "list"
    summary = "Show status of new or running jobs"
    usage = """

    The optional parameters specify search filters.

    [ -m ]                   show only my jobs
    [ -U <user> ]            show only <user>'s jobs
    [ -n ]                   show only new jobs
    [ -r ]                   show only running jobs
    [ -s ]                   show testrun suite IDs if known
    [ -S <suiterun id> ]     show only jobs from this suite run
    [ -D <description> ]     show only jobs matching the given JOBDESC
    """
    group = "Scheduler"

    def run(self, args):
        showsr = False
        details = {}
        quiet = False
        jobdesc = None
        try:
            optlist, optx = getopt.getopt(args, 'mU:nrsS:qD:')
            for argpair in optlist:
                (flag, value) = argpair
                if flag == "-U":
                    details["user"] = [value]
                elif flag == "-S":
                    details["suiterun"] = [value]
                elif flag == "-m":
                    details["user"] = "${user}"
                elif flag == "-n":
                    details["status"] = ["new"]
                elif flag == "-r":
                    details["status"] = ["running"]
                elif flag == "-s":
                    showsr = True
                elif flag == "-q":
                    quiet = True
                elif flag == "-D":
                    jobdesc = value
                else:
                    raise Exception("Unknown argument '%s'\n" % (flag))
        except getopt.GetoptError:
            raise Exception("Unknown argument\n")
        if not "status" in details:
            details['status'] = ["new", "running"]
        jobs = self.xenrt.get_jobs(params=True, limit=10000, results=True, **details)
        
        fieldwidths = [7, 8, 12, 9, 13, 18, 21]
        fmt = string.join(map(lambda x:"%%-%us" % x, fieldwidths))
        fmt = fmt + "\n"
        if not quiet:
            self.printout(fmt % ("ID",
                                 "Status",
                                 "Host(s)",
                                 "Version",
                                 "Revision",
                                 "Desc",
                                 "User"))
            self.printout("--------------------------------------------------"
                          "-----------------------------\n")
        for s in ['new', 'running']:
            for job in sorted([x for x in jobs.keys() if jobs[x]['status'] == s], key=int, reverse=True):
                j = jobs[job]
                
                if jobdesc and j['description'] != jobdesc:
                    continue
                status = j['status']

                if "paused" in [x['result'] for x in j['results'].values()]:
                    status = "paused"

                if showsr and j['suiterun']:
                    desc = "SR%d %s" % (j['suiterun'], j['description'] or "")
                else:
                    desc = j['description'] or ""

                ll = [
                    str(j['id']),
                    status,
                    ",".join(j['machines']),
                    j['params'].get("VERSION", ""),
                    j['params'].get("REVISION", ""),
                    desc,
                    j['user']]

                for i in range(len(fieldwidths)):
                    if len(ll[i]) > fieldwidths[i]:
                        ll[i] = ll[i][0:fieldwidths[i]]

                self.printout(fmt % tuple(ll))
                          
        
class XenRTComplete(XenRTCommand):

    name = "complete"
    summary = "Mark a job as complete (server call only)"
    mandatory = "<jobid>"
    group = "Job"

    def run(self, args):
        if len(args) != 1:
            raise Exception("Invalid usage\n")
        id = string.atoi(args[0])
        self.xenrt.update_job(id, complete=True)

class XenRTEmail(XenRTCommand):

    name = "email"
    summary = "Send job summary email (if necessary) (server call only)"
    mandatory = "<jobid>"
    hide = True

    def run(self, args):
        if len(args) != 1:
            raise Exception("Invalid usage\n")
        id = string.atoi(args[0])
        self.xenrt.send_job_email(id)

class XenRTLogServer(XenRTCommand):
    name = "logserver"
    summary = "Get URL to default log server (server call only)"
    mandatory = ""
    hide = True

    def run(self, args):
        self.printout("%s\n" % self.xenrt.get_logserver())

class XenRTSetResult(XenRTCommand):

    name = "setresult"
    summary = "Set test result (server call only)"
    mandatory = "<jobid> <phase> <test> <result>"
    hide = True

    def run(self, args):
        if len(args) != 4:
            raise Exception("Invalid usage\n")
        self.xenrt.set_result(int(args[0]), args[1], args[2], args[3])

class XenRTEvent(XenRTCommand):

    name = "event"
    summary = "Record an event"
    mandatory = "<type> <subject> '<data>'"
    hide = True
    
    def run(self, args):
        if len(args) != 3:
            raise Exception("Invalid usage\n")
        self.xenrt.new_event(args[0], args[1], args[2])
    
class XenRTLogData(XenRTCommand):

    name = "logdata"
    summary = "Set test data fields (server call only)"
    mandatory = "<jobid> <phase> <test> <key> <value>"
    hide = True
    
    def run(self, args):
        if len(args) != 5:
            raise Exception("Invalid usage\n")
        self.xenrt.new_logdata(int(args[0]), args[1], args[2], args[3], args[4])

class XenRTRemove(XenRTCommand):

    name = "remove"
    summary = "Remove a job from the system"
    mandatory = "<jobid>"
    group = "Job"

    def run(self, args):
        if len(args) == 0:
            raise Exception("Invalid usage\n")
        ids = [int(x) for x in args]
        self.xenrt.remove_jobs(ids)

class XenRTSubmit(XenRTCommand):

    name = "submit"
    summary = "Submit a XenRT run to the work queue"
    usage = XenRTCommand.job_usage
    group = "Job"
    
    def run(self, args, preconf=None):
        defaults = {
            'OPTIONS': 'x86-32',
            'XENRT_PQ_NAME': 'xenrt.pq.hg'
            }
            
        details, secondary = self.parse_options(args)
        if preconf:
            preconf.update(details)
            details = preconf
        defaults.update(details)
        details = defaults
        if not details:
            raise Exception("No parameters specified")
        email = None
        if details.has_key("EMAIL"):
            email = details["EMAIL"]
            del details["EMAIL"]
        if details.has_key("USERID"):
            self.xenrt.customHeaders['X-Fake-User'] = details["USERID"]
            del details["USERID"]
        for key in details.keys():
            if len(details[key]) == 0:
                del details[key]

        job = self.xenrt.new_job(params=details,email=email)
        attachUrl = job['attachmentUploadUrl']

        # Do we have a patch to upload?
        if secondary.has_key("patch"):
            self.upload(attachUrl, secondary["patch"], "patch")
        if secondary.has_key("seqfiles"):
            for sf in string.split(secondary['seqfiles'], ","):
                self.upload(attachUrl, sf, os.path.basename(sf))
        if secondary.has_key("testcasefiles"):
            for tf in string.split(secondary['testcasefiles'], ","):
                self.upload(attachUrl, tf, os.path.basename(tf))
        if secondary.has_key("customupdates"):
            for cu in string.split(secondary['customupdates'], ","):
                self.upload(attachUrl, cu, os.path.basename(cu))
        if secondary.has_key("perfdata"):
            self.upload(attachUrl, secondary['perfdata'], "perfdata")
        if secondary.has_key("perfregress"):
            self.upload(attachUrl, secondary['perfregress'], "perfregress")
        
        self.printout("%d\n" % job['id'])

class XenRTRerun(XenRTSubmit):

    name = "rerun"
    summary = "Rerun a previously existed job"
    mandatory = "<jobid> [faithful|testrun|custom]"
    usage = """
Among the three rerun modes, 'custom' mode is the default choice if none is
present. After the mandatory arguments above, any 'xenrt submit' option can
be used to shadow the original setttings in any mode. Note that it might not
make sense to overwrite settings in the 'faithful' and 'testrun' mode, but we
preserve such possibilities.
    """

    faithful_ex = [ "XMLRPC", "HARNESS_PID", "JOBID", "RUNDIR", "XENRT_VERSION",
                    "JOB_SUBMITTED", "STARTED", "JOBSTATUS", "SCHEDULEDON.*",
                    "CHECK", "RETURN", "UPLOADED", "FINISHED", "PASSRATES",
                    "REGRESSION", "REMOVED", "PREPARE_FAILED" ]
    testrun_ex = [ "MACHINE", "PXE_BIOS_BOOT_RETRY" ]
    custom_ex = [ "JOBGROUP", "TESTRUN.*", "JIRA.*", "USERID",
                  "EMAIL.*", ".*EMAIL_TO", "AUTO_BUG.*" ]

    def run(self, args):
        if len(args) < 1:
            raise Exception("Invalid usage\n")
        jobid = args[0]
        options = self.xenrt.get_job(int(jobid))['params']
        if len(args) < 2:
            mode = 'custom'
            noptions = []
        elif args[1].startswith('-'):
            mode = 'custom'
            noptions = args[1:]
        elif args[1] in ['faithful', 'custom']:
            mode = args[1]
            noptions = args[2:]
        else:
            raise Exception("Invalid rerun mode\n")
        # Exclude patterns
        excludes = []
        # Include options dict
        includes = {}
        if mode == 'faithful':
            excludes += self.faithful_ex
        if mode == 'custom':
            excludes += self.faithful_ex + self.testrun_ex + self.custom_ex
        excludes = map(lambda s: '^' + s + '$', excludes)
        options=dict(filter(lambda (k,v): not (re.match("|".join(excludes), k)), options.iteritems()))
        options.update(includes)
        options['ORIGINAL_JOBID'] = jobid
        XenRTSubmit.run(self, noptions, preconf=options)

class XenRTUpdate(XenRTCommand):

    name = "update"
    summary = "set/update a field in a job description"
    mandatory = "<jobid> <field> <value>"
    group = "Job"
    
    def run(self, args):
        if len(args) != 3:
            raise Exception("Invalid usage\n")
        id = string.atoi(args[0])
        self.xenrt.update_job(id, {args[1]: args[2]})

class XenRTMBroken(XenRTCommand):
    name = "mbroken"
    summary = "Mark a machine as broken"
    mandatory = "<machine> <reason>"
    group = "Machine"

    def run(self, args):
        if len(args) < 2:
            raise Exception("Invalid usage\n")
        machine = args[0]
        reason = " ".join(args[1:])

        m = re.search("([A-Z]+-\d+)", reason)
        if m:
            ticket = m.group(1)
        else:
            ticket = ""

        self.xenrt.update_machine(machine, broken = {"broken": True, "info": reason, "ticket": ticket})

class XenRTMFixed(XenRTCommand):
    name = "mfixed"
    summary = "Mark a machine as fixed"
    mandatory = "<machine>"
    group = "Machine"

    def run(self, args):
        if len(args) != 1:
            raise Exception("Invalid usage\n")
        machine = args[0]
        self.xenrt.update_machine(machine, broken = {"broken": False})


class XenRTMUpdate(XenRTCommand):

    name = "mupdate"
    summary = "set/update a field of machine data"
    mandatory = "<machine> <field> <value>"
    group = "Machine"

    def run(self, args):
        if len(args) != 3:
            raise Exception("Invalid usage\n")
        machine = args[0]
        self.xenrt.update_machine(machine, params={args[1]: args[2]})

class XenRTMRes(XenRTMUpdate):
    name = "mres"
    summary = "set/update a resource on a machine"
    mandatory = "<machine> <resource>"
    group = "Machine"

    def run(self, args):
        if len(args) != 2:
            raise Exception("Invalid usage\n")
        if len(args[1].split("=", 1)) != 2:
            raise Exception("Invalid usage\n")
        machine = args[0]
        (resource, value) = args[1].split("=", 1)
        self.xenrt.update_machine(machine, resources={resource: value})

class XenRTMStatus(XenRTCommand):

    name = "mstatus"
    summary = "update the status field of machine data"
    mandatory = "<machine> <status>"
    group = "Machine"

    def run(self, args):
        if len(args) != 2:
            raise Exception("Invalid usage\n")

        machine = args[0]
        self.xenrt.update_machine(machine, status=args[1])

class XenRTBorrow(XenRTCommand):

    name = "borrow"
    summary = "Borrow a machine"
    mandatory = "<machine>"
    usage = """
    -h <hours>     Number of hours to borrow for (default is 24)
    -d <days>      Number of days to borrow for (default is 1)
    -r <reason>    Reason for borrowing machine
    -b             Best effort borrow (borrow for as long as long as policy allows, don't fail if can't be borrowed)
    -p             Borrow on a preemptable basis - can be taken back for scheduled testing (ACL policy dependent)
    """
    group = "Machine"

    def run(self, args):
        if len(args) == 0:
            raise Exception("Invalid usage\n")
        machine = args[0]
        duration = 24
        reason = ""
        force = False
        user = None
        besteffort = False
        preemptable = False
        adminoverride = False
        try:
            optlist, optx = getopt.getopt(args[1:], 'h:u:d:fr:bpa', "force")
            for argpair in optlist:
                (flag, value) = argpair
                if flag == '-h':
                    duration = int(value)
                elif flag == '-d':
                    duration = int(value) * 24
                elif flag == "-r":
                    reason = value
                elif flag == '-u':
                    user = value
                elif flag == "-f":
                    duration = 0
                elif flag == "--force":
                    force = True
                elif flag == "-b":
                    besteffort = True
                elif flag == "-p":
                    preemptable = True
                elif flag == "-a":
                    adminoverride = True
        except getopt.GetoptError:
            raise Exception("Unknown argument\n")
        if user:
            self.xenrt.customHeaders['X-Fake-User'] = user

        self.xenrt.lease_machine(machine, duration, reason, force=force, besteffort=besteffort, preemptable=preemptable, admin_override=adminoverride)

class XenRTReturn(XenRTCommand):

    name = "return"
    summary = "Return a borrowed machine"
    mandatory = "<machine>"
    group = "Machine"

    def run(self, args):
        if len(args) == 0:
            raise Exception("Invalid usage\n")
        machine = args[0]
        force = False
        try:
            optlist, optx = getopt.getopt(args[1:], '', "force")
            for argpair in optlist:
                (flag, value) = argpair
                if flag == "--force":
                    force = True
        except getopt.GetoptError:
            raise Exception("Unknown argument\n")
        self.xenrt.return_leased_machine(machine, force=force)

class XenRTProp(XenRTCommand):

    name = "prop"
    summary = "Add/remove dynamic machine properties"
    mandatory = "<machine> add|del <property>"
    group = "Machine"

    def run(self, args):
        if len(args) != 3:
            raise Exception("Invalid usage\n")
        machine = args[0]
        details = {}
        if args[1] == "add":
            details["addflags"] = [args[2]]
        elif args[1] == "del" or args[1] == "rem":
            details["delflags"] = [args[2]]
        else:
            raise Exception("Invalid usage: need \"add\" or \"del\".\n")
        self.xenrt.update_machine(machine, **details)

class XenRTSFlag(XenRTCommand):

    name = "sflag"
    summary = "Add/remove site flags"
    mandatory = "<site> add|del <property>"
    group = "Site"

    def run(self, args):
        if len(args) != 3:
            raise Exception("Invalid usage\n")
        machine = args[0]
        details = {}
        if args[1] == "add":
            details["addflags"] = [args[2]]
        elif args[1] == "del" or args[1] == "rem":
            details["delflags"] = [args[2]]
        else:
            raise Exception("Invalid usage: need \"add\" or \"del\".\n")
        self.xenrt.update_site(machine, **details)

class XenRTPool(XenRTCommand):

    name = "pool"
    summary = "Set machine pool"
    mandatory = "<machine> [<poolname>]"
    usage = """
The first usage is for setting the pool a machine is in. A blank <poolname>
moves the machine to the "DEFAULT" pool.
    """
    group = "Machine"

    def run(self, args):
        if len(args) < 1:
            raise Exception("Invalid usage\n")            

        # Machine update
        machine = args[0]
        pool = "DEFAULT"
        if len(args) > 1:
            pool = args[1]
        self.xenrt.update_machine(machine, params={"POOL": pool})

class XenRTDetailID(XenRTCommand):

    name = "detailid"
    summary = "Get the detailid for a specified jobid, phase and test"
    mandatory = "<jobid> <phase> <test>"
    hide = True

    def run(self, args):
        if len(args) <> 3:
            raise Exception("Invalid usage\n")
        jobid = int(args[0])
        phase = args[1]
        test = args[2]

        job = self.xenrt.get_job(jobid)
        detailids = [x['detailid'] for x in job['results'].values() if x['phase'] == phase and x['test'] == test]
        if not detailids:
            raise Exception("Test not found")

        self.printout("%d\n" % detailids[0])

class XenRTHelp(XenRTCommand):

    name = "help"
    summary = "Get a summary of commands"
    group = "Miscellaneous"

    def run(self, args):
        cx = {}
        for c in self.commands.getall():
            if not c.hide:
                if not cx.has_key(c.group):
                    cx[c.group] = []
                cx[c.group].append(c)
        first = ["Job", "Suite", "Machine", "Site"]
        last = ["Miscellaneous", "Other"]
        order = []
        order.extend(first)
        for g in cx.keys():
            if not g in first and not g in last:
                order.append(g)
        order.extend(last)
        for g in order:
            self.printout("\n%s commands:\n\n" % (g))
            for c in cx[g]:
                self.printout("  %-10s %s\n" % (c.name, c.summary))
        self.printout("\nRun \"%s <command> --help\" for usage information.\n"
                      % (sys.argv[0]))        

class XenRTSubResults(XenRTCommand):

    name = "subresults"
    summary = "Upload a XML subresults file for a job (server call only)"
    mandatory = "<jobid> <phase> <test>"
    usage = """
    [ -f <filename> ]         Filename to upload (default is STDIN)
    """
    hide = True

    def run(self, args):
        if len(args) < 3:
            raise Exception("Invalid usage\n")
        filename = None
        try:
            optlist, optx = getopt.getopt(args[3:], 'f:')
            for argpair in optlist:
                (flag, value) = argpair
                if flag == '-f':
                    filename = value
        except getopt.GetoptError:
            raise Exception("Unknown argument\n")
        self.xenrt.upload_subresults(int(args[0]), args[1], args[2], filename)

class XenRTPerfData(XenRTCommand):

    name = "perfdata"
    summary = "Upload a XML performance data file (server call only)"
    usage = """
    [ -f <filename> ]         Filename to upload (default is STDIN)
    """
    hide = True

    def run(self, args):
        filename = None
        try:
            optlist, optx = getopt.getopt(args, 'f:')
            for argpair in optlist:
                (flag, value) = argpair
                if flag == '-f':
                    filename = value
        except getopt.GetoptError:
            raise Exception("Unknown argument\n")
        self.xenrt.upload_perfdata(filename)

class XenRTUpload(XenRTCommand):

    name = "upload"
    summary = "Upload a results file for a job (server call only)"
    mandatory = "<jobid>"
    usage = """
    [ -p <phase> -t <test> ]  Phase and test for per-test results file.
    [ -f <filename> ]         Filename to upload (default is STDIN)
    """
    hide = True

    def run(self, args):
        details = {'action': 'upload'}
        if len(args) < 1:
            raise Exception("Invalid usage\n")
        id = string.atoi(args[0])
        filename = None
        prefix = None
        phase = None
        test = None
        try:
            optlist, optx = getopt.getopt(args[1:], 'p:t:f:P:')
            for argpair in optlist:
                (flag, value) = argpair
                if flag == '-p':
                    phase = value
                if flag == '-t':
                    test = value
                if flag == '-f':
                    filename = value
                if flag == '-P':
                    prefix = value
        except getopt.GetoptError:
            raise Exception("Unknown argument\n")

        job = self.xenrt.get_job(id) 
        remoteFn = None
        localFn = None

        if filename:
            localFn = filename

        if prefix:
            url = job['attachmentUploadUrl']
            remoteFn = prefix
        elif phase and test:
            url = [x['logUploadUrl'] for x in job['results'].values() if x['phase'] == phase and x['test'] == test][0]
        else:
            url = job['logUploadUrl']
        return self.upload(url, localFn, remoteFn) 

class XenRTDownload(XenRTCommand):

    name = "download"
    summary = "Download a results file for a job"
    mandatory = "<jobid>"
    usage = """
    [ -p prefix ]   Prefix for file to download.
    [ -f filename ] File to write results to (defaults to stdout)
    [ -g phase ]    Optional phase (test also required)
    [ -t test ]     Optional test (phase also required)
    """
    group = "Miscellaneous"

    def run(self, args):
        filename = None
        if len(args) < 1:
            raise Exception("Invalid usage\n")
        id = string.atoi(args[0])
        prerun = False
        prefix = None
        phase = None
        test = None
        try:
            optlist, optx = getopt.getopt(args[1:], 'op:f:g:t:')
            for argpair in optlist:
                (flag, value) = argpair
                if flag == '-p':
                    prefix = value
                elif flag == "-f":
                    filename = value
                elif flag == "-g":
                    phase = value
                elif flag == "-t":
                    test = value
                elif flag == "-o":
                    prerun = True
        except getopt.GetoptError:
            raise Exception("Unknown argument\n")

        if prefix:
            if prerun:
                url = self.xenrt.get_job_attachment_pre_run(id, prefix)
            else:
                url = self.xenrt.get_job_attachment_post_run(id, prefix)
        elif phase and test:
            job = self.xenrt.get_job(id)
            urls = [x['logUrl'] for x in job['results'].values() if x['phase'] == phase and x['test'] == test]
            if not urls:
                raise Exception("Phase/test not found for job")
            elif not urls[0]:
                raise Exception("Log has not been uploaded for test")
            else:
                url = urls[0]
        else:
            job = self.xenrt.get_job(id)
            url = job['logUrl']

        if filename:
            f = open(filename, "wb")
        else:
            f = sys.stdout
        try:
            r = requests.get(url, stream=True)
            r.raise_for_status()
            for chunk in r.iter_content(512):
                f.write(chunk)
        finally:
            if filename:
                f.close()

class XenRTSchedule(XenRTCommand):

    name = "schedule"
    summary = "No longer supported"
    usage = ""
    group = "Scheduler"
    
    def run(self, args):
        self.printout("On demand schedule no longer supported - performed in background only\n")

class XenRTMDefine(XenRTCommand):

    name = "mdefine"
    summary = "Define a new test machine"
    mandatory = "<machine> <site>"
    usage = """
    -c <cluster>
    -p <pool>
    -r <resourcestring>
    -f <flags>
    -d <description>
    """
    group = "Machine"

    def run(self, args):
        if len(args) < 2:
            raise Exception("Invalid usage\n")
        machine = args[0]
        site = args[1]
        details = {}
        try:
            optlist, optx = getopt.getopt(args[2:], 'c:p:r:f:d:')
            for argpair in optlist:
                (flag, value) = argpair
                if flag == '-c':
                    details["cluster"] = value
                if flag == '-p':
                    details["pool"] = value
                if flag == '-r':
                    details["resources"] = dict([y for y in [x.split("=",1) for x in value.split("/")]])
                if flag == '-f':
                    details["flags"] = value.split(",")
                if flag == '-d':
                    details["description"] = value
        except getopt.GetoptError:
            raise Exception("Unknown argument\n")
        self.xenrt.new_machine(machine, site, **details)

class XenRTMUndefine(XenRTCommand):

    name = "mundefine"
    summary = "Remove a machine from the system"
    mandatory = "<machine>"
    group = "Machine"

    def run(self, args):
        if len(args) != 1:
            raise Exception("Invalid usage\n")
        self.xenrt.remove_machine(args[0])

class XenRTInteract(XenRTCommand):

    name = "interact"
    summary = "Interact with a running job"
    mandatory = "{<jobid>|<host:port>}"
    usage = """
    [-l]               List running test cases
    [-c] <testcase>    Continue a paused testcase
    [-u] <testcase>    Continue a paused testcase without blocking on failure
    [-n] <testcase>    Abort a paused testcase
    [-x] <testcase>    Remove the automatic 24 hours unpause of a testcase
    [-C]               Continue any paused testcases
    [-N]               Abort any paused testcases
    [-R] <tcase>=<res> Set a testcase result (PASS|FAIL|ERROR|SKIPPED)
    [-D] <var>=<value> Set a config variable (var may be a /-separated split)
    [-d]               Dump config
    [-A]               Abort a run (after the current tests have completed)
    [-f]               Flush harness.err and harness.out log buffers
    [-I]               Display information about the hosts and guest
    [-r] <guest>:<filename>
                       Read a file from a named guest
    [-s]               Interactive shell
    [-S]               Print the stack trace
    [-L]               Show last 1000 log output lines
    """
    group = "Job"

    def run(self, args):

        if len(args) == 0:
            raise Exception("Invalid usage\n")
        if string.find(args[0], ":") > -1:
            hostport = args[0]
        else:
            try:
                id = string.atoi(args[0])
            except:
                id = self.xenrt.get_machine(args[0])['jobid']
            
            job = self.xenrt.get_job(id)
            if not "XMLRPC" in job['params']:
                raise Exception("Job has no XMLRPC variable\n")
            hostport = job['params']["XMLRPC"]
        (host, port) = hostport.split(":")
        s = xmlrpclib.Server("http://%s/xmlrpc/%s" % (host,port))
        shell = False
        logger = False
        stack = False
        try:
            optlist, optx = getopt.getopt(args[1:], 'lc:CD:dAfIr:sLR:n:Nu:x:S')
            for argpair in optlist:
                (flag, value) = argpair
                if flag == '-l':
                    for t in s.getRunningTests():
                        print "%-30s %s" % (t[0], t[1])
                elif flag == '-c':
                    s.setRunningStatus(value, "Continue")
                elif flag == '-u':
                    s.setBlockingStatus(value, False)
                    s.setRunningStatus(value, "Continue")
                elif flag == '-n':
                    s.setRunningStatus(value, "NotContinue")
                elif flag == '-x':
                    s.setRunningStatus(value, "Indefinite")
                elif flag == '-R':
                    l = string.split(value, '=')
                    testcase = l[0]
                    if l[1] == 'PASS':
                        result = 1
                    elif l[1] == 'FAIL':
                        result = 2
                    elif l[1] == 'ERROR':
                        result = 4
                    elif l[1] == 'SKIPPED':
                        result = 5
                    else:
                        raise Exception("Unknown result '%s'" % (l[1]))
                    s.setTestResult(testcase, result)
                elif flag == '-C':
                    for t in s.getRunningTests():
                        if t[1] == "Paused":
                            s.setRunningStatus(t[0], "Continue")
                elif flag == '-N':
                    for t in s.getRunningTests():
                        if t[1] == "Paused":
                            s.setRunningStatus(t[0], "NotContinue")
                elif flag == "-D":
                    try:
                        var, varval = string.split(value, "=", 1)
                        if string.find(var, "/") > -1:
                            var = string.split(var, "/")
                        s.setConfigVariable(var, varval)
                    except:
                        raise Exception("Error parsing -D variable '%s'\n" %
                                         (value))
                elif flag == "-d":
                    print s.dumpConfig()
                elif flag == "-A":
                    s.abortRun()
                elif flag == "-f":
                    s.flushLogs()
                elif flag == "-I":
                    hosts = s.getHostList()
                    for h in hosts:
                        info = s.getHostInfo(h)
                        print "Host: %s (%s)" % (info[0], info[1])
                    guests = s.getGuestList()
                    for g in guests:
                        info = s.getGuestInfo(g)
                        print "Guest: %s (%s) uuid=%s" % \
                              (info[0], info[1], info[5])
                elif flag == "-r":
                    ll = string.split(value, ":", 1)
                    if len(ll) != 2:
                        raise Exception("Error parsing -r variable '%s'\n" %
                                         (value))
                    data = s.getGuestFile(ll[0], ll[1])
                    sys.stdout.write(data.data)
                elif flag == "-s":
                    shell = True
                elif flag == "-L":
                    logger = True
                elif flag == "-S":
                    stack = True
                    
        except getopt.GetoptError:
            raise Exception("Unknown argument\n")

        if stack:
            loaded = json.loads(s.xmlrpcShell("json.dumps([[[y.name for y in threading.enumerate() if y.ident==x[0]],x[0],traceback.extract_stack(x[1])] for x in sys._current_frames().items()])"))

            for ll in loaded:
                (tname, tident, traceback) = ll
                if tname:
                    tname = tname[0]
                else:
                    tname = "Thread-%d" % tident
                print tname
                for t in traceback:
                    print "    %s:%d - %s %s" % (t[0], t[1], t[2], t[3])

        if shell:
            print "XenRT interactive Python shell."
            import readline
            while True:
                try:
                    command = raw_input(">>> ")
                    print s.xmlrpcShell(command)
                except EOFError, e:
                    print ""
                    break

        if logger:
            print string.join(s.xmlrpcLogger(), '')

class XenRTSChildren(XenRTCommand):
    name = "schildren"
    summary = "Get a list of child sites for a site"
    group = "Site"
    mandatory = "<site>"

    def run(self, args):
        sites = self.xenrt.get_sites(parent = args[0])
        for site in sites.keys():
            self.printout("%s\n" % site)

class XenRTSList(XenRTCommand):

    name = "slist"
    summary = "Get a list of sites"
    usage = """
    [-p]             Show site properties
    [-q]             Quiet mode (don't show header)
    """
    group = "Site"

    def run(self, args):
        props = False
        quiet = False
        try:
            optlist, optx = getopt.getopt(args, 'pq')
            for argpair in optlist:
                (flag, value) = argpair
                if flag == "-q":
                    quiet = True
                elif flag == "-p":
                    props = True
                else:
                    raise Exception("Unknown argument '%s'\n" % (flag))
        except getopt.GetoptError:
            raise Exception("Unknown argument\n")

        sites = self.xenrt.get_sites()

        if props:
            chead = "Properties"
        else:
            chead = "Description"

        fmt = "%-8s %-7s %s\n"
        if not quiet:
            self.printout(fmt % ("Site", "Status", chead))
            self.printout("============================================================================\n")
        
        for s in sorted(sites.keys()):
            item = ""
            if props:
                item = ",".join(sites[s]['flags'])
            else:
                item = sites[s]['description']
            self.printout(fmt % (s, sites[s]['status'], item))

class XenRTSDefine(XenRTCommand):

    name = "sdefine"
    summary = "Define a new site"
    mandatory = "<site>"
    usage = """
    -f <flags>
    -d <description>
    """
    group = "Site"

    def run(self, args):
        if len(args) < 1:
            raise Exception("Invalid usage\n")
        details = {}
        try:
            optlist, optx = getopt.getopt(args[1:], 'f:d:')
            for argpair in optlist:
                (flag, value) = argpair
                if flag == '-f':
                    details["flags"] = value.split(",")
                elif flag == '-d':
                    details["description"] = value
        except getopt.GetoptError:
            raise Exception("Unknown argument\n")
        
        self.xenrt.new_site(args[0], **details)

class XenRTSUndefine(XenRTCommand):

    name = "sundefine"
    summary = "Remove a site from the system"
    mandatory = "<site>"
    group = "Site"

    def run(self, args):
        if len(args) != 1:
            raise Exception("Invalid usage\n")
        self.xenrt.remove_site(args[0])

class XenRTSUpdate(XenRTCommand):

    name = "supdate"
    summary = "set/update a field of site data"
    mandatory = "<site> <field> <value>"
    group = "Site"

    def run(self, args):
        details = {'action': 'supdate'}
        if len(args) != 3:
            raise Exception("Invalid usage\n")
        site = args[0]
        
        key = args[1].lower()
        value = args[2]
        if key == "flags":
            value = value.split(",")
        elif key == "maxjobs":
            value = int(value)
        elif key == "sharedresources":
            value = dict([y for y in [x.split("=",1) for x in value.split("/")]])
        details = {key: value}
        self.xenrt.update_site(site, **details)

class XenRTSite(XenRTCommand):

    name = "site"
    summary = "Get data for the specified site"
    mandatory = "<site>"
    group = "Site"

    def run(self, args):
        if len(args) != 1:
            raise Exception("Invalid usage\n")
        site = self.xenrt.get_site(args[0])
        
        self.printout("SITE='%s'\n" % site['name'])
        self.printout("STATUS='%s'\n" % site['status'])
        self.printout("DESCRIPTION='%s'\n" % site['description'])
        if site['ctrladdr']:
            self.printout("CTRLADDR='%s'\n" % site['ctrladdr'])
        self.printout("FLAGS='%s'\n" % ",".join(site['flags']))
        if site['maxjobs']:
            self.printout("MAXJOBS='%d'\n" % site['maxjobs'])
        if site['sharedresources']:
            self.printout("SHAREDRESOURCES='%s'\n" % "/".join(["%s=%s" % (x,y) for (x,y) in site['sharedresources'].iteritems()]))
        if site['location']:
            self.printout("LOCATION='%s'\n" % site['location'])
        if site['parent']:
            self.printout("PARENT='%s'\n" % site['parent'])
        # TODO Add availableresources


class XenRTRunSuite(XenRTCommand):

    name = "runsuite"
    summary = "Run a test suite on a specified revision"
    mandatory = "<suite> <revision>"
    usage = """
    [-b <branch>]     Branch to test
    [-s <sku>]        SKU to test
    [-S <seqs>]       Suite sequences to run
    [-x <xenrt_branch>] XenRT branch to use to submit suite (note that XRTBRANCH also needs to be set in the suite)
    [-R]              Attempt to rerun if suite already exists
    """
    group = "Suite"

    def run(self, args):
        data = {}
        data['suite'] = args[0]
        data['version'] = args[1]
        data['params'] = {}
        if len(args) > 2:
            optlist, optx = getopt.getopt(args[2:], 'b:s:D:S:RVx:', ['devrun'])
            for argpair in optlist:
                (flag, value) = argpair
                if flag == "-b":
                    data['branch'] = value
                elif flag == "-x":
                    data['xenrtbranch'] = value
                elif flag == "-s":
                    data['sku'] = "%s" % value.replace(".sku", "")
                elif flag == "-S":
                    data['seqs'] = value.split(",")
                elif flag == "-D":
                    (k, v) = value.split("=", 2)
                    data['params'][k] = v
                elif flag == "-V":
                    self.verbose = True
                elif flag == "-R":
                    data['rerunifneeded'] = True
                elif flag == "--devrun":
                    data['devrun'] = True
        
        token = self.xenrt.start_suite_run(**data)['token']

        self.printout("Got token %s\n" % token)
        
        info = self.xenrt.get_start_suite_run_status(token)
        console = info['console']
        self.printout("Console log at %s\n" % console)
        i = 0
        while info['status'] == "running":
            self.printout("Suite is starting\n")
            if i > 30:
                raise Exception("Starting suite timed out - see %s for more information" % console)
            i += 1
            info = self.xenrt.get_start_suite_run_status(token)
            time.sleep(5)

        if info['status'] != "success":
            raise Exception("Error starting suite - see %s for more information" % console)
            
        reply = ["SR%d\n" % info['suiterun']]
        for j in info['jobs'].keys():
            reply.append("%s:%s\n" % (j, info['jobs'][j]))
        for r in reply:
            self.printout(r)

class _XenRTControllerOperation(XenRTCommand):
    def getSiteDetailsForMachine(self,machine,args):
        site = None
        self.overrideconsole = None
        try:
            optlist, optx = getopt.getopt(args, 's:',['override-console='])
            for argpair in optlist:
                (flag, value) = argpair
                if flag == "-s":
                    site = value
                elif flag == "--override-console":
                    self.overrideconsole = value
                else:
                    raise Exception("Unknown argument '%s'\n" % (flag))
        except getopt.GetoptError:
            raise Exception("Unknown argument\n")
        if not site:
            site = self.xenrt.get_machine(machine)['site']
        return self.xenrt.get_site(site)

class XenRTSNetwork(_XenRTControllerOperation):
    name = "snetwork"
    summary = "Show network info for a site"
    mandatory = "<site>"
    group = "Site"
    
    def run(self, args):
        if len(args) < 1:
            raise Exception("Invalid usage\n")
        site = args[0]

        site = self.xenrt.get_site(site)
        details = {}
        if site['parent']:
            details['vsite'] = site['name']
        r = requests.get("http://%s/xenrt/api/controller/network" % site['ctrladdr'], params=details)
        r.raise_for_status()
        self.printout(r.text)

class XenRTMConfig(_XenRTControllerOperation):
    name = "mconfig"
    summary = "See config for a machine"
    mandatory = "<machine>"
    usage = """
    [ --generated ] See the generated config looked up from RackTables
    """
    group = "Machine"

    def run(self, args):
        details = {}
        if len(args) == 0:
            raise Exception("Invalid usage\n")
        machine = args[0]
        details["machine"] = machine
        try:
            optlist, optx = getopt.getopt(args[1:], '', "generated")
            for argpair in optlist:
                (flag, value) = argpair
                if flag == '--generated':
                    details["generated"] = "yes"
        except getopt.GetoptError:
            raise Exception("Unknown argument\n")
        site = self.getSiteDetailsForMachine(machine, [])
        if site['parent']:
            details['vsite'] = site['name']

        r = requests.get("http://%s/xenrt/api/controller/machinecfg" % site['ctrladdr'], params=details)
        r.raise_for_status()
        self.printout(r.text)
        
class XenRTInstall(XenRTCommand):
    name = "install"
    summary = "Install a host"
    mandatory = "<machine>"
    usage = """
    --inputs <inputdir>
    -D <param>=<value>
    """
    group = "Machine"

    def run(self, args):
        if len(args) < 1:
            raise Exception("Invalid usage\n")
        machine = args[0]
        kwargs={"params":{}}
        try:
            optlist, optx = getopt.getopt(args[1:], 'D:', ["inputs="])
            for argpair in optlist:
                (flag, value) = argpair
                if flag == "-D":
                    (param, pvalue) = value.split("=", 1)
                    kwargs["params"][param] = pvalue
                elif flag == "--inputs":
                    kwargs["inputdir"] = value
                else:
                    raise Exception("Unknown argument '%s'\n" % (flag))
        except getopt.GetoptError, e:
            raise Exception("Unknown argument: %s\n" % (str(e)))
        
        job = self.xenrt.new_job(
            deployment = {"hosts":[{"id": 0}]},
            specified_machines = [machine],
            **kwargs)
        self.printout("%d\n" % job['id'])


class XenRTPower(XenRTCommand):

    name = "power"
    summary = "Power control a machine"
    mandatory = "<machine> off|on|reboot|nmi"
    usage = """
    --bootdev <device>      IPMI boot device
    --force                 Ignore lease/running state of machine
    """
    group = "Machine"

    def run(self, args):
        if len(args) < 2:
            raise Exception("Invalid usage\n")
        machine = args[0]
        operation = args[1]
        details = {}
        try:
            optlist, optx = getopt.getopt(args[2:], 'a', ["force", "bootdev"])
            for argpair in optlist:
                (flag, value) = argpair
                if flag == "--force":
                    details['force'] = True
                elif flag == "--bootdev":
                    details['bootdev'] = value
                elif flag == "-a":
                    details['admin_override'] = True
        except getopt.GetoptError:
            raise Exception("Unknown argument\n")
        if operation not in ["off", "on", "reboot", "nmi"]:
            raise Exception("Invalid power operation - must be one of off, on or reboot\n")
        result = self.xenrt.power_machine(machine, operation, **details)
        self.printout("%s\n" % result['output'])

class XenRTConsole(_XenRTControllerOperation):
    name = "console"
    summary = "Serial console for a machine"
    mandatory = "<machine>"
    group = "Machine"

    def run(self, args):
        if len(args) < 1:
            raise Exception("Invalid usage\n")
        global _console
        global _conserver
        results = None
        console = os.popen("which console").read().strip()
        if console == "":
            console = _console
        machine = args[0]
        if machine == "-l":
            machines = xenrt.get_machines().keys()

            results = []

            for l in os.popen("%s -p 3109 -U -M %s -u" % (console, _conserver)).readlines():
                m = re.match(" (.+?)\s+(.+?)\s+(.+?)", l)
                if m:
                    if not m.group(1) in machines:
                        self.printout("%s\n" % m.group(1))
        else:
            sitedetails = self.getSiteDetailsForMachine(machine, args[1:])
            addr = sitedetails.get("ctrladdr")
            if not addr:
                addr = _conserver
            if self.overrideconsole:
                console = self.overrideconsole
            if os.environ.get("SSH_CONSOLE") != "yes" and not "sshconsole" in sitedetails['flags']:
                os.system("%s -p 3109 -U -M %s %s" % (console, addr, machine.split(".")[0].split("_")[0]))
            else:
                out = re.sub("(\S{64}) ", "\\1\n", _conskey)
                out = re.sub("(-{5}) ", "\\1\n", out)
                out = re.sub(" (-{5})", "\n\\1", out)
                f = tempfile.NamedTemporaryFile()
                f.file.write(out)
                f.file.close()
                os.system("ssh -t -i %s -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o CheckHostIP=no cons@%s %s" % (f.name, addr, machine.split(".")[0].split("_")[0]))

class XenRTGlobalResLock(XenRTCommand):
   
    name = "globalreslock"
    summary = "Lock a global resource (controller command)"
    mandatory = "<type> <jobid> <site>"
    group = "resources"

    def run(self, args):
        data = self.xenrt.lock_global_resource(args[0], args[2], int(args[1]))
        self.printout(json.dumps(data))


class XenRTGlobalResRelease(XenRTCommand):
    name = "globalresrelease"
    summary = "Release a global resource"
    mandatory = "<job id or resource name>"
    group = "resources"

    def run(self, args):
        details = {}
        try:
            details['job'] = int(args[0])
        except:
            details['name'] = args[0]
        self.xenrt.release_global_resource(**details)

class XenRTSanityCheck(XenRTCommand):
    name = "sanity"
    summary = "Sanity check for the CLI"
    hide = True
    usesAPI = False

    def run(self, args):
        self.printout("Sanity check passed")

class XenRTMGetResource(_XenRTControllerOperation):
    name = "mgetresource"
    summary = "Get a controller resource for a machine"
    mandatory = "<machine> <resource type> [<args>]"

    def run(self, args):
        if len(args) < 2:
            raise Exception("Invalid usage\n")
        machine = args[0]
        site = self.getSiteDetailsForMachine(machine, args[2:])
        if not site['ctrladdr']:
            raise Exception("Controller for machine %s not found\n" % machine)
        details = {"machine": args[0], "type": args[1]}
        if site['parent']:
            details['vsite'] = site['name']
        if len(args) > 2:
            details['args'] = " ".join(args[2:])
        r = requests.get("http://%s/xenrt/api/controller/getresource" % site['ctrladdr'], params=details)
        r.raise_for_status()
        self.printout(r.text)

class XenRTMListResources(_XenRTControllerOperation):
    name = "mlistresources"
    summary = "List controller resources for a machine"
    mandatory = "<machine>"

    def run(self, args):
        if len(args) < 1:
            raise Exception("Invalid usage\n")
        machine = args[0]
        site = self.getSiteDetailsForMachine(machine, args[2:])
        if not site['ctrladdr']:
            raise Exception("Controller for machine %s not found\n" % machine)
        details = {"machine": args[0]}
        if site['parent']:
            details['vsite'] = site['name']
        r = requests.get("http://%s/xenrt/api/controller/listresources" % site['ctrladdr'], params=details)
        r.raise_for_status()
        self.printout(r.text)

class XenRTMReleaseResources(_XenRTControllerOperation):
    name = "mreleaseresources"
    summary = "Relase controller resources"
    mandatory = "<machine> [<resource1> <resource2>...]"

    def run(self, args):
        if len(args) < 2:
            raise Exception("Invalid usage\n")
        machine = args[0]
        site = self.getSiteDetailsForMachine(machine, args[2:])
        if not site['ctrladdr']:
            raise Exception("Controller for machine %s not found\n" % machine)
        details = {"resource": []}
        if site['parent']:
            details['vsite'] = site['name']
        for r in args[1:]:
            details['resource'].append(r)

        r = requests.get("http://%s/xenrt/api/controller/releaseresources" % site['ctrladdr'], params=details)
        r.raise_for_status()
        self.printout(r.text)

#############################################################################

class Commands:

    def __init__(self):
        self.commands = {}
        self.add(XenRTSubmit)
        self.add(XenRTStatus)
        self.add(XenRTHelp)
        self.add(XenRTComplete)
        self.add(XenRTEmail)
        self.add(XenRTList)
        self.add(XenRTUpdate)
        self.add(XenRTUpload)
        self.add(XenRTDownload)
        self.add(XenRTRemove)
        self.add(XenRTSetResult)
        self.add(XenRTLogData)
        self.add(XenRTShowLog)
        self.add(XenRTMachine)
        self.add(XenRTMList)
        self.add(XenRTMConfig)
        self.add(XenRTMList2)
        self.add(XenRTMUpdate)
        self.add(XenRTBorrow)
        self.add(XenRTReturn)
        self.add(XenRTEvent)
        self.add(XenRTPool)
        self.add(XenRTProp)
        self.add(XenRTMRes)
        self.add(XenRTMBroken)
        self.add(XenRTMFixed)
        self.add(XenRTSFlag)
        self.add(XenRTMStatus)
        self.add(XenRTSchedule)
        self.add(XenRTMDefine)
        self.add(XenRTMUndefine)
        self.add(XenRTSubResults)
        self.add(XenRTInteract)
        self.add(XenRTPerfData)
        self.add(XenRTDetailID)
        self.add(XenRTSChildren)
        self.add(XenRTSList)
        self.add(XenRTSDefine)
        self.add(XenRTSUndefine)
        self.add(XenRTSUpdate)
        self.add(XenRTSite)
        self.add(XenRTRunSuite)
        self.add(XenRTRerun)
        self.add(XenRTInstall)
        self.add(XenRTPower)
        self.add(XenRTSNetwork)
        self.add(XenRTConsole)
        self.add(XenRTGlobalResLock)
        self.add(XenRTGlobalResRelease)
        self.add(XenRTMGetResource)
        self.add(XenRTMListResources)
        self.add(XenRTMReleaseResources)
        self.add(XenRTLogServer)
        self.add(XenRTSanityCheck)

    def add(self, cl):
        c = cl(self)
        self.commands[c.name] = c

    def has(self, command):
        return self.commands.has_key(command)

    def run(self, command, args):
        return self.commands[command].dispatch(args)

    def getall(self):
        return self.commands.values()

#############################################################################

def xenrt_profile_stats():
    global XRTPROF_FILENAME
    global XRTPROF

    XRTPROF.close()
    
    stats = hotshot.stats.load(XRTPROF_FILENAME)
    stats.strip_dirs()
    stats.sort_stats('time', 'calls')
    stats.print_stats(120)


if __name__ == '__main__':
    if PROFILER_ENABLED:
        import hotshot
        import hotshot.stats
        import atexit

        XRTPROF = hotshot.Profile(XRTPROF_FILENAME)
        atexit.register(xenrt_profile_stats)

    com = Commands()

    if len(sys.argv) < 2:
        sys.stderr.write("Usage: %s <command> [options]\n" % (sys.argv[0]))
        sys.stderr.write("  run '%s help' for a list of commands\n" % \
                         (sys.argv[0]))
        sys.exit(1)

    if com.has(sys.argv[1]):
        if PROFILER_ENABLED:
            rc = XRTPROF.runcall(com.run, sys.argv[1], sys.argv[2:])
        else:
            rc = com.run(sys.argv[1], sys.argv[2:])
        sys.exit(rc)

    if sys.argv[1] == '--help':
        sys.exit(com.run("help", sys.argv[2:]))

    sys.stderr.write("Command '%s' not recognised.\n" % (sys.argv[1]))
    sys.stderr.write("  run '%s help' for a list of commands\n" %
                     (sys.argv[0]))
    sys.exit(1)


