#!/usr/bin/env python

##
## merge patches from a ticket and optionally test
## everything
##
## TODO:
##  * add logging, maybe with files based on ticket number
##  * write code that uses qsave in apply_patches below
##  * more options for running tests (optional, etc.)
##  * make the list of *what* to doctest load from a file
##    instead of being hardcoded
##  * make sphinx warnings show up as errors
##

import os, sys, shutil, urllib
from optparse import OptionParser, OptionGroup

import sage
import sage.misc
import sage.misc.hg as hg
##
## various globals
##
SAGE_ROOT = os.environ['SAGE_ROOT']
SAGE_COMMAND = os.path.join(SAGE_ROOT, 'sage')
SAGE_TRAC = 'http://trac.sagemath.org'
TRAC_TICKET_PATH = '/sage_trac/ticket/'

oldpath = sys.path
sys.path = oldpath + [os.path.join(SAGE_ROOT, 'devel', 'sage', 'doc', 'common')]
# List of language directories for documentation:
from build_options import LANGUAGES
sys.path = oldpath

DOCTEST_DIRS = [os.path.join(SAGE_ROOT, 'devel', 'sage', 'doc', L)
                for L in LANGUAGES]
DOCTEST_DIRS.append(os.path.join(SAGE_ROOT, 'devel', 'sage', 'sage'))

DOCTEST_OPTIONS = ['none', 'files', 'directory', 'long']

repository_ls = [ hg.hg_sage, hg.hg_scripts, hg.hg_extcode, hg.hg_root ]
repositories = { 'sage': hg.hg_sage,
                 'main': hg.hg_sage,
                 'scripts': hg.hg_scripts,
                 'bin': hg.hg_scripts,
                 'extcode': hg.hg_extcode,
                 'root': hg.hg_root }
parser = OptionParser("""sage -merge [options] [ticket-number]

Tries to automate the process of merging tickets in release management.
See http://wiki.sagemath.org/devel/ReleaseManagement for more info.

Example usage:

sage -merge -c or sage -merge --candidates

    * List all candidates for merging, i.e. all trac tickets with positive review. 

sage -merge <ticket_number> <options>

    * Download patches from trac for the given ticket number, merge them, run tests, and report the results. 

sage -merge -a <options> or sage -merge --all <options>

    * For each ticket on trac with a positive review, download the patches, apply them, and run doctests. At the end, report which tickets passed, which failed, and which didn't have any files to doctest (as is commonly the case with tickets for new spkg's). 
""")

behavior = OptionGroup(parser, 'Behavior',
                       'Behavior after merging. Default is to pop after merging')

behavior.add_option('-l', '--leave-in-queue', action='store_const',
                    const='leave', dest='behavior',
                    help='Leaves the patches in mercurial queue. Conflicts with --finish.')
behavior.add_option('-f', '--finish', action='store_const',
                    const='finish', dest='behavior',
                    help='Performs qfinish, commiting the patches. '
                    'Conflicts with --leave-in-queue.')

parser.add_option_group(behavior)


parser.add_option('-a', '--all', action='store_true',
                  help='For each ticket on trac with a positive review, '
                  'download, apply, and test each. At the end, report which '
                  'pass, fail, and have no files to doctest')
parser.add_option('-c', '--candidates', action='store_true',
                  help='List all candidates for merging, i.e., all trac '
                  'tickets with positive review')
parser.add_option('-v', '--verbose', action='store_true',
                  help='Display the results of each test')
parser.add_option('-n', '--num-threads', action='store', type='int', default=2,
                  help='Number of threads to work with.  Default 2.')
parser.add_option('-r', '--repository', 
                  choices = repositories.keys(), default='sage',
                  help="Which repository to apply to. Choices: %r.  Default 'sage'." % repositories.keys())
parser.add_option('-t', '--test', default='files',
                  help='What things to doctest. Choices: %r or the first letter(s) any choice.' % DOCTEST_OPTIONS)
parser.add_option('-o', '--overwrite', action='store_true', default=False,
                  help='Whether to overwrite files when downloading.  Default False')
parser.add_option('-d', '--directory', default='',
                  help='Directory to store patches in.  Default: a temporary directory.')

parser.set_defaults(behavior='pop')

def all_patches(n):
    """
    Given a ticket number (either as a string or integer), return the list
    of all patches stored on the Sage trac server on that ticket.
    """
    ticket_url = SAGE_TRAC + TRAC_TICKET_PATH + str(n)
    print "Fetching %s ..."%ticket_url
    try:
        f = urllib.urlopen(ticket_url)
    except IOError:
        raise IOError, "could not fetch %s"%ticket_url

    attachment_lines = []
    comment_lines = []
    found_dd = False
    for line in f.readlines():
        if '/attachment/' in line:
            attachment_lines.append(line)
            comment_lines.append('')
        if found_dd:
            if len(attachment_lines) == 0:
                raise RuntimeError, "Failure parsing trac... sorry"  #blame Tom
            comment_lines[-1] = line.strip()
            found_dd = False
        if '<dd>' in line:
            found_dd = True
    f.close()

    attachment_names = []
    comments = []
    for x,y in zip(attachment_lines,comment_lines):
        beg = x.find('/sage_trac/')
        end = x.find('" title')
        if not end > beg >= 0:
            continue
        name = SAGE_TRAC + x[beg:end]
        name = name.replace('/attachment/', '/raw-attachment/', 1)
        if name not in attachment_names:
            attachment_names.append(name)
            comments.append(y)

    return attachment_names, comments

def get_all_patches(patch_ls, directory=None, overwrite=False):
    """
    Given a list of patches, create a temporary directory and put all
    the patches there. if a directory is given, use that instead.
    """
    if directory is None:
        import sage.misc.misc as misc
        directory = misc.tmp_dir('release')
    else:
        directory = os.path.abspath(directory)
        try:
            os.makedirs(directory)
        except OSError:
            if not os.path.isdir(directory): 
                raise

    os.chdir(directory)
    print "Fetching patches and storing in %s:"%directory

    patch_files = []
    for patch_url in patch_ls:
        patch_filename = os.path.split(patch_url)[1]
        if os.path.exists(patch_filename):
            if overwrite:
                os.remove(patch_filename)
            else:
                raise OSError, "patch file %s already exists"%(os.path.join(directory, patch_filename))
        print "Fetching %s ..."%patch_url
        try:
            urllib.urlretrieve(patch_url, filename=patch_filename)
        except IOError:
            os.chdir('..')
            shutil.rmtree(directory)
            raise IOError, "error fetching %s"%patch_url
        patch_files.append(patch_filename)
            
    return patch_files, directory

def do_patch_queue_editing(order, comments, default_repo):
    import sage.misc.misc as misc
    tmp_file = os.path.normpath(misc.tmp_filename())
    f = open(tmp_file, 'w')
    for patchname, comment in zip(order,comments):
        f.write(patchname + "\n")
        if comment:
            f.write("# " + comment + "\n")

    f.write(r"""
# Delete and order patches to apply.  Lines beginning with '#' are removed.
# If a ticket should be applied to any repo other than sage, prepend 'repo|',
# for example, to apply the patch to the 'scripts' repo,
# scripts|http://trac...
""")
    f.close()
    if not os.environ.has_key('EDITOR'):
        print "No $EDITOR variable present, using pico."
        os.environ['EDITOR'] = 'pico'
    res = os.system("$EDITOR %s" % tmp_file)
    if res:
        raise ValueError, "could not start editor, aborting."

    f =  open(tmp_file, 'r')
    patchnames = f.readlines()
    f.close()

    new_order = []
    with_repos = []
    for patchname in patchnames:
        patchname = patchname.strip()
        if (not patchname) or patchname.startswith('#'):
            continue

        if '|' in patchname:
            r, patchname = patchname.split('|')
            repo = repositories[r]
        else:
            repo = default_repo

        if not patchname in order:
            raise ValueError, "%s not a patchname in %s" % (patchname, order)
        new_order.append(patchname)
        patchfile = patchname.split('/')[-1]
        with_repos.append((patchfile,repo))

    if not new_order:
        raise ValueError, "No patches to be applied -- aborting"

    return new_order, with_repos
    

def apply_patches(patch_ls, patch_directory, default_repo, order=None, dry_run=True, behavior='pop'):
    """
    Applies a list of patches to sage.
    """
    if order is None:
        order = [ (x, default_repo) for x in patch_ls ]
    else:
        for i in range(len(order)):
            item = order[i]
            if isinstance(item, tuple):
                if len(item) != 2 or (not isinstance(item[0], str)) or \
                   (item[1] not in repository_ls):
                    raise ValueError, "unknown patch %s"%item
                repository = item[1]
                item = item[0]
            else:
                repository = default_repo

            if isinstance(item, str):
                if not item in patch_ls:
                    raise ValueError, "unknown patch %s"%item
                order[i] = (item, repository)
            else:
                try:
                    ind = int(item)
                except ValueError:
                    raise ValueError, "unknown patch %s"%item
                if (0 > ind) or (ind > len(patch_ls)):
                    raise IndexError, "no patch of index %s"%ind
                order[i] = (patch_ls[ind], repository)

    repos_used = {}
    for _, repository in order:
        repos_used[repository] = True
    repos_used = repos_used.keys()

    if dry_run:
        for patch, repository in order:
            print "applying patch %s/%s to repository %s"%(patch_directory, patch, repository.dir())
        return

    # make sure we have queues enabled in all the repos
    for repo in repos_used:
        res, error = repo('qinit', interactive=False)
        if 'unknown command' in error:
            raise NotImplementedError, "please enable Mercurial queues"
        series, err = repo('qser', interactive=False)
        if behavior == 'finish' and series != '':
            # TODO: use qsave/qrestore to make this possible. not hard,
            # but I need to make sure I understand what happens to the live
            # changes when I do this. Or maybe just qtop?
            #
            # here's a way to get the appropriate revision number to save
            # *after* we repo('qsave'):
            # repo('tip', interactive=False)[0].split()[1].split(':')[0]
            raise NotImplementedError, "cannot merge patches with nonempty queue series"

    # start applying some patches
    applied_patches = []
    for patch, repo in order:
        patch_filename = os.path.join(patch_directory, patch)
        if not os.path.exists(patch_filename):
            raise OSError, "could not find patch file" + patch_filename

        # here we should use -n to prepend the patch number
        msg, err = repo('qimport %s' % patch_filename, interactive=False)
        if 'abort' in err:
            if behavior == 'pop' or behavior == 'finish':
                print "Popping patches from queue ..."
                clear_queue(applied_patches, repos_used)
            raise ValueError, "error applying patch %s"%patch_filename

        msg, err = repo('qpush', interactive=False)
        if 'abort' in err or 'rejects' in err:
            if behavior == 'pop' or behavior == 'finish':
                print "Popping patches from queue ..."
                clear_queue(applied_patches, repos_used)
            raise ValueError, "error applying patch %s"%patch_filename

        applied_patches.append((patch, repo))

    return applied_patches, repos_used, order

def commit_queue(repos_used):
    """
    Given a list of repositories, commit the queue in each into the
    repository. (That is, call qfinish on each.)
    """
    for repo in repos_used:
        msg, err = repo('qfinish -a', interactive=False)
        if 'abort' in err:
            raise OSError, "error committing to repository %s: %s"%(repo.dir(), err)

def clear_queue(applied_patches, repos):
    """
    Given a list patch_ls, each entry of which is of the form (patch,
    repo), qpop and qdelete each of them from the corresponding
    repository.
    """
    for repo in repos:
        msg, err = repo('qpop -a', interactive=False)
        if 'abort' in err:
            raise ValueError, "error popping patches from repository %s: %s"%(repo.dir(), err)
    
    for patch, repo in applied_patches:
        ## now the stack is clean, delete the patches
        msg, err = repo('qdelete %s'%patch, interactive=False)
        if 'abort' in err:
            raise ValueError, "error deleting patch %s from repository %s: %s"%(patch, repo.dir(), err)

#################################################################
## Utility functions
#################################################################


def print_section(s, surrounding_lines=True):
    """
    Print a section heading for output. This is here because
    I'm finnicky about making them uniform and want to change
    them later at will.
    """
    if isinstance(s, str):
        lines = s.splitlines()
    elif isinstance(s, list):
        lines = s
    else:
        raise ValueError, "unknown section header %s"%s
    max_width = max([len(x) for x in lines])

    sep_char = '='
    left = ' >>> '
    right = ' <<< '
    print_width = max_width + len(left) + len(right)

    if surrounding_lines: print
    print sep_char*print_width
    for line in lines:
        left_gap = ' '*((max_width - len(line)) // 2)
        right_gap = ' '*(max_width - len(line) - len(left_gap))
        print "%s%s%s%s%s"%(left, left_gap, line, right_gap, right)
    print sep_char*print_width
    if surrounding_lines: print

def get_touched_files_for_patches(patches, directory):
    touched_files = []
    for patch in patches:
        filename = os.path.join(directory, patch)
        f = open(filename)
        import re
        rx = re.compile(r'^\+\+\+ b/(.*?)\s') # boundary of a word

        for line in f.readlines():
            # line = line.replace('\t', ' ') # some diffs have tabs, which don't show up as word boundaries
            m = rx.search(line)
            if m:
                # relative to SAGE_ROOT
                touched_files.append('devel/sage/' + m.groups()[0])
        f.close()
    return touched_files

def get_patches_with_users_help(n,
                                default_repo,
                                directory=None,
                                overwrite=False):


    ## get the patches from the ticket
    patch_url_ls,patch_comments = all_patches(n)
    if len(patch_url_ls) == 0:
        raise ValueError, "no patches to merge"

    print "Found %s patches: "%len(patch_url_ls)
    for patch_url in patch_url_ls:
        print "  ", patch_url
    print

    old_patch_url_ls = patch_url_ls

    print "Please edit the queue before I apply"
    patch_url_ls, order = do_patch_queue_editing(patch_url_ls, patch_comments,
                                                 default_repo)

    if len(patch_url_ls) == 0:
        raise ValueError, "no patches to merge"

    ## grab all patch files from trac
    patches, directory = get_all_patches(patch_url_ls,
                           directory=directory,
                           overwrite=overwrite)

    return patches, directory, order

def collect_patches_for_run(default_repo):
    ticket_ls = all_tickets(report='11')

    ticket_bundle = []    

    for number, title in ticket_ls:
        try:
            print_section('Fetching ticket number %s'%number)
            patches, directory, order = get_patches_with_users_help(number, default_repo)
            ticket_bundle.append((number,title,patches,directory, order))
        except:
            print "failed"
    return ticket_bundle

def log_ticket(ticket,message,log={},init=False,return_log=False):
    if return_log:
        return log
    elif init:
        for k in log.keys():
            del log[k]
    else:
        log[ticket] = message


#################################################################
## Main functions
#################################################################

def merge_and_run(n,
                  default_repo,
                  directory=None,
                  overwrite=False,
                  behavior='pop',
                  verbose=False,
                  num_threads=2):
    """
    Merge patches from ticket n and test the sage library.
    Returns True when everything finishes cleanly, and raises
    a ValueError otherwise.
    """

    print_section("Merging patches from ticket number %s."%n)

    patches, directory, order = get_patches_with_users_help(n, default_repo,
                                                            directory, overwrite) 

    return apply_and_test(n, patches, directory, behavior, default_repo,
                          verbose=verbose, order=order, num_threads=num_threads)


def apply_and_test(n, patches, directory, behavior, default_repo, verbose=False,
                   archive=False, order=None, num_threads=2):

    start_dir = os.getcwd()
    os.chdir(SAGE_ROOT)

    import sage.misc.misc as misc
    tmp_file = misc.tmp_filename()

    ## apply the patches to the various repos
    applied_patches, repos_used, order = apply_patches(patches,
                                                       directory,
                                                       default_repo,
                                                       dry_run=False,
                                                       behavior=behavior,
                                                       order=order)

    try:
        error = run_tests(patches,
                          directory,
                          behavior,
                          applied_patches,
                          repos_used,
                          order,
                          tmp_file,
                          verbose=verbose,
                          num_threads=num_threads)
    except:
        error = 'unexpected exception'

    os.chdir(start_dir)

    if error:
        if behavior == 'pop' or behavior == 'finish':
            print "Popping patches from queue ..."
            clear_queue(applied_patches, repos_used)

        log_ticket(n,'error, %s'%error)
        if archive:
            archive_log(n,tmp_file)
        raise ValueError, error
    else:
        log_ticket(n,'success')

def archive_log(ticketnum, tmp_file):
    dir = os.getcwd()
    os.chdir(SAGE_ROOT+'/tmp')
    os.system('cp %s %s-mergelog'%(tmp_file,ticketnum))
    os.system('tar -rf mergelog.tar %s-mergelog'%(ticketnum))
    os.system('rm %s-mergelog'%(ticketnum))
    os.chdir(dir)

def verbosity_command(command, tmp_file, verbose):
    if verbose:
        #hack to get return code from teed process.
        #note, $PIPESTATUS[0] should work, but it's broken on every
        #bash I've tried.

        os.system("bash -c '%s; echo $? > %s.err' 2>&1 | tee -a %s"%(command,tmp_file,tmp_file))
        return int( file('%s.err'%tmp_file).read() )
#        os.system("%s 2>&1 | tee -a %s"%(command, tmp_file))
    else:
        return os.system("%s 2>&1 >> %s"%(command, tmp_file))

def run_tests(patches,
              directory,
              behavior,
              applied_patches,
              repos_used,
              order,
              tmp_file,
              verbose=False,
              num_threads=2):
              
    print_section('Rebuilding Sage')

    ## now we test everything.

    ## rebuild sage with the new changes.
    
    res = verbosity_command('sage -b', tmp_file, verbose=verbose)
    if res:
        return "sage failed to build"

    ## first, check that sage even starts up.
    res = os.system('sage-starts')
    if res:
        return "sage cannot start with patches applied"

    if False:
        print_section('Building Sage documentation')

        # currently, we only check to see if the doc build script actually
        # raised an exception.  i don't know if it's possible that it
        # could do something else to signal bad behavior.
        verbosity_command("%s -docbuild all --jsmath html"%SAGE_COMMAND, tmp_file, verbose)

        f = open(tmp_file)
        for line in f.readlines():
            if line == 'Traceback (most recent call last):':
                f.close()
                return "error building html documentation."
        f.close()

    test_success = None

    if test_type == 'none':
        test_success = True
    else:
        print_section('Running doctest suite')

        if test_type == 'files':
            files_to_test = get_touched_files_for_patches([ x[0] for x in applied_patches ], directory)
        elif test_type == 'directory':
            filenames = get_touched_files_for_patches([ x[0] for x in applied_patches ], directory)
            directories = [ os.path.dirname(filename) for filename in filenames ]
            files_to_test = directories
            # things like touching module_list.py trigger an entire test when its not usually wanted
            if 'devel/sage' in files_to_test:
                files_to_test.remove('devel/sage')
            # testing sage/rings and sage/rings/number_field ends up testing number_field twice
            files_to_test.sort() # this orders shorter names before longer names
            keepers = []
            while files_to_test:
                keeper = files_to_test.pop(0)
                keepers.append(keeper)
                files_to_test = [ f for f in files_to_test if not f.startswith(keeper) ]
            files_to_test = keepers

        elif test_type == 'long':
            files_to_test = DOCTEST_DIRS
        else:
            return "unknown test type %s"%test_type

        ## we run the docs and library individually, so that
        ## we can produce better error messages.

        files_to_test = ' '.join(list(set(files_to_test)))
        test_command = '%s -tp %s -long %s' % (SAGE_COMMAND, num_threads, files_to_test)
        print test_command
        print


        verbosity_command(test_command, tmp_file, verbose)

        f = open(tmp_file)
        line = f.readlines()[-3]
        f.close()
        if line == 'All tests passed!\n':
            # in this case, tests succeeded
            test_success = True
        else:
            # here, tests failed
            test_success = False

    if test_success:
        # we want to either commit or leave the queue as-is for
        # further testing
        if test_type != 'none':
            test_msg = "All tests passed! "
        else:
            test_msg = "Doctests skipped. "
        if behavior == 'leave':
            print test_msg + "Leaving patches in queue."
        elif behavior == 'finish':
            print test_msg + "Committing queue to repository..."
            commit_queue(repos_used)
            print test_msg + "Committing queue to repository... DONE"
        elif behavior == 'pop':
            print test_msg + "Popping patches from queue ..."
            clear_queue(applied_patches, repos_used)
        return None
    else:
        # tests failed, so don't commit, but either leave tickets
        # applied or clear the queue
        return "tests failed"


def all_tickets(report='11'):
    r"""
    Return the list of all tickets with positive review.
    """
    url = SAGE_TRAC + '/sage_trac/report/' + report
    print "Fetching %s ..." % url
    try:
        f = urllib.urlopen(url)
    except IOError:
        raise IOError, "could not fetch %s" % url

    import re
    title_regexp = re.compile('href="/sage_trac/ticket/(\d+)">(?:\[.*\])?(?:\s*)(.*?)</a>$')
    tickets = []
    for line in f.readlines():
        if 'View ticket' not in line or '#' in line:
            continue
        title_match = title_regexp.search(line)
        if title_match:
            number, title = title_match.groups()
            tickets.append((number, title))
            
    f.close()
    return tickets

def test_all_tickets(behavior, default_repo, verbose=False):
    ticket_ls = all_tickets(report='11')
    working_tickets = []
    untested_tickets = []
    failed_tickets = []

    os.system('tar -cf %s/tmp/mergelog.tar COPYING.txt'%SAGE_ROOT) #I'd like to make an empty tar, but tar cowardly refuses
    
    ticket_bundle = collect_patches_for_run(default_repo)
    for number,title,patches,directory,order in ticket_bundle:
        print_section('Testing ticket number %s'%number)
        try:
            apply_and_test(number, patches, directory, behavior, default_repo,
                           verbose=verbose, archive=True, order=order)
            print "Ticket #%s passed!" % number
            working_tickets.append(number)
        except:
            exception, msg, traceback = sys.exc_info()
            if msg == 'no tickets to merge':
                print "Ticket #%s had no attached files." % number
                untested_tickets.append(number)
            else:
                print "Ticket #%s failed with %s: %s" % (number, exception.__name__, msg)
                failed_tickets.append(number)


    print
    print "Ticket Log"
    log = log_ticket(None,None,return_log=True)
    for n in log.keys():
        print "#%s -- %s"%(n,log[n])
   
    print
    print "Tickets to merge: ", working_tickets
    print "Tickets I couldn't test: ", untested_tickets
    print "Tickets needing work: ", failed_tickets
    

##
## stuff to test:
##
## make ptestall
## make ptestlong
## sage -docbuild reference html
## sage -docbuild reference pdf
## sage -startuptime
##

#################################################################
## actually execute
#################################################################

if __name__ == '__main__':

    num = 0
    directory = None
    test_type = 'files'

    if len(sys.argv) < 2:
        parser.print_help()
        sys.exit(1)
        
    (options, args) = parser.parse_args()
    
    if options.directory:
        directory = os.path.abspath(options.directory)
        try:
            os.makedirs(directory)
        except OSError:
            if not os.path.isdir(directory): 
                raise

    # Returns the first known doctest option that the argument is a prefix of.
    test_type = reduce(lambda x,y: x or y,
              [option for option in DOCTEST_OPTIONS if option.startswith(options.test)],
              False)
    
    if not test_type:
        print ("Error: test type %s must be (a prefix of) one of %s" %
               (options.test, DOCTEST_OPTIONS))
        sys.exit(1)

    default_repo=repositories[options.repository]
        
    ## make sure we got a ticket number to test, unless we were called with
    ## -c, in which case we print and exit.
    if options.candidates:
        ls = all_tickets(report='11')
        ls.sort()
        print_section('Tickets with positive review')
        for number, title in ls:
            print "#%4s: %s"%(number, title)
        print
        sys.exit(0)
    elif options.all:
        test_all_tickets(behavior=options.behavior,
                         default_repo=default_repo,
                         verbose=options.verbose)
        sys.exit(0)
    else:
        try:
            num = args[0]
        except:
            print "Error: could not convert %s to a ticket number."%args[0]
            sys.exit(1)

    try:
        # we don't need the return value from merge_and_run here
        merge_and_run(num,
                      default_repo=default_repo,
                      overwrite=options.overwrite,
                      directory=directory,
                      behavior=options.behavior,
                      verbose=options.verbose,
                      num_threads = options.num_threads)
    except:
        exc, msg, traceback = sys.exc_info()
        print "Building failed with %s: %s"%(exc.__name__, msg)
        sys.exit(1)
    sys.exit(0)
