#!/usr/bin/perl -T --
# -*- Mode: perl; indent-tabs-mode: nil -*-
#

# processmail - the mail processing engine for tinderbox.  The build
# machines (clients) send their build logs to the server to report on
# the status of the builds and this program accepts the mail (via an
# MTA like sendmail) and acts as the MDA (mail delivery agent).  This
# program processes the build logs for the tinderbox server (tinder.cgi)
# and informs the server of any status updates.  It also stores the
# logs in compressed form for future references by the tinderbox
# server.  No locks are used by the mail processes, data is passed to
# the tinderbox server in a maildir like format.

# $Revision: 1.36 $ 
# $Date: 2004/07/18 18:37:46 $ 
# $Author: kestes%walrus.com $ 
# $Source: /cvsroot/mozilla/webtools/tinderbox2/src/bin/processmail_builds,v $ 
# $Name:  $ 


# The contents of this file are subject to the Mozilla Public
# License Version 1.1 (the "License"); you may not use this file
# except in compliance with the License. You may obtain a copy of
# the License at http://www.mozilla.org/NPL/
#
# Software distributed under the License is distributed on an "AS
# IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
# implied. See the License for the specific language governing
# rights and limitations under the License.
#
# The Original Code is the Tinderbox build tool.
#
# The Initial Developer of the Original Code is Netscape Communications
# Corporation. Portions created by Netscape are
# Copyright (C) 1998 Netscape Communications Corporation. All
# Rights Reserved.
#

# complete rewrite by Ken Estes for contact info see the
#     mozilla/webtools/tinderbox2/Contact file.
# Contributor(s): 


# Standard perl libraries
use File::Basename;
use Sys::Hostname;
use File::stat;
use Getopt::Long;

# Tinderbox libraries
use lib '/usr/share/tinderbox/lib', '/etc/tinderbox';

use TinderConfig;
use TreeData;
use FileStructure;
use HTMLPopUp;
use VCDisplay;
use Error_Parse;
use BuildStatus;
use Utils;
use Persistence;
use TinderDB::Build;
use MailProcess;

# name of the version control system
$REGENERATE_AFTER_MAIL = $TinderConfig::REGENERATE_AFTER_MAIL || "0";



sub usage {


    my $usage =<<EOF;

$0	[--version] [--help]  
$0	[filename]



Informational Arguments


--version	Print version information for this program

--help		Show this usage page

--force-time	Ignore the: 'starttime', 'timenow' attributes of the mail 
		message and set new ones based off the current time. This 
		option is to allow for easier testing the functionality of 
		the program. This flag is not recommended for production use.

--skip-check	Turn off the strict checking of input files, to allow for
		easier testing the functionality of the program. This flag is
		not recommended for production use.


Synopsis


This program handles mail acceptance for the tinderbox system.  The
mail transfer agent runs this program to deliver the mail.  Mail is
recived on stdin, this makes it particularly easy to test this script.
The script accepts no arguments and will exit with return code of zero
if successful.  

Should errors occur during the processing of the mail message, the
message is bounced by exiting with a non zero exit code and the error
is written to the error log and standard err.

Mail is expected to consist of 

	1) A set of tinderbox variables with values applicable to this
	build. This includes information on which regular expressions
	to use when parsing the build log and whether the build was
	successful or not.

	2) The build/test log as output by the tool which was run to
	generate the build.

	3) The binary which results from the build, if desired.

Mail messages are parsed to extract the tinderbox variables and
database update files are written for the tinderbox server to pickup
when it next wakes up.  By convention all database update files are
stored in a file name which matches the module name which will accept
them.  We are creating information for the database controled by the
module TinderDB::Build so its update file must be named
$TinderConfig::FILE{'TinderDB::Build::Update'} with some unique id
placed on the end.  This file contains all the tinderbox variable
information extracted from the mail message and any new variables
derived while parsing the log file.

The build logs are converted into HTML and are split into full and
summary form and stored on the disk.  Both logs' basenames are the
same unique id which is found by using the time this program was
started and its pid.  The logs are stored in separate directories and
compressed.  The server knows how to contruct URLS to the log files
because that information is passed in the database update files.
There may be serveral instances of this program running so each
database update file is written to the disk with a unique name.  No
locks are held by this code.

Builds which take a long time to complete can send multiple status
mails each one containing the log file which has been completed so
far.  All mails sent before the build is finished should have the
status 'building'.  The last mail must contain the results of the
build.  The website will make each build log available so that
everyone can see the log file and get a feel for how far into the
compilation the build machine has progressed.

HTTP Post

This script can also process mail which is sent via HTTP Post rather
then SMTP.  To use this feature simply put this script in the CGI bin
directory and make sure that your webserver will accept posts to this
executable.  This may require changing the name of this executable to
end in '.cgi' or some other extension as your local security policy
requires.

Mail Format

The mail has a very simple format and is not MIME encoded and has no
attachments. 

The tinderbox variables must be at the top of the mail and the last
variable must be 'END'.  We ignore the line following the variables,
it should be blank.  Tinderbox variables are specified by a line which
looks like this string:

tinderbox: variablename: variablevalue

Any uuencoded binaries in the mail message are extracted and uudecoded
and stored on disk in the filename specified by the tinderbox variable
binaryname.

The bulk of the mail message is the output from the compilers and test
tools which created the build.  Each tinderbox server must be
customized with the set of regular expressions which describe the
errors which are produced as part of the build process.  These regular
expressions are stored in the module Error_Parse.pm.  Each mail
contains the tinderbox variable 'errorparser' which chooses one set of
regular expressions to use when processing the mail's contents.


Tinderbox Variables:

Tinderbox variables are name/value pairs which appear at the top of
the mail message.  Lines can appear in any order but END should be the
last line followed by one blank line. If the tinderbox variable name
is repeated on different lines, then its value is taken to be the
concatenation of all values listed.

The required tinderbox variable:

	tinderbox: administrator : {the mailing address of the maintainer
	                             of this build machine}
	tinderbox: builddate : deprecated {the time the build was started in time()
	                          format, this is effectively the build number}
	tinderbox: starttime : {the time the build was started in time()
	                          format, this is effectively the build number}
	tinderbox: buildname : {the official name of the build, often the name
				of the machine where the build was performed 
				and the OS of the machine}
	tinderbox: errorparser : {the error parser to use when 
	                            processing the log file}
	tinderbox: status :{the results of the build, must be a valid status 
				as determined by
			BuildStatus::is_status_valid(); }
	tinderbox: timenow : {the time the update was sent, in time() format.
				There can be several updates for a given build
				as long as they return the status 'building'
				it is legal to mail the log file at different 
				stages of the build.  In each update the timenow
				will be different but the starttime will be the same.}
	tinderbox: tree : {the name of the tree this build was for.  Must be
				a valid tree as termined by:
                                  TreeData::tree_exists();
			}
	tinderbox: END



Binaries:

If there is a tinderbox variable:

  tinderbox: binaryname 

and there is uuencoded text in the build log then the text is
extracted uudecoded and stored in the directory
FileStructure::get_filename(\$tree, 'build_bin_dir') with the basename
as found by the basename of the tinderbox variable 'binaryname'.



Bloat Statistics:

The bloat statistics are a Mozilla specific test.  They help track the
amount of leaked memory in an application.  The bloat statistics are
found between lines which match:

	m/^\#+ BLOAT STATISTICS/)
	m/^--NEW-BLOAT--/ )

Their line has this format:

	TOTAL <absolute leaks> <% leaks delta> <absolute bloat> <% bloat delta>

The leakage information is removed from this line, formated into valid
HTML and passed to the Tinderbox server for displaying on the Status
page.  The rendered HTML is found in the 

  tinderbox: bloatdata : 

entry of the update file.

TinderboxPrint:

Arbitrary links and text can be embedded in the tinderbox build
cells. Any log file which contains a line beginning with
"TinderboxPrint: " will be copied into the build cell.  If you are
embedding links, you should put exactly one link per TinderboxPrint
command since the links are checked to ensure that they are well
formed. Here are some examples of embedded performance numbers (and
links to the web page relevant to these numbers) from one mozilla
build.

	TinderboxPrint: <a title="Leaks: total bytes 'malloc'ed and not 'free'd"href="http://tegu.mozilla.org/graph/query.cgi?testname=trace_malloc_leaks&units=bytes&tbox=backupboy&autoscale=1&days=7&avg=1">Lk:301KB</a>


	TinderboxPrint: <a title="Maximum Heap: max (bytes 'malloc'ed - bytes 'free'd) over run"href="http://tegu.mozilla.org/graph/query.cgi?testname=trace_malloc_maxheap&units=bytes&tbox=backupboy&autoscale=1&days=7&avg=1">MH:7.82MB</a>


	TinderboxPrint: <a title="Allocations: number of calls to 'malloc' and friends"href="http://tegu.mozilla.org/graph/query.cgi?testname=trace_malloc_allocs&units=bytes&tbox=backupboy&autoscale=1&days=7&avg=1">A:284K</a>


Log Files

The basename of the log files is passed to the tinderbox server via
the tinderbox variables:

	tinderbox: full-log
	tinderbox: brief-log

The files are found in the directories

	FileStructure::get_filename(\$tree, 'full-log')
	FileStructure::get_filename(\$tree, 'brief-log')




Files:

File names for the output files are constructed from the values returned by:

	FileStructure::get_filename(\$tree, 'brief-log')
	FileStructure::get_filename(\$tree, 'build_bin_dir')
	FileStructure::get_filename(\$tree, 'full-log')
	FileStructure::get_filename(\$tree, 'brief-log')
	FileStructure::get_filename(\$tree, 'TinderDB_Dir')


Example:

This is the set of tinderbox variables which are used in the logsample
mail message.  See the logsample file in the test directory for a more
complete example of tinderbox build mail.

tinderbox: tree: Project_A
tinderbox: starttime: 934395485
tinderbox: timenow: 934395489
tinderbox: status: success
tinderbox: buildname: worms-SunOS-sparc-5.6-Depend-apprunner
tinderbox: errorparser: unix
tinderbox: END


Time Format

Most OS have an internal time format of "seconds since the epoch".
This format is easiest to use if all your machines have the same OS or
if you standardize on some epoch for the purposes of tinderbox
mails. Times can also be given in the format "MM/DD/YY HH:MM:SS" if
this is more convenient.


EOF

    print $usage;
    exit 0;

} # usage



sub set_filenames {

  # These are temporary files, used by the mail processsor and will be
  # erased before the program exits.  We use filenames which end in
  # html for debugging the mail processor in isolation.  We do not
  # glob on the prefix in the mailprocessor.

  my ($tree) = $TINDERBOX{'tree'};

  %TMP_FILE = (
               'errorpick' => (FileStructure::get_filename($tree, 'brief-log').
                               "$tree_dir/Tmp.header.$main::UID.html"),
               
               'binaryfile' => (FileStructure::get_filename($tree, 'build_bin_dir').
                                "/Tmp.$main::UID.html"),

               'full-log' => (FileStructure::get_filename($tree, 'full-log').
                              "/Tmp.$main::UID.html"),
               
               'brief-log' => (FileStructure::get_filename($tree, 'brief-log').
                               "/Tmp.$main::UID.html"),

              );


  %FILE = (

           # there is also a binaryfile created in assemble_files()
           # which gets created using information from get_filename()
           # and $TINDERBOX{}

           'full-log' => (FileStructure::get_filename($tree, 'full-log').
                          "/$main::UID.html"),
           
           'brief-log' => (FileStructure::get_filename($tree, 'brief-log').
                               "/$main::UID.html"),

           'update_time_stamp' => (FileStructure::get_filename($tree, 'update_time_stamp').
                                   ""),
              );

  foreach $file ( (values %FILE), (values %TMP_FILE) ) {
    main::mkdir_R(dirname($file));
  }

  # set the URL for how to get to the logfiles we will generate.

  # We can not end the cgibin calls in '.html.gz' or IE will think we
  # are returning a gziped file. Gunzip knows to add the extension and
  # to add the dirname to find the file.


  $TINDERBOX{'brieflog'} = 
    ("$FileStructure::URLS{'gunzip'}?".
     "tree=$tree&brief-log=$main::UID");
  
  $TINDERBOX{'fulllog'} = 
    ("$FileStructure::URLS{'gunzip'}?".
     "tree=$tree&full-log=$main::UID");

  # record the basename of the log file (not used but useful for debugging.)

  $TINDERBOX{'full-log'} = File::Basename::basename($FILE{'full-log'});
  $TINDERBOX{'brief-log'} = File::Basename::basename($FILE{'brief-log'});
  
  return 1;
}



# If the field is a date prefer the output of fix_time_format to what
# is passed into the program.  We may receive human readable strings
# and these need to be converted to time() format.


sub fix_tinderboxvar_time_format {

  foreach $key (@DATE_FIELDS) {
    ($TINDERBOX{$key}) || next;
    $TINDERBOX{$key} = main::fix_time_format($TINDERBOX{$key});
  }

  return 1;
}

# this function is used to change the tinderbox variables from the old
# format into the new format. It will help ease transitions to the new
# tinderbox code.  I believe the new format will be more maintainable
# going forward.

sub backward_compatibility {

  if ($TINDERBOX{'status'} eq 'busted') {
    $TINDERBOX{'status'} = 'build_failed';
  }

  if ($TINDERBOX{'status'} eq 'testfailed') {
    $TINDERBOX{'status'} = 'test_failed';
  }

  if ($TINDERBOX{'status'} eq 'buildstarted') {
    $TINDERBOX{'status'} = 'building';
  }

  if ($TINDERBOX{'builddate'}) {
    $TINDERBOX{'starttime'} = $TINDERBOX{'builddate'};
    $TINDERBOX{'builddate'} = undef;
  }

  if ($TINDERBOX{'build'}) {
    $TINDERBOX{'buildname'} = $TINDERBOX{'build'};
    $TINDERBOX{'build'} = undef;
  }

  # traditional mozilla name shortening conventions.

  $TINDERBOX{'buildname'} =~ s/Clobber/Clbr/g;
  $TINDERBOX{'buildname'} =~ s/Depend/Dep/g;    

  # keep funky characters out of the buildname.  We use ',' as a
  # separator so that must not appear.

  $TINDERBOX{'buildname'} =~ s/[,;:!|\$\*\&\%\^\@\`\'\"\+]+/_/g;
  $TINDERBOX{'buildname'} =~ s/__+/_/g;
 
  return 1;
}




=pod

"errorparser" is a bad name, we want a "buildtools" which describes
the compilers and make programs.  The buildtools module is shared with
the buildscript and the logparser.  We might want to have the
configure output drive the choice of errorparser or tell configure
what choices to make based on the error parser.  Notice that if we
take this to extremes we need to have a set of Object Oriented parsers
(cc, cpp, make, perl) each looking for errors it knows how to handle.


May want further checks on who can submit what information to
tinderbox.

need to also check that the current time on the server agrees with
current time on client within 10 minutes.

need to check that this update either 

  1) has the same starttime as the last update of this tree/build
  2) has a starttime greater than min time between builds.

=cut
  ;


# The next set of functions wrap all implementations from package
# Error_Parse.  They change their implementations based on the global
# variable $TINDERBOX{'errorparser'}, so this value must be extracted
# from the message before they are called.


sub check_required_vars {

  my ($err_string) = '';

  if( $TINDERBOX{'tree'} eq ''){
    $err_string .= "Variable: 'tinderbox: tree' not set.\n";
  } elsif ( !(TreeData::tree_exists($TINDERBOX{'tree'})) ) {

    $err_string .= "Variable: 'tinderbox: tree' ".
      "is not set to a valid tree.\n";
  }

  if(($MAIL_HEADER{'To'} =~ m/external/i ||
         $MAIL_HEADER{'CC'} =~ m/external/i) &&
        $TINDERBOX{'tree'} !~ m/external/i) {
    $err_string .= "Data from an external source didn't specify ".
      "an 'external' tree.\n";
  }
  
  ($TINDERBOX{'buildname'}) ||
    ($err_string .= "Variable: 'tinderbox: buildname:' not set.\n");
  
#  ($TINDERBOX{'administrator'}) ||
#    ($err_string .= "Variable: 'tinderbox: administrator:' not set.\n");
#  
#  ($TINDERBOX{'administrator'} =~ m/\@\w/) ||
#    ($err_string .= "Variable: 'tinderbox: administrator:' ".
#     "is not a valid email address.\n");

  ($TINDERBOX{'errorparser'}) ||
    ($err_string .= "Variable: 'tinderbox: errorparser:' not set.\n");

  {
      my ($func) = "Error_Parse::$TINDERBOX{'errorparser'}::parse_errorline";
      my ($ignore) = eval &$func(@_);    
      
      ($@) &&
          ($err_string .= (
                           "Errorparser: ".
                           "'$TINDERBOX{'errorparser'}::parse_errorline'".
                           " does not exist.\n"
                           ));
  }
  {
      ($func) = "Error_Parse::$TINDERBOX{'errorparser'}::line_type";
      ($ignore) = eval &$func(@_);    
      ($@) &&
          ($err_string .= "Errorparser: '$TINDERBOX{'errorparser'}::line_type'".
           " does not exist.\n");
  }
  
  # Note at this point we could have sourced only the implementation
  # of the error parser that we need.  I think programs which use an
  # autoload for libraries are messy since they cause variable run
  # time requirements.

  $TINDERBOX{'timenow'} = main::extract_digits($TINDERBOX{'timenow'});
  if ($TINDERBOX{'timenow'} eq '') {
      # fudge the timenow
      $TINDERBOX{'timenow'} = time();
#    $err_string .= "Variable: 'timenow' not set.\n";
  } else {
    
    if( main::is_time_in_sync($TINDERBOX{'timenow'}) ){
      $timenow = $TINDERBOX{'timenow'};
    } else {
      $err_string .= (
                      "Variable: 'tinderbox: timenow:',".
                      " is not of the form ".
                      "MM/DD/YY HH:MM:SS, or unix date, ".
                      "or your clock is set incorrectly, ".
                      "or the mail was delayed for a long time.\n".
                      "variable timenow: $TINDERBOX{'timenow'}, ".
                      "timenow: $TIME,  (".
                      ( ($TINDERBOX{'timenow'}-$TIME) / 
                        $main::SECONDS_PER_MINUTE ).
                      " minutes)\n".
                     "");
    }
    
  }

  $TINDERBOX{'starttime'} = main::extract_digits($TINDERBOX{'starttime'});
  if( $TINDERBOX{'starttime'} eq ''){
    $err_string .= "Variable: 'tinderbox: starttime' not set.\n";
  } else {
    if( $TINDERBOX{'starttime'} <= $TINDERBOX{'timenow'} ){
      $starttime = $TINDERBOX{'starttime'};
    } else {
      $err_string .= (
                      "Variable: 'tinderbox: starttime:',".
                      " is greater than the tinderbox: timenow.\n".
                      "starttime: $TINDERBOX{'starttime'}, ".
                      "timenow: $TINDERBOX{'timenow'}".
                      "\n".
                      "");
    }
    
  }
  
  # Build Status
  
  ($TINDERBOX{'status'}) ||
    ($err_string .= "Variable: 'tinderbox: status:' not set.\n");
  
     ( BuildStatus::is_status_valid($TINDERBOX{'status'}) ) ||
     ($err_string .= "Variable: 'tinderbox: status:' must be one of: ".
      join( ', ', BuildStatus::get_all_sorted_status()).
      "Variable: 'tinderbox: status is set to $TINDERBOX{status}', \n".
     "\n");
  
  # Report errors by bouncing mail
  
  ($err_string) &&
    die($err_string);
  
}


sub format_bloat_delta {
  my ($value,) = @_;
  my $units = ' ';
  $value = $value || 0;

  if ($value >= 1000000) {
    $value = int($value / 1000000);
    $min =   int($min / 1000000);
    $units = 'M';
  } elsif ($value >= 1000) {
    $value = int($value / 1000);
    $min =   int($min / 1000);
    $units = 'K';
  }

  return "$value$units";
}




# prepare bloat data for display

sub process_bloat_data {
  my ($line, $func_line_type,) = @_;
  
  # we have to call &highlight_errors() even though there are no
  # errors, or the line numbers will get messed up
  
  my ($line_type) = &$func_line_type($line);
  @html = &highlight_errors($line, $line_type, $lineno, \$next_err, 
                            $func_parse_errorline);
  print FULL $html[1];
  if ( $line =~ m/^TOTAL/) {
    # Line format:
    #  TOTAL <absolute leaks> <% leaks delta> <absolute bloat> <% bloat delta>
    chomp;
    my ($leaks, $bloat) = (split /\s+/, $line)[1,3];
    my $bloat_string = (
                        "<br>Lk:". format_bloat_delta($leaks).
                        "<br>Bl:". format_bloat_delta($bloat,)
                       );
    
    $TINDERBOX{'bloatdata'} = $bloat_string;
  }
  return ;
}

sub log_links {
  my ($tree, $logtype) = @_;

  my $notlogtype = ( $logtype eq 'full' ? "brief" : "full");
  my $out;

  $out .= "\n";
  $out .= "<font size=\"+1\">\n";
  $out .= "<dt>";
  $out .= HTMLPopUp::Link(
                          "linktxt"=>"Show <b>$notlogtype</b> Log\n", 
                          "href"=> ("$FileStructure::URLS{'gunzip'}?".
                                    "tree=$tree&".
                                    "$notlogtype-log=$main::UID"),
                         );
  $out .= "<dt>";
  $out .= HTMLPopUp::Link(
                          "linktxt"=>("Return to the $TINDERBOX{'tree'} ".
                                      "Status Page\n"), 
                          "href"=>(FileStructure::get_filename($tree, 
                                                               'tree_URL').
                                   "/status\.html"),
                         );
  $out .= "</font>\n";
  

  return $out
}

# create the HTML file headers for both the full and brief log
# files. This must run after the message body has been parsed as some
# of the data comes from there.

sub log_header {
  my ($logtype) = @_;
  
  my ($out) = '';
  my ($tree) = $TINDERBOX{'tree'};

  $out .= HTMLPopUp::page_header('title'=>"$logtype Build Log ".
                                 "for tree: $tree ");

  $out .= log_links($tree, $logtype)."\n";
  $out .= "<H2>Build Data</H2>\n";
  
  $out .= "<pre>\n";
  
  # notice that since we generate this output after processing the
  # whole log file the tinderbox variables may contain 'derived' data
  # which was not in the original mail message.

  $out .= MailProcess::format_tinder_vars(%TINDERBOX);
  $out .= "</pre>\n";
  
  $out .= "<H2>Pick an error message to see Log</H2>\n";

  return $out;

}

sub log_footer {
  my ($logtype) = @_;
  
  my ($out) = '';
  my ($tree) = $TINDERBOX{'tree'};

  $out .= log_links($tree, $logtype)."\n";
  $out .= "</BODY>\n";
  $out .= "</HTML>\n";

  return $out;
}


# create the HTML links for errors this is for both 
# 1) the $headerline,  which will appear in the error picklist 
#      at the top of the log file and points to the error 
#      in the same HTML file.
# 2) the $logline which is the actual error displayed as a HTML 
#     link which points to the CVS file and line which is the problem.


sub highlight_errors {
  my( $line, $line_type, $lineno, 
      $next_err_ref, $func_parse_errorline) = @_;
  
  my( @error ) = ();
  
  # clean up any embedded HTML in the mail
  $line = HTMLPopUp::escapeHTML($line);
  
  my ($logline) = '';
  my ($headerline) = '';
  
  # All log lines get line numbers so users can send mail pointing to
  # any line in the log file, example:
  #         Someone should look at the Makefile, this 
  #           make line <LINK> 
  #           is the same as this one <LINK>

  # line numbers start AFTER the tinderbox variables have been read.

  $logline .= HTMLPopUp::Link(
                              "name"=>$lineno,
                              "linktxt"=>$lineno,
                              "href"=>"\#$lineno",
                             );

  # add spaces so line numbers are left justified to 6 chars

  $logline .= (' ' x ($Error_Parse::LINENO_COLUMN-length($lineno)));

  if ( ( $line_type eq 'error' ) && ( !($last_was_error) ) ) {
    
    my ($err_ref) = $$next_err_ref;
    $$next_err_ref++;
    
    # tag this line and link to next error line
    
    $logline .= HTMLPopUp::Link(
                                "name"=>"err".($err_ref),
                                "linktxt"=>"NEXT", 
                                "href"=>"\#err".($$next_err_ref),
                               );
    
    $headerline .= HTMLPopUp::Link(
                                   "linktxt"=>$line, 
                                   "href"=>"\#err".($err_ref),
                                  );
    
  } else {
    
    # indent, so first column can be the 'next' link 
    
    $logline .= (" " x length("NEXT"));
    $headerline .= '';
    
  }
  
  # separate the logs from "NEXT" just a bit more.
  
  $logline .= "    ";
  
  # markup the line if it looks interesting
  
  if ($line_type ne 'info') {
    @error= &$func_parse_errorline($line);
    if ("@error") {
      $line = VCDisplay::guess
        (
         'tree' => $TINDERBOX{'tree'}, 
         'file' => $error[0],
         'line' => $error[1],
         'linktxt' =>  $line, 
        );
    }
    
    my $color = Error_Parse::type2color($line_type);
    ($color) &&
      ($color = "color=$color");

    $logline .=  ("<font $color>".
                  $line.
                  "</font>");
  } else {
    $logline .= $line;
  }
  
  
  return ($headerline, $logline);
}


sub parse_mail_body {

  # create several files with a single pass over the mail message:
  
  # BINARY:      the uuencoded file, if present, in the mailmessage.
  # FULL:        a complete copy of the logfile with HTML markup.
  # BRIEF:       contains only the error messages with some lines of 
  #                 surrounding context and HTML markup.
  # ERROR_PICK:  a list of html links to error messages, this will appear
  #                 on the top of both brief and full when our 
  #                 processing is complete. 

  # it is a bug in processing should this function be called before
  # the tinderbox variables are set. We intentionally leave off the \n
  # on some of the die's to get tracebacks of internal errors.

  # We must use HTMLPopUp::escapeHTML to clean up any embedded HTML in the
  # mail messege to prevent attacks.  Though it may be more friendly
  # to let compilers and things embed URLS in their error messages:

  # http://www.ciac.org/ciac/bulletins/k-021.shtml
  
  #   When a victim with scripts enabled in their browser reads this
  #   message, the malicious code may be executed
  #   unexpectedly. Scripting tags that can be embedded in this way
  #   include <SCRIPT>, <OBJECT>, <APPLET>, and <EMBED>.
  
  

  # It is a bit cleaner to convert these symbolic references to hard
  # references in the eval at the top of the function instead of
  # calling the symbolic reference to the function continually.  This
  # is also how we check that the function exists.

    my $func_line_type = '';
    my $func_parse_errorline = "";

    my ($out, $func);

    $func = "Error_Parse::$TINDERBOX{'errorparser'}::line_type";
    $out = eval "\$func_line_type = \\&${func};";
    ($@) &&
        die("Errorparser is not set. $@");

    $func = "Error_Parse::$TINDERBOX{'errorparser'}::parse_errorline";
    $out = eval "\$func_parse_errorline = \\&${func};";
    ($@) &&
        die("Errorparser is not set. $@");

  open(BINARY, ">$TMP_FILE{'binaryfile'}") ||
    die("Could not write to file: '$TMP_FILE{'binaryfile'}'. $!\n");
  open(FULL, ">$TMP_FILE{'full-log'}") ||
    die("Could not write to file: '$TMP_FILE{'full-log'}'. $!\n");
  open(BRIEF, ">$TMP_FILE{'brief-log'}") ||
    die("Could not write to file: '$TMP_FILE{'brief-log'}'. $!\n");
  open(ERROR_PICK, ">$TMP_FILE{'errorpick'}") ||
    die("Could not write to file: '$TMP_FILE{'errorpick'}'. $!\n");

  print ERROR_PICK "<!-- error pick menu -->\n<pre>\n";
  print ERROR_PICK HTMLPopUp::Link(
                                   "linktxt"=>"End of Log File", 
                                   "href"=>"\#EOF",
                                   ).
                                   "\n";
    

  my ($next_err) = 1;
  my ($lines_since_error) = 0;
  my ($error_lines) = 0;
  my (@lastlines) = ();

  while (defined($line = <>)) {

    $lineno++;

    my (@html) = '';

    # uuencoded binaries get pulled out into a special file

    if ( ( $line =~ m/^begin [0-7][0-7][0-7] / ) ..  
         ( $line =~ m/^end\n/ ) 
       ) {

      print BINARY $line;
      next;
    }

    # Find and save the TinderboxPrint data

    if ($line =~ m/\s*TinderboxPrint\s*:\s*(.*)/) {
        my $print = $1;

        # Long lines might get split by mailing software but then we
        # can't reconstitute them, since we will not know how many
        # lines to take.  Make sure refs, if they exist, are closed;

        my $good_line = (
                         ($print !~ m!<\s*a\s+!i) ||
                         ($print =~ m!>[^<>]*<\s*/a\s*>!i)
                         );

        if ($good_line) {
            $TINDERBOX{'print'} .= "&nbsp; ".$print;
        }

        print FULL $line;
        next;
    }

    # process bloat statistics 

    if ( ($line =~ m/^\#+ BLOAT STATISTICS/) ..  
         ( $line =~ m/^--NEW-BLOAT--/ ) 
       ) {
      process_bloat_data($line,$func_line_type,);
      next;
    }

    $lines_since_error++;

    # prepare the line for browser display
    my ($line_type) = &$func_line_type($line);
    my %args = (
                'lineno' => $lineno, 
                'line_type' => $line_type, 
                'line' => $line,
                );

    Error_Parse::run_status_handler(%TINDERBOX, %args);

    @html = &highlight_errors($line, $line_type, $lineno, \$next_err, 
                                   $func_parse_errorline);


    print FULL $html[1];

    # brief log processing

    if ($html[0]) {

      my ($lines_context) = min( ($lines_since_error - $Error_Parse::LINES_AFTER_ERROR - 2),
                                 $Error_Parse::LINES_BEFORE_ERROR);
      my ($context_index) = max($#lastlines - $lines_context, 0);
      my ($lines_skiped)  = max ( $lines_since_error - 
                                  ($Error_Parse::LINES_BEFORE_ERROR + $Error_Parse::LINES_AFTER_ERROR+1), 
                                  0);
      
      print ERROR_PICK $html[0];

      if ($lines_skiped) {
        print BRIEF ("\n<i><font size=\"+1\">".
                     " Skipping $lines_skiped Lines...".
                     "</font></i>\n\n");

      }

      print BRIEF @lastlines[$context_index .. $#lastlines];

      $print_to_brief = 1 + $Error_Parse::LINES_AFTER_ERROR;

      $lines_since_error = 0;
      $error_lines++;
    } 

    if ($print_to_brief) {
      print BRIEF $html[1];
      $print_to_brief--;
    }
    

    push @lastlines, $html[1];
    
    # don't keep more lines of history then we need
    
    if ( !( $#lastlines & 127 ) ){
      my ($first) = $#lastlines - $Error_Parse::LINES_BEFORE_ERROR; 
      my ($last) = $#lastlines;
      @lastlines = @lastlines[ $first .. $last ];      
    }

  } # while

  # We always want to see the very last line of the build, whether or
  # not the build failed.  So treat the last line like an error and
  # print the context surrounding it.

  my ($lines_context) = min( ($lines_since_error - $Error_Parse::LINES_AFTER_ERROR - 2),
                             $Error_Parse::LINES_BEFORE_ERROR);
  my ($context_index) = max($#lastlines - $lines_context, 0);
  my ($lines_skiped)  = max ( $lines_since_error - 
                              ($Error_Parse::LINES_BEFORE_ERROR + $Error_Parse::LINES_AFTER_ERROR+1), 
                              0);
      
  print ERROR_PICK $line;

  if ($lines_skiped) {
    print BRIEF ("\n<i><font size=\"+1\">".
                 " Skipping $lines_skiped Lines...".
                 "</font></i>\n\n");
    
  }
  
  print BRIEF @lastlines[$context_index .. $#lastlines];

  my ($last_errline) = ("</pre><p>".
                        "<font size=\"+1\">".
                        HTMLPopUp::Link(
                                        "name"=>"err".($next_err_ref),
                                        "linktxt"=>"No More Errors", 
                                        "href"=>"\#err".(1),
                                        ).
                        "</font>".
                        HTMLPopUp::Link(
                                        "name"=>"EOF",
                                        ).
                        "<br>");
  
  print BRIEF $last_errline;
  print FULL  $last_errline;
  
  print ERROR_PICK "\n";
  print ERROR_PICK "<!-- end error pick menu -->\n";
  print ERROR_PICK "\n";
  print ERROR_PICK "<H2>Build Log</H2>\n";
  print ERROR_PICK "<pre>\n";
  print ERROR_PICK "\n\n";

  close(BINARY) ||
    die("Could not close file: '$TMP_FILE{'binaryfile'}'. $!\n");
  close(FULL) ||                 
    die("Could not close file: '$TMP_FILE{'full-log'}'. $!\n");
  close(BRIEF) ||                
    die("Could not close file: '$TMP_FILE{'brief-log'}'. $!\n");
  close(ERROR_PICK) ||           
    die("Could not close file: '$TMP_FILE{'errorpick'}'. $!\n");

  # store the count of errors
  $TINDERBOX{'errors'} = $error_lines;


  return ;  
}



sub assemble_files {

# piece together the permanent files from the temporary ones.

# much of this could be done in shell scripts (cat) but we have more
# error control in perl and it is more portable to non-unix OS.

  my $tree = $TINDERBOX{'tree'};


  {

    # append the headers to the top of the full log and compress the result
    
    open(ERROR_PICK, "<$TMP_FILE{'errorpick'}") ||
      die("Could not read from file: '$TMP_FILE{'errorpick'}'.\n");
    open(TMP_FULL, "<$TMP_FILE{'full-log'}") ||
      die("Could not read from file: '$TMP_FILE{'full-log'}'.\n");
    open(FULL, ">$FILE{'full-log'}") ||
      die("Could not write to file: '$FILE{'full-log'}'.\n");
    
    print FULL log_header('full');
    
    while (defined($line=<ERROR_PICK>)) {
      print FULL $line;
    }
    
    while (defined($line=<TMP_FULL>)) {
      print FULL $line;
    }
    
    print FULL log_footer('full');
    
    close(FULL) ||
      die("Could not close file: '$FILE{'full-log'}': waitstatus: $? : $! \n");
    close(TMP_FULL) ||
      die("Could not close file: '$TMP_FILE{'full-log'}': $! \n");
    close(ERROR_PICK) ||
      die("Could not close file: '$TMP_FILE{'errorpick'}': $! \n");
  }
  
  {

    # append the headers to the top of the brief log

    open(ERROR_PICK, "<$TMP_FILE{'errorpick'}") ||
      die("Could not read from file: '$TMP_FILE{'errorpick'}'.\n");
    open(TMP_BRIEF, "<$TMP_FILE{'brief-log'}") ||
      die("Could not read from file: '$TMP_FILE{'brief-log'}'.\n");
    open(BRIEF, ">$FILE{'brief-log'}") ||
      die("Could not write to file: '$FILE{'brief-log'}'.\n");
    
    print BRIEF log_header('brief');

    while (defined($line=<ERROR_PICK>)) {
      print BRIEF $line;
    }
    while (defined($line=<TMP_BRIEF>)) {
      print BRIEF $line;
    }
    
    print BRIEF log_footer('brief');

    close(BRIEF) ||
      die("Could not close file: '$FILE{'brief-log'}'.\n");
    close(TMP_BRIEF) ||            
      die("Could not close file: '$TMP_FILE{'brief-log'}'.\n");
    close(ERROR_PICK) ||           
      die("Could not close file: '$TMP_FILE{'errorpick'}'.\n");
    
  }
  
  if( 
     ($TINDERBOX{'binaryname'}) && 
     (!(-z $TMP_FILE{'binaryfile'})) 
    ) {
    my $bin_dir = FileStructure::get_filename($tree, 'build_bin_dir');
    $bin_dir =~ s/ //g;
    
    mkdir_R($bin_dir);
    
    $TINDERBOX{'binaryname'} = File::Basename::basename($TINDERBOX{'binaryname'});
    my $outfile = "$bin_dir/".$TINDERBOX{'binaryname'};

    my (@cmd) = (
                 @UUDECODE,
                 $outfile,
                 $TMP_FILE{'binaryfile'},
                );

    system(@cmd) ||
      die("Could not run cmd: '@cmd'. $!\n");
    
    # the builder tells us the basename of the binary, we tell the
    # server the full path name of the binary.

  }

  {
      my (@cmd) = (@GZIP, $FILE{'full-log'});
      system(@cmd) &&
          die("Could not run '@cmd'. $!\n");
  }

  {
      my (@cmd) = (@GZIP, $FILE{'brief-log'});
      system(@cmd) &&
          die("Could not run '@cmd'. $!\n");
  }

  return ;
}


# --------------------main-------------------------
{
  set_static_vars();
  get_env();
  MailProcess::parse_mailprocess_args();
  chk_security();

  # if we are being run via sendmail then we should have no output and
  # errors should be sraight text, if we are being run via HTTP Post,
  # our errors must be valid html and we must have valid HTML output;

  if (defined ($ENV{'HTTP_USER_AGENT'})) {
      $SIG{'__DIE__'} = \&::fatal_error;
      $OUT=   (
               "Content-type: text/html\n\n".
               "<html></html>\n".
               '');
  } else {
      $SIG{'__DIE__'} = \&MailProcess::fatal_mailprocessing_error;
      $OUT = '';
  }

  %MAIL_HEADER =  MailProcess::parse_mail_header();
  %TINDERBOX = MailProcess::parse_tinderbox_vars( main::hash2string(%MAIL_HEADER) );
  @DATE_FIELDS = qw(builddate starttime timenow);

  fix_tinderboxvar_time_format();
  backward_compatibility();
  set_filenames();

  # Testing is easier if the update files appear at the top of the
  # status page, even though they are checked out of CVS.

  if ($FORCE_TIME) {
    $TINDERBOX{'starttime'} = time() - ($main::SECONDS_PER_MINUTE*10);
    $TINDERBOX{'timenow'} = time() - 10;
  }

  # when we are testing turn off the checks

  ($SKIP_CHECK) ||
    check_required_vars();

  parse_mail_body();
  assemble_files();

  my ($uniq_id) = join('.', 
                       $TINDERBOX{'$tree'},
                       $TINDERBOX{'buildname'},
                       $TINDERBOX{'timenow'}
                      );

  MailProcess::write_update_file('Build', $uniq_id, 
                                 $TINDERBOX{'tree'}, %TINDERBOX);

  # clean up all the temp files created in this script
  
  foreach $tmpfile (values %TMP_FILE) {

    (!(-e $tmpfile)) ||
      unlink($tmpfile) ||
        die("Could not unlink: '$tmpfile'. $!\n");    

  }

  if ($REGENERATE_AFTER_MAIL) {
      HTMLPopUp::regenerate_HTML_pages();
  }

  print $OUT;

  exit 0;
}

