#!/usr/bin/perl

# CGI script to facilitate simple retrieval search for the Textpresso system.
# Output is sorted by year.
# 
# copyright (c) Hans-Michael Muller, Pasadena, California, 2002-2004
#

use strict;
use CGI;
use POSIX;

### globals
#

# handle to this CGI and own address
my $query = new CGI;
my $absmyself = $query->url(absolute=>1);

# directories of importance

my $databaseroot = "/var/www/html/text/tdb/";

my $indexcat = $databaseroot . "ind/cat/";
my $indexkey = $databaseroot . "ind/key/";
# add more as you go along

# cookies
my %generalsettings = $query->cookie('generalsettings'); # general settings
my %displayoptions; # display options
if ($query->cookie('displayoptions')) {
    %displayoptions = $query->cookie('displayoptions');
} else {
    $displayoptions{'Source Type'} = 1;
    $displayoptions{Title} = 1;
    $displayoptions{Abstract} = 1;
    $displayoptions{Author} = 1;
    $displayoptions{Journal} = 1;
    $displayoptions{Year} = 1;
    $displayoptions{Citation} = 1;
    $displayoptions{'Number of matches'} = 1;
    $displayoptions{Select} = 1;  
}
my %displayvalues; # display values
if ($query->cookie('displayvalues')) {
    %displayvalues = $query->cookie('displayvalues');
} else {
    $displayvalues{ResultsPerPage} = 10;
    $displayvalues{SentencesPerMatch} = 10;
}

my %lastsimplequery = $query->cookie('lastsimplequery'); # last query

# other globals of interest
my %targetdirectories = (Author => 'aut/', Title => 'tit/',
			 Abstract => 'abs/', Paper => 'art/',
			 Year => 'yea/');
my %searchtargets = (Author => 0, Title => 0, 
		     Abstract => 0, Paper => 0,
		     Year => 0); 
                                                             # This hash defines 
                                                             # the search targets
my %labels = ReadElements("/var/www/html/text/textpresso.dtd"); # read in
                                                                # elements that
                                                                # can be 
                                                                # searched for.
my @elementkeys = sort keys % labels; # list of elements, abbreviated. 

# format of result: $searchresult{ searchtarget }{ file }{ sentence } = no. of occ.;
my %searchresults = ();
my $lastsearchfilename = $query->param('lsfilename'); 
my $displaypage = 1;
my @displayedkeys = $query->param('dispkeys');
my @expandedabs = $query->param('expandedabs');
my $totalpages = $query->param('totalpages');
my $execcommand = "";

#
### end globals;

### process form
#

my $abstractkeyhit = 0;
foreach my $key (@displayedkeys) {
    if ($query->param("expanded$key")) {
	push @expandedabs, $key;
	$abstractkeyhit = 1;
    }
    if ($query->param("collapse$key")) {
	for (my $i = 0; $i < @expandedabs; $i++) { delete $expandedabs[$i] if ($expandedabs[$i] eq $key) }
	$abstractkeyhit = 1;
    }
}

if ($query->param('searchtargets')) {
    foreach my $key (keys % searchtargets) { $searchtargets{$key} =  0}
    foreach my $key ($query->param('searchtargets')) {
	$searchtargets{$key} = 1;
    }
}
    
if ($query->param('lastquery')) {
    for (my $i = 0; $i < 4; $i++) {
	my $varname = "inc" . $i;
	$query->param(-name=>$varname, -value=>$lastsimplequery{$varname});
    }
    $query->param(-name=>'simplesearchstring', 
		  -value=>$lastsimplequery{'simplesearchstring'});
}

if ($query->param('search')) {
    my @includes = ();
    if($searchtargets{Paper} || $searchtargets{Abstract} || $searchtargets{Title}) {
	for (my $i = 0; $i < 4; $i++) {
	    my $varname = "inc" . $i;
	    push @includes, $query->param($varname);
	}
    }
    my @auxkeys = split(/\s+/, $query->param('simplesearchstring'));
    my @keywords;
    foreach my $word (@auxkeys) {
	push @keywords, $word if ($word ne "");
    }
    if ($query->param('exactmatch') ne 'on') {
	for (my $i = 0; $i < @keywords; $i++) { $keywords[$i] .= "*" }
    }
    my $mode = $query->param('searchmode');
    %searchresults = SimpleSearch($indexcat, $indexkey, \%targetdirectories,
				  \@includes, \@keywords, \%searchtargets,
				  $mode);

    my $tmp1filename;
    do { $tmp1filename = tmpnam() } until (!-e "$databaseroot/$tmp1filename");
    open (OUT1,">$databaseroot/$tmp1filename");
    print OUT1 join(",", @includes), "\n";
    print OUT1 join(",", @keywords), "\n";
    foreach my $target (keys % searchresults) {
	foreach my $key (keys %{ $searchresults{$target} }) {
	    print OUT1 $target . "," . $key . ",";
	    foreach my $sentence (keys % { $searchresults{$target}{$key} }) {
		print OUT1 $sentence . ",";
	    }
	    print OUT1 "\n";
	}
    }
    close (OUT1);
    $lastsearchfilename = $tmp1filename;
    $query->param(-name => 'lsfilename', -value => $lastsearchfilename);
    $query->param(-name => 'page', -value => 1);
}

if (($query->param('lastsearch')) || ($query->param('emailresults'))
    || ($query->param('previouspage')) || ($query->param('nextpage'))
    || $abstractkeyhit) {
    %searchresults = ();
    if ($lastsearchfilename) {
	open (IN, "$databaseroot/$lastsearchfilename");
	my $junk = <IN>;
	$junk = <IN>; # first two lines (query) are of no interest here.
	while (my $line = <IN>) {
	    chomp ($line);
	    my @items = split (/\,/, $line);
	    my $target = shift(@items);
	    my $key = shift(@items);
	    while (my $sentence = shift(@items)) {
		$searchresults{$target}{$key}{$sentence} = 1;
	    }
	}
	close (IN);
    } else {
	my @includes = ();
	if($searchtargets{Paper} || $searchtargets{Abstract} || $searchtargets{Title}) {	
	    for (my $i = 0; $i < 4; $i++) {
		my $varname = "inc" . $i;
		push @includes, $query->param($varname);
	    }
	}
	my @auxkeys = split(/\s+/, $query->param('simplesearchstring'));
	my @keywords;
	foreach my $word (@auxkeys) {
	    push @keywords, $word if ($word ne "");
	}
	if ($query->param('exactmatch') ne 'on') {
	    for (my $i = 0; $i < @keywords; $i++) { $keywords[$i] .= "*" }
	}
	my $mode = $query->param('searchmode');
	%searchresults = SimpleSearch($indexcat, $indexkey, \%targetdirectories,
				      \@includes, \@keywords, \%searchtargets,
				      $mode);
    }
    if ($query->param('previouspage')) {
	if ($query->param('page') == 1) {
	    $displaypage = 1;
	    $query->param(-name => 'page', -value => 1);
	} else {
	    $displaypage = $query->param('page') - 1;
	    $query->param(-name => 'page', -value => $displaypage);
	}
    } elsif ($query->param('nextpage')) {
	$displaypage = $query->param('page') + 1;
	$displaypage = ($displaypage > $totalpages) ? $totalpages : $displaypage;
	$query->param(-name => 'page', -value => $displaypage);
    } else {
	$displaypage = $query->param('page');
    }
}    
    
if ($query->param('emailresults')) {
    if ($query->param('recipient') ne "") {
	my @includes = ();
	for (my $i = 0; $i < 4; $i++) {
	    my $varname = "inc" . $i;
	    push @includes, $query->param($varname) if ($query->param($varname) ne "");
	}
	my @auxkeys = split(/\s+/, $query->param('simplesearchstring'));
	my @keywords;
	foreach my $word (@auxkeys) {
	    push @keywords, $word if ($word ne "");
	}
	my $subject = "Your request from Textpresso - Query ";
	if (@includes) {
	    $subject .= "categories - ";
	    foreach my $inc (@includes) {
		$subject .= $labels{$inc} . " + ";
	    }
	}
	if (@keywords) {
	    $subject .= "keywords - ";
	    $subject .= join(', ', @keywords);
	} else {
	    chop ($subject);
	    chop ($subject);
	    chop ($subject);
	}
	my $recipient = $query->param('recipient');
	my $matchsentences = -1; # negative number indicates no sentences incl.
	if ($query->param('emailmatches')) {
	    $matchsentences = $displayvalues{SentencesPerMatch};
	}
	my $options = "";
	foreach my $key (keys % displayoptions) {
	    $options .= "\"" . $key . "\" " if ($displayoptions{$key});
	}
	$execcommand = "echo \'nice /var/www/html/text/tdb/scr/mailresults.pl $recipient \"$subject\" \" \" $lastsearchfilename $matchsentences $options\' \| at now";

    }
}

#
###

### Set cookie (last simple query) to current query, if current query is not empty
#

my $realquery = 0;
for (my $i = 0; $i < 4; $i++) {
    my $varname = "inc" . $i;
    if ($query->param($varname) ne "") { $realquery = 1 }
}
if ($query->param('simplesearchstring')) { $realquery = 1 }

if ($realquery) {
    for (my $i = 0; $i < 4; $i++) {                    # change behaviour.
	my $varname = "inc" . $i;
	$lastsimplequery{$varname} = $query->param($varname);
	$lastsimplequery{'simplesearchstring'} = $query->param('simplesearchstring');
    }
}
my $lsqcookie = $query->cookie(-name => 'lastsimplequery', 
			       -value => \%lastsimplequery,
			       -path => '/cgi-bin/',
			       -expires => '+730d');

#
###

PrintHeader($query, $lsqcookie);

### start form
#

print $query->start_form(-method => 'POST', -action => $absmyself);

#
###

### present input fields
#

print $query->font({-face => 'verdana, helvetica', -size => '2', -color => 'red'},
		   'This is a quick fix for having a simple retrieval output sorted by year. ');
print $query->font({-face => 'verdana, helvetica', -size => '2', -color => 'red'},
		   'For large returns, it might be a little slow. ');
print $query->br;
print $query->font({-face => 'verdana, helvetica', -size => '2', -color => 'red'},
		   'An implementation for sorting an output according to any column will follow later. ');
print $query->br;
print $query->font({-face => 'verdana, helvetica', -size => '2'},
		   'The simple search allows for any ');
print $query->font({-face => 'verdana, helvetica', -size => '2'},
		   'combination of category and keyword searches. ');
print $query->br;
print $query->font({-face => 'verdana, helvetica', -size => '2'},
		   'You can also do a keyword search, a category ');
print $query->font({-face => 'verdana, helvetica', -size => '2'},
		   'search, or a combination of both.');
print $query->p;
print $query->font({-face => 'verdana, helvetica', -size => '2'},
		   'Query should be met in a ');
my %locallabel;
$locallabel{sentence} = " sentence";
$locallabel{publication} = " publication";
print $query->radio_group(-name => 'searchmode',
			  -linebreak => 0,
			  -value => ['sentence', 'publication'],
			  -default => 'sentence',
			  -labels => \%locallabel);
print $query->font({-face => 'verdana, helvetica', -size => '2'},
		   '.');
print $query->p;
print $query->font({-face => 'verdana, helvetica', -size => '2'},
		   'Type in keywords to be searched for, ');
print $query->font({-face => 'verdana, helvetica', -size => '2'},
		   'separated by ');
print $query->start_b;
print $query->font({-face => 'verdana, helvetica', -size => '2'},
		   'white spaces. ');
print $query->end_b;
print $query->font({-face => 'verdana, helvetica', -size => '2'},
		   'The wild card sign (*) may be used:');
print $query->p;
print $query->textfield(-name => 'simplesearchstring', -size => 50, -maxlength => 255);
print $query->checkbox(-name => 'exactmatch',
		       -label => 'Exact match');
print $query->p;
print $query->font({-face => 'verdana, helvetica', -size => '2'},
		   'Specify categories that should also be met: ');
print $query->p;
print $query->popup_menu(-name => 'inc0', 
			 -values => [@elementkeys], 
			 -labels=>\%labels);
print $query->font({-face => 'verdana, helvetica', -size => '2'},
		   ', ');
print $query->popup_menu(-name => 'inc1', 
			 -values => [@elementkeys], 
			 -labels=>\%labels);
print $query->font({-face => 'verdana, helvetica', -size => '2'},
		   ', ');
print $query->popup_menu(-name => 'inc2', 
			 -values => [@elementkeys], 
			 -labels=>\%labels);
print $query->font({-face => 'verdana, helvetica', -size => '2'},
		   ', ');
print $query->popup_menu(-name => 'inc3', 
			 -values => [@elementkeys], 
			 -labels=>\%labels);
print $query->font({-face => 'verdana, helvetica', -size => '2'},
		   '. ');
print $query->p;
print $query->font({-face => 'verdana, helvetica', -size => '2'},
		   'Search in ');
my @defaults = ("Abstract", "Paper");
foreach my $key (sort keys % searchtargets) {
    if ($searchtargets{$key}) { push @defaults, $key}
}
my %locallabel2;
foreach my $key (keys % searchtargets) {
    if ($key eq "Paper") {
        $locallabel2{$key} = " Full Text";
    } else {
        $locallabel2{$key} = " " . $key;
    }
}
print $query->checkbox_group(-name=>'searchtargets',
			     -value=>[sort keys % searchtargets],
			     -defaults=>[sort @defaults],
			     -linebreak=>'false',
			     -rows=> 1,
			     -labels => \%locallabel2);
print $query->p;
print $query->submit(-name => 'search', -value => 'Search!');
print $query->font("&nbsp;");
print $query->submit(-name => 'lastquery', -value => 'Load last query!');
print $query->font("&nbsp;");
print $query->reset('Undo current changes!');

#
###

### output goes here
#

if (%searchresults) {
    my $total = 0;
    my $occurrence = 0;
    my %keychecker = ();
    foreach my $target (keys % searchresults) {
	foreach my $key (keys %{ $searchresults{$target} }) {
	    if ((%{$searchresults{$target}{$key}}) && (!$keychecker{$key})) {$total++}
	    $keychecker{$key} = 1;
	    foreach my $sentence (keys % { $searchresults{$target}{$key} }) {
		$occurrence ++;
	    }
	}
    }
    if ($occurrence > 0) {
	print $query->hr;
	print $query->start_b;
	print $query->font({-face => 'verdana, helvetica', -size => '2'},
			   $occurrence, ' matches ', ' in ', $total, ' publication(s) found.');
	print $query->end_b;
	print $query->p;
	print $query->submit(-name => 'lastsearch', -value => 'Display');
	print $query->font({-face => 'verdana, helvetica', -size => '2'},
			   ' page ');
	my @choices;
	$totalpages = int($total/($displayvalues{ResultsPerPage} || 10) + 1);
	$query->param(-name => 'totalpages', -value => $totalpages);
	for (my $j = 0; $j < $totalpages; $j++) {
	    push @choices, $j + 1;
	}
	print $query->popup_menu(-name =>'page',
				 -default => $displaypage,
				 -values => \@choices);
	print $query->font({-face => 'verdana, helvetica', -size => '2'},
			   ', or ');
	print $query->submit(-name =>'previouspage', -value => 'previous');
	print $query->font({-face => 'verdana, helvetica', -size => '2'},
			   ' or ');
	print $query->submit(-name =>'nextpage', -value => 'next');
	print $query->font({-face => 'verdana, helvetica', -size => '2'},
			   ' page.');
	print $query->p;
	print $query->submit(-name => 'emailresults', -value => 'E-mail');
	print $query->font({-face => 'verdana, helvetica', -size => '2'},
			   ' results to ');
	print $query->textfield(-name => 'recipient', 
				-size => 35, 
				-maxlength => 255,
				-default => $generalsettings{'Email'});
	print $query->font({-face => 'verdana, helvetica', -size => '2'},
			   ', ');
	print $query->checkbox(-name => 'emailmatches',
			       -checked => 'checked',
			       -value => 'ON',
			       -label => 'including matches.');
	
	(my $selectid = $lastsearchfilename) =~ s/\/tmp\/file//g;
	
	my @giftable = ();
	push @giftable, $query->a({-href => "introducepdf.cgi?id=$selectid", 
				   -target => '_blank'},  
				  $query->img({-src => "/text/gif/resultsinpdf.gif", 
					       -border => 0, -alt => 'results in PDF'})) 
	                          .
				  $query->a({-href => "exportendnote.cgi?mode=aen&id=$selectid"},
					    $query->img({-src => "/text/gif/allend.jpg", 
							 -border => 0, -alt => 'export endnote (all)'}))
				  .
				  $query->br
				  .
				  $query->a({-href => "showmatches.cgi?id=$selectid", 
					     -target => '_blank'}, 
					    $query->img({-src => "/text/gif/viewallmatches.gif", 
							 -border => 0, -alt => 'view all matches'}))
				  .
				  $query->a({-href => "exportendnote.cgi?mode=aea&id=$selectid"},
					    $query->img({-src => "/text/gif/alleab.jpg", 
							 -border => 0, -alt => 'export endnote (all) with abstract'}));
	my @minitable = ();
	push @minitable, $query->td({-align => 'center', -valign => 'center'},
				    [
				     $query->img({-src => "/text/gif/vm.jpg", -width => "100%",
						  -border => 0, -align => 'center'}),
				     $query->font({-size => 1}, "view matches for respective reference"),
				     $query->img({-src => "/text/gif/ot.jpg", -width => "100%",
						  -border => 0, -align => 'center'}),
				     $query->font({-size => 1}, "journal's online text"),
				     $query->img({-src => "/text/gif/ra.jpg", -width => "100%",
						  -border => 0, -align => 'center'}),
				     $query->font({-size => 1}, "related articles (PubMed)")
				     ]);
	push @minitable, $query->td({-align => 'center', -valign => 'center'},
				    [
				     $query->img({-src => "/text/gif/en.jpg", -width => "100%",
						  -border => 0, -align => 'center'}),
				     $query->font({-size => 1}, "export bibliography for Endnote"),
				     $query->img({-src => "/text/gif/ea.jpg", -width => "100%",
						  -border => 0, -align => 'center'}),
				     $query->font({-size => 1}, "export bibliography including abstracts for Endnote"),
				     $query->img({-src => "/text/gif/pf.jpg", -width => "100%",
						  -border => 0, -align => 'center'}),
				     $query->font({-size => 1}, "download PDF of article ", $query->b("(Caltech only)"))
				     ]);
	push @giftable, $query->table({-border => '1', -cellpadding => '0',
				       -cellspacing => '0', -width => '80%'},
				      $query->caption("Abbreviation Index"),
				      $query->Tr([@minitable]));

	print $query->p;
	print $query->table({-border => '0', -cellpadding => '4',
			     -cellspacing => '1', -width => '100%'},
			    $query->Tr([$query->td({-align => 'left', -valign => 'center'},[@giftable])]));
	
	@displayedkeys = DisplayAsHTML($query, $displaypage, \@expandedabs, 
				       \%searchresults, $databaseroot, $lastsearchfilename,
				       \%displayoptions, \%displayvalues);
	$query->param(-name => 'dispkeys', -value => [@displayedkeys]);
	$query->param(-name => 'expandedabs', -value => [@expandedabs]);
	print $query->p;
	print $query->font({-face => 'verdana, helvetica', -size => '2'},
			   'Go to ');
	print $query->submit(-name =>'previouspage', -value => 'previous');
	print $query->font({-face => 'verdana, helvetica', -size => '2'},
			   ' or ');
	print $query->submit(-name =>'nextpage', -value => 'next');
	print $query->font({-face => 'verdana, helvetica', -size => '2'},
			   ' page.');
	print $query->p;
    } else {
        print $query->hr;
        print $query->start_b;
        print $query->font({-face => 'verdana, helvetica', -size => '2', -color => 'red'},
			   'No matches found.');
	print $query->end_b;
    }
} elsif (($lastsearchfilename ne "")) {
    print $query->hr;
    print $query->start_b;
    print $query->font({-face => 'verdana, helvetica', -size => '2', -color => 'red'},
                       'No matches found.');
    print $query->end_b;
}

#
###

### end form here
#

# pass last search result file name to next round
print $query->hidden(-name => 'lsfilename', -default => $lastsearchfilename);
print $query->hidden(-name => 'dispkeys', -default => [@displayedkeys]);
print $query->hidden(-name => 'expandedabs', -default => [@expandedabs]);
print $query->hidden(-name => 'totalpages', -default => $totalpages);

print $query->end_form;
print $query->end_html;

#
###

### exec system command 'offline'
#

system($execcommand);

#
###

sub PrintHeader { # begin PrintHeader
    
    my $query = shift;
    my @cookies = ();
    while (@_) { push @cookies, shift(@_)}
    
    print $query->header(-cookie=>[@cookies]);
    print $query->start_html(-title=>'Simple Retrieval', -author=>'Hans-Michael Muller',
			     -bgcolor=>'#e6e6f6',
			     -text=>'#000000', -link=>'#662222', -vlink=>'#993333');
    print $query->br;
    print $query->font({-face => 'verdana, arial, helvetica', -size => '5'}, 'Simple Retrieval');
    print $query->p;
    
} # end PrintHeader

sub PrintBottom { # begin PrintBottom
    my $query = shift;
}   # end PrintBottom

sub ReadElements {

    my $dtdfile = shift;
    my %labels = ();
    $labels { "" } = "none"; # include empty tag for search
    open (DTDFILE, "$dtdfile") or die "Can't open DTD file $dtdfile.";   

    while (my $line = <DTDFILE>) {
	chomp ($line);
	if ($line =~ /<!ELEMENT\s+(\w+)\s\(\#PCDATA\)> <\!--\s((\w|\s)+)\s-->/) {
	    my $key = $1;
	    my $label = $2;
	    if (($key ne "sentence") && ($key ne "article")){  #because pattern above is more specific, this isn't really necessary
		 $labels { substr($key, 0, 2) } = $label;
	    }
	}
    }
    close (DTDFILE);
    return %labels;
}

sub SimpleSearch {
    
    my $indexcat = shift;
    my $indexkey = shift;
    my $pTargetdirectories = shift;
    my $pIncludes = shift;
    my $pKeywords = shift;
    my $pSearchtargets = shift;
    my $mode = shift; # algorithm assumes 'sentence' as a default;

    my %initialresult = (); # This is the first key or cat searched, from which we will 
                            # proceed.

    foreach my $target (keys % $pSearchtargets) {
	
	# copy categories into auxiliary list, 
	# get rid of the 'none' option, resulting in "" for includes
	
        my @auxinc = ();
	foreach my $category (@$pIncludes) {
	    push @auxinc, $category unless ($category eq "");
	}

	# remove identical entries in query
	for (my $j = 0; $j < @auxinc; $j++) {
	    for (my $k = $j+1; $k < @auxinc; $k++) { 
		delete $auxinc[$k] if ($auxinc[$j] eq $auxinc[$k]);
	    }
	}

	# copy keywords into auxiliary list
	my @auxkey = ();
	foreach my $keyword (@$pKeywords) {
	    push @auxkey, $keyword;
	}

	# remove identical entries in query
	for (my $j = 0; $j < @auxkey; $j++) {
	    for (my $k = $j+1; $k < @auxkey; $k++) {
		delete $auxkey[$k] if ($auxkey[$j] eq $auxkey[$k]);
	    }
	}

	if ($$pSearchtargets{$target}) {
	    my $catdir = $indexcat . $$pTargetdirectories{$target};
	    my $keydir = $indexkey . $$pTargetdirectories{$target};

	    if (@auxkey) {
		my $firstkeyword = shift @auxkey;
		my $lowered = "\L$firstkeyword\E";
		my @keywordfiles = <$keydir/$lowered>;
		foreach my $file (@keywordfiles) {
		    open (INKEY, "$file");
		    while (my $line = <INKEY>) {
			chomp($line);
			my @entry = split(/\,/, $line);
			my $key = shift @entry;
			foreach my $sentence (@entry) {
			    $initialresult{$target}{$key}{$sentence} = 1;
			}
		    }
		    close (INKEY);
		}
	    } elsif (@auxinc) { 
		my $firstcategory = shift @auxinc;
		open (INCAT, "$catdir/$firstcategory");
		while (my $line = <INCAT>) {
		    chomp($line);
		    my @entry = split(/\,/, $line);
		    my $key = shift @entry;
		    foreach my $sentence (@entry) {
			$initialresult{$target}{$key}{$sentence} = 1;
		    }
		}
		close (INCAT);
	    }
    
# refining of search starts here.
	    
	    if (defined(%{$initialresult{$target}})) {
		foreach my $keyword (@auxkey) {
		    my $lowered = "\L$keyword\E";
		    my @keywordfiles = <$keydir/$lowered>;
		    my %checklist = ();
		    foreach my $file (@keywordfiles) {
			open (INKEY, "$file");
			while (my $line = <INKEY>) {
			    chomp($line);
			    my @entry = split(/\,/, $line);
			    my $key = shift @entry;
			    foreach my $sentence (@entry) {
				$checklist{$key}{$sentence} = 1;
			    }
			}
			close (INKEY);
		    }
		    foreach my $key (keys % {$initialresult{$target}}) {
			if ($mode eq 'publication') {
			    if (!$checklist{$key}) {
				foreach my $sentence (keys % {$initialresult{$target}{$key}}) {
				    delete $initialresult{$target}{$key}{$sentence};
				}
				delete $initialresult{$target}{$key};
			    } else {
				foreach my $sentence (keys % {$checklist{$key} }) {
				    $initialresult{$target}{$key}{$sentence} = 1;
				}
			    }
			} else {
			    foreach my $sentence (keys % {$initialresult{$target}{$key}}) {
				if (!$checklist{$key}{$sentence}) {
				    delete $initialresult{$target}{$key}{$sentence};
				}
			    }
			}
		    }
		}
		foreach my $category (@auxinc) {
		    my %checklist = ();
		    open (INCAT, "$catdir/$category");
		    while (my $line = <INCAT>) {
			chomp($line);
			my @entry = split (/\,/, $line);
			my $key = shift @entry;
			foreach my $sentence (@entry) {
			    $checklist{$key}{$sentence} = 1;
			}
		    }
		    close (INCAT);
		    foreach my $key (keys % {$initialresult{$target}}) {
			if ($mode eq 'publication') {
			    if (!$checklist{$key}) {
				foreach my $sentence (keys % {$initialresult{$target}{$key}}) {
				    delete $initialresult{$target}{$key}{$sentence};
				}
				delete $initialresult{$target}{$key};
			    } else {
				foreach my $sentence (keys % {$checklist{$key} }) {
				    $initialresult{$target}{$key}{$sentence} = 1;
				}
			    }
			} else {
			    foreach my $sentence (keys % {$initialresult{$target}{$key}}) {
				if (!$checklist{$key}{$sentence}) {
				    delete $initialresult{$target}{$key}{$sentence};
				}
			    }
			}
		    }
		}
	    }
	}
    }
    return %initialresult;
}

sub DisplayAsHTML {

    my $query = shift;
    my $page = shift;
    my $pExpandedabs = shift;
    my $pSearchresults = shift;
    my $databaseroot = shift;
    my $selectfilename = shift;
    my $pDisplayoptions = shift;
    my $pDisplayvalues = shift;

    my %scorelist = RetrieveScoreList($pSearchresults);
    my %sortscorelist = SortedListCritDir($pSearchresults, "$databaseroot/yea/");

    my @tablecontent = ();
    my @displayedkeys = ();
    
    # make  headline
    my @columns = DefineColumns($pDisplayoptions);
    push @tablecontent, $query->th([@columns]);
    
    # fill table
    my $counter = 0;
    foreach my $score (sort numerically keys % sortscorelist) {
	foreach my $key (sort keys %{$sortscorelist{$score}}) {
	    $counter++;
	    next unless ($counter > $displayvalues{ResultsPerPage}*($page-1));
	    next unless ($counter <= $displayvalues{ResultsPerPage}*$page);
	    my $expanded = 0;
	    foreach my $element (@$pExpandedabs) { next unless ($element eq $key); $expanded = 1;}
	    my @columns = FillColumns($query, $databaseroot, $selectfilename, $pDisplayoptions,
				      $pSearchresults, $key, $scorelist{$key}, $expanded);
	    push @tablecontent, $query->td([@columns]);
	    push @displayedkeys, $key;
	}
    }
    
    print $query->table({-border => '1', -cellpadding => '4',
			 -cellspacing => '1', -width => '100%'},
			$query->caption($query->b("Search Results")),
			$query->Tr([@tablecontent]));
    
    return @displayedkeys;
}

sub RetrieveScoreList {

    my $pSearchresults = shift;
    
    my %scorelist = ();
    foreach my $target (keys % {$pSearchresults}) {
	foreach my $key (keys % { $$pSearchresults{$target} }) {
	    foreach my $sentence (keys % { $$pSearchresults{$target}{$key} }) {
		$scorelist{$key} += $searchresults{$target}{$key}{$sentence};
	    }
	}
    }
   
    return %scorelist;
}

sub SortScoreList {


    my $pScorelist = shift;
    my %sortscorelist = ();
    foreach my $key (keys % {$pScorelist}) {
	$sortscorelist{$$pScorelist{$key}}{$key} = 1;
    }    

    return %sortscorelist;
}

sub SortedListCritDir {

    my $pSearchresults = shift;
    my $critdir = shift;
    
    my %scorelist = ();
    foreach my $target (keys % {$pSearchresults}) {
	foreach my $key (keys % { $$pSearchresults{$target} }) {
	    my $content = join (" ", GetContents("$critdir/$key"));
	    $scorelist{$key} = $content;
	}
    }

    my %sortscorelist = ();
    foreach my $key (keys % scorelist) {
	$sortscorelist{$scorelist{$key}}{$key} = 1;
    }    
    
    return %sortscorelist;
}

sub numerically { $b <=> $a }

sub DefineColumns {

    my $pDisplayoptions = shift;
    my @columns = ();
    if ($$pDisplayoptions{'Source Type'}) { push @columns, 'Publication type' }
    if ($$pDisplayoptions{Title}) { push @columns, 'Title' }
    if ($$pDisplayoptions{Abstract}) { push @columns, 'Abstract' }
    if ($$pDisplayoptions{Author}) { push @columns, 'Author' }
    if ($$pDisplayoptions{Journal}) { push @columns, 'Journal' }
    if ($$pDisplayoptions{Year}) { push @columns, $query->font({-color => 'red'},'Year') }
    if ($$pDisplayoptions{Citation}) { push @columns, 'Citation' }
    if ($$pDisplayoptions{'Number of matches'}) { push @columns, 'Number of matches' }
    if ($$pDisplayoptions{Select}) { push @columns, 'Select' }

    return @columns;
}

sub FillColumns {

    my $query = shift;
    my $databaseroot = shift;
    my $selectfilename = shift;
    my $pDisplayoptions = shift;
    my $pSearchresults = shift;
    my $key = shift;
    my $score = shift;
    my $expanded = shift;
    my @columns = ();

    if ($$pDisplayoptions{'Source Type'}) { 
	my $content = join ("\n", GetContents("$databaseroot/typ/$key"));
	$content =~ s/\_/ /g;
	push @columns, "\L$content\E";
    }

    if ($$pDisplayoptions{Title}) { 
	my @aux = GetContents("$databaseroot/tit/xml/$key");
	my $content = join ("\n", WormBaseLinks ("$databaseroot/ind/wbl/tit/$key", 
						 "$databaseroot/wbl/", $query, @aux));
	push @columns, $content;
    }
    
    if ($$pDisplayoptions{Abstract}) { 
	my @aux = GetContents("$databaseroot/abs/xml/$key");
	my @neat = WormBaseLinks ("$databaseroot/ind/wbl/abs/$key",
			       "$databaseroot/wbl/", $query, @aux);
	my $content;
	if ($expanded) {
	    $content = join ("\n", @neat);
	    if ($content) {
		$content .= $query->submit(-name => "collapse$key", 
					   -value => 'Collapse abstract');
	    }
	} else {
	    $content = $neat[3] . $neat[4];
	    if ($content) {
		$content .= $query->submit(-name => "expanded$key", 
					   -value => 'Expand abstract');
	    }
	}
	push @columns, $content;
    }
    
    if ($$pDisplayoptions{Author}) { 
	my $content = join ("\n", GetContents("$databaseroot/aut/$key"));
	$content =~ s/\n/<br>/g;
	push @columns, $content;
    }
    
    if ($$pDisplayoptions{Journal}) { 
	my $content = join ("\n", GetContents("$databaseroot/jou/$key"));
	push @columns, $content;
    }

    if ($$pDisplayoptions{Year}) { 
	my $content = join ("\n", GetContents("$databaseroot/yea/$key"));
	push @columns, $content;
    }

    if ($$pDisplayoptions{Citation}) { 
	my $content = join ("\n", GetContents("$databaseroot/cit/$key"));
	$content =~ s/P\:\s+(\d+)\s+(\d+)/Pages $1-$2/g;
	$content =~ s/V\:/<br>Vol\./g;
	push @columns, $content;
    }

    if ($$pDisplayoptions{'Number of matches'}) { 
	push @columns, $score;
    }
    
    if ($$pDisplayoptions{Select}) {
	my $content = $query->start_center;
	$content .= "$key";
	$content .= $query->end_center;
	if (!-e "$databaseroot/$selectfilename\.$key") {
	    open (IN, "$databaseroot/$selectfilename");
	    my $firstline = <IN>;
	    my $secondline = <IN>;
	    close (IN);
	    open (OUT, ">$databaseroot/$selectfilename\.$key");
	    print OUT $firstline;
	    print OUT $secondline; # make sure queries are saved.
	    foreach my $target (keys %{ $pSearchresults}) {
		print OUT $target . "," . $key . ",";
		foreach my  $sentence (keys % { $$pSearchresults{$target}{$key} }) {
		    print OUT $sentence . ",";
		}
		print OUT "\n";
	    }
	    close (OUT);
	}
	(my $selectid = "$selectfilename\.$key") =~ s/\/tmp\/file//g;
	$content .= $query->start_center;
	$content .= $query->a({-href => "showmatches.cgi?id=$selectid", -target => '_blank'}, 
			      $query->img({-src => "/text/gif/vm.jpg", 
					  -border => 0, -alt => 'view matches'}));
	if (-e "$databaseroot/pdf/$key.pdf") {
	    if ($query->remote_host() =~ /(131\.215\.|\.caltech\.edu)/) {
		$content .= $query->a({href => "/text/tdb/pdf/$key.pdf", target => "_blank"}, 
				      $query->img({-src => "/text/gif/pf.jpg", -border => 0, -alt => 'PDF'}));
	    }
	}
	if (-e "$databaseroot/url/$key") {
	    open (URL, "$databaseroot/url/$key");
	    my $url = <URL>;
	    chomp ($url);
	    $content .= $query->a({href => "$url", target => "_blank"}, 
				  $query->img({-src => "/text/gif/ot.jpg", -border => 0, -alt => 'article text from journal'}));	    
	    close(URL);
	}
	if (-e "$databaseroot/rel/$key") {
	    open (URL, "$databaseroot/rel/$key");
	    my $url = <URL>;
	    chomp ($url);
	    $content .= $query->a({href => "$url", target => "_blank"}, 
				  $query->img({-src => "/text/gif/ra.jpg", -border => 0, -alt => 'related articles'}));	    
	    close(URL);
	}
	if (-e "$databaseroot/end/$key") {
	    $content .= $query->a({href => "exportendnote.cgi?mode=sen&id=$key"},
				  $query->img({-src => "/text/gif/en.jpg", -border => 0, -alt => 'Endnote'}));
	}				  
	if (-e "$databaseroot/eab/$key") {
	    $content .= $query->a({href => "exportendnote.cgi?mode=sea&id=$key"},
				  $query->img({-src => "/text/gif/ea.jpg", -border => 0, -alt => 'Endnote (w/ abstract)'}));
	}				  
	$content .= $query->end_center;
	push @columns, $content;
    }

    return @columns;
}

sub GetContents {
    
    my $filename = shift;
    my @return = ();
    open (IN,"$filename");
    while (my $line = <IN>) { 
	chomp ($line);
	push @return, $line;
    }
    close (IN);
    return @return;
    
}

#sub lengthwise { length($b) <=> length($a) }

sub WormBaseLinks {
    
    my $indexfile = shift;
    my $linkdir = shift;
    my $query = shift;
    my @lines = ();
    while (@_) {push @lines, shift;}
    if (-e "$indexfile") {
	my %indexcontent = ();
	open (INDEX, $indexfile);
	while (my $line = <INDEX>) {
	    chomp $line;
	    my @words = split(/\t/, $line);
	    my $id = shift(@words);
	    while (@words) { push @{ $indexcontent{$id} }, shift(@words)};
	}
	close (INDEX);
	for (my $i = 0; $i < @lines; $i++) {
	    if ($lines[$i] =~ /<sentence id=\'(s\d+)\'/) {
		my $id = $1;
		$lines[$i] =~ s/<.+?>//g; # remove xml brackets
		$lines[$i] =~ s/<\/.+?>//g;
		if ($indexcontent{$id}) {
		    while (@{$indexcontent{$id}}) {
			my $term = shift (@{$indexcontent{$id}});
			(my $lookup = $term) =~ s/\s//g;
			my $lcflookup = lcfirst($lookup);
			open (URL, "$linkdir/$lookup") or open(URL, "$linkdir/$lcflookup");
			my $url = <URL>;
			close (URL);
			if ($url) {
			    chomp($url);
			    my $replacement = " " . $query->a({-href => $url, -target => '_blank'}, "$term") . " ";
			    $lines[$i] =~ s/(^|\s)$term($|\s)/$replacement/g;
			}
		    }
		}
	    } else {
		$lines[$i] =~ s/<.+?>//g; # remove xml brackets
		$lines[$i] =~ s/<\/.+?>//g;		
	    }
	}
    }
    return @lines;
}