~n3npq/lsb/distribution-checker

#!/usr/bin/perl -w

# Distribution Checker
# Main Executable Script (dist-checker.pl)
#
# Copyright (C) 2007-2009 The Linux Foundation. All rights reserved.
#
# This program has been developed by ISP RAS for LF.
# The ptyshell tool is originally written by Jiri Dluhos <jdluhos@suse.cz>
# Copyright (C) 2005-2007 SuSE Linux Products GmbH
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# version 2 as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #

use strict;
use IO::Handle; # for autoflush();
use FindBin;
use File::Basename qw(dirname basename);

sub BEGIN {
	# Add current directory to the @INC to be able to include modules
	# from the same directory where this script is located.
	unshift @INC, $FindBin::Bin, $FindBin::Bin.'/Tests';
	
	$| = 1; # Immediate flush of all output
	
	### LSB 3.1 / 16.4. Path For System Administration Utilities ##
	# Certain utilities used for system administration (and other privileged
	# commands) may be stored in /sbin, /usr/sbin, and /usr/local/sbin. 
	# Applications requiring to use commands identified as system 
	# administration utilities should add these directories to their PATH.
	$ENV{PATH} .= ":/sbin:/usr/sbin";
}

use Misc;
Misc::Init();
use Manifest;
use Manifest_gen;
use Packages;
use Download;
use Test_common;
use UserProfile;
use Report;

# Array of tests to run; they will be run in the specified _order_. 
my @tests_to_run = ();

#----------------------------------------------------------------------------

$SIG{__WARN__} = sub {
		die "Error: Compilation error: ".$_[0] if !defined $^S && $_[0]!~/^Error: Compilation error: /; # Compilation error
		
		local $Error::Depth = $Error::Depth + 1;
		local $Error::Debug = 0;
		warning $_[0];
	};

$SIG{__DIE__} = sub {
		die @_ if $^S; # Do nothing if called inside eval{ }
		die "Error: Compilation error: ".$_[0] if !defined $^S && $_[0]!~/^Error: Compilation error: /; # Compilation error
		
		local $Error::Depth = $Error::Depth + 1;
		local $Error::Debug = 0;
		fail "Died. ".$_[0];
	};

my $need_cleanup = 0;

my $_END = sub {
	print_stderr "Finished.\n" if $globals->{'webui'};
};

#=============================================================================
# Subroutines of the main script
#=============================================================================

# Prints a short help text.
sub print_help {
	
	my $text = <<DATA;
 Usage: $0 [OPTIONS] TESTS
 Runs one or more tests and produces HTML report.
 
 Examples:
   Run a set of tests:
     $0 -D -s '$DEFAULT_STANDARD_VER'  cmdchk libchk

   Run all automated certification tests from snaphots:
     $0 -v2 -D -s '$DEFAULT_STANDARD_VER' -S 'snapshot'  all
   
 Options:
  
  -a,--arch <architecture>   Set machine architecture. [autodetect]
  -b,--batch                 Download all files before running tests.
  --cert                     Enable certification mode.
  --check-only               Do some checks and exit.
  -D,--download              Allow downloading needed files from the Internet.
                             Use -Dftp to prefer FTP protocol rather than HTTP.
  --comment '<text>'         Any comments for this test run.
  --force-reinstall          Reinstall packages.
  -h,--help                  Show this help and exit.
  --ignore-check             Ignore failed checks.
  -I,--ignore-unavailable    Don't fail if can't run some tests.
  --list                     List all available tests.
  -M,--mail-to <email>       Send test results to e-mail.
  --not-run                  Do some initialization, then exit (for debug).
  -m,--package-manager <name>   Package manager: 'rpm' or 'dpkg' [autodetect].
  -f,--profile <file>        Take settings from <file>.
  --report <result dir>      Build report for results in directory <result dir>.
  --post-cmd '<cmd>'         Execute '<cmd> <result>.tgz' at the end.
  -s,--standard '$DEFAULT_STANDARD_VER'    Set standard version. [autodetect via lsb_release]
  -S,--status <status>       Choose tests with this status (e.g. 'beta').
  -p,--std-profile <prof>    Use standard profile <prof> (e.g. 'core,c++').
  -r,--testrun-id <name>     Use <name> as a result subdirectory name.
  --update                   Download latest data files and test modules.
  -v,--verbose <N>           Verbose level:
                               0 - quiet, 1 - normal, 2 - verbose, 3 - debug.

 Proxy Settings:
  -x,--proxy [<user>:<password>@]<host>:<port>[,<auth>][,notunnel]   Setup proxy.
                          <auth> - authentication method (see 'man curl')
                          'notunnel' - see '--proxytunnel' on the curl manpage.
  --http-proxy <...>    Proxy settings can be specified separately for HTTP
  --ftp-proxy  <...>      and FTP.
  --no-proxy            Don't use proxy at all.

 WARNING! PLEASE DO NOT USE THIS SCRIPT ON ANY MACHINE THAT HOLDS
 IMPORTANT DATA, OR PROVIDES IMPORTANT SERVICES!
 The tests sometimes put a heavy stress on the tested computer,
 and in rare unfortunate cases they can crash the system or damage
 the stored data. They also are a security risk as they work with
 root priviledges and create users with default passwords.
 
 Please report any bugs and improvements to linux\@ispras.ru.
DATA
	
	print $text;
}

#----------------------------------------------------------------------------

# Read the command line parameters.
sub get_options {
	my @args = @ARGV;
	if ( @args < 1 ) {
		print "Read $0 --help for information about usage.\n";
		exit 0;
	}

	my $params = [];
	push @$params, {LONG_NAME=>'arch',               SHORT_NAME=>'a',	HAS_VALUE=>1};
	push @$params, {LONG_NAME=>'batch',              SHORT_NAME=>'b'};
	push @$params, {LONG_NAME=>'cert'};
	push @$params, {LONG_NAME=>'check-only'};
	push @$params, {LONG_NAME=>'comment'};
	push @$params, {LONG_NAME=>'download',           SHORT_NAME=>'D'};
	push @$params, {LONG_NAME=>'force-reinstall'};
	push @$params, {LONG_NAME=>'help',               SHORT_NAME=>'h'};
	push @$params, {LONG_NAME=>'ignore-check'};
	push @$params, {LONG_NAME=>'ignore-unavailable', SHORT_NAME=>'I'};
	push @$params, {LONG_NAME=>'keep'};  # Internal use
	push @$params, {LONG_NAME=>'list',               SHORT_NAME=>'l'};
	push @$params, {LONG_NAME=>'mail-to',            SHORT_NAME=>'M',	HAS_VALUE=>1};
	push @$params, {LONG_NAME=>'not-run'};
	push @$params, {LONG_NAME=>'package-manager',    SHORT_NAME=>'m',	HAS_VALUE=>1};
	push @$params, {LONG_NAME=>'post-cmd', HAS_VALUE=>1}; # Execute a user-defined command to the result tarball. Warning! The command is executed as root!
	push @$params, {LONG_NAME=>'profile',            SHORT_NAME=>'f',	HAS_VALUE=>1};
	push @$params, {LONG_NAME=>'report', HAS_VALUE=>1};
	push @$params, {LONG_NAME=>'standard',           SHORT_NAME=>'s',	HAS_VALUE=>1};
	push @$params, {LONG_NAME=>'status',             SHORT_NAME=>'S',	HAS_VALUE=>1};
	push @$params, {LONG_NAME=>'std-profile',        SHORT_NAME=>'p',	HAS_VALUE=>1};
	push @$params, {LONG_NAME=>'testrun-id',         SHORT_NAME=>'r',	HAS_VALUE=>1};
	push @$params, {LONG_NAME=>'update'}; # Check for updates of Manifests, problem_db and test modules
	push @$params, {LONG_NAME=>'verbose',            SHORT_NAME=>'v',	HAS_VALUE=>1};
	push @$params, {LONG_NAME=>'webui',              SHORT_NAME=>'w'}; # Internal use
	push @$params, {LONG_NAME=>'proxy',              SHORT_NAME=>'x',	HAS_VALUE=>1};
	push @$params, {LONG_NAME=>'ftp-proxy', HAS_VALUE=>1};
	push @$params, {LONG_NAME=>'http-proxy', HAS_VALUE=>1};
	push @$params, {LONG_NAME=>'no-proxy'};

	my $options = {};
	
	while ( @args ) {
		my $arg = shift @args;
		
		if ( $arg =~ /^-/ ) {
			my $found = 0;
			foreach my $opt ( @$params ) {
				my $long_name = $opt->{'LONG_NAME'};
				my $short_name = $opt->{'SHORT_NAME'};
				my $value = undef;
				
				if ( $long_name && $arg =~ m/^--\Q$long_name\E=(.+)$/ ) {
					$value = $1;
				}
				elsif ( $short_name && $arg =~ m/^-\Q$short_name\E(.+)$/ ) {
					$value = $1;
				}
				elsif ( $long_name && $arg =~ m/^--\Q$long_name\E$/ ) {
					# OK
				}
				elsif ( $short_name && $arg =~ m/^-\Q$short_name\E$/ ) {
					# OK
				}
				else {
					next;
				}
				
				if ( !defined $value ) {
					if ( $opt->{HAS_VALUE} ) {
						$value = shift @args;
						( defined $value ) or return error "Option $arg must be followed by a value";
					} else {
						$value = 1;
					}
				}
				
				my $result_name = ($long_name or $short_name);
				
				$options->{$result_name} = $value;
				$found = 1;
				last;
			}
			if ( !$found ) {
				fail "Unexpected parameter: '$arg'";
			}
		} else {
			# this is not a parameter, so it's probably a test name
			push @{$options->{TESTS}}, $arg;
		}
	}
	return $options;
}
#----------------------------------------------------------------------------

sub Setup {
	# Some global initializations
	$globals->{'start_time'} = time();
	# Process ID
	$globals->{'PID'} = $$;
	# Script directory
	$globals->{'script_directory'} = $FindBin::Bin;
	# Current profile
	$globals->{'std_profile'} = ''; # no profile
	# File with the current run status
	$globals->{'status_file'} = $RESULTS_DIR.'/'.$STATUS_FILE;
	# Ptyshell tool location
	$globals->{'ptyshell'} = $FindBin::Bin."/../ptyshell/ptyshell";
	
	# Parse command line args
	my $options = get_options();
	is_ok($options) or fail "Error in command line parameters", $Error::Last;
	
	# Read the profile before processing other options
	if ( $options->{'profile'} ) {
		my $profile_filename = $options->{'profile'};
		
		if ( !defined $profile_filename || !-f $profile_filename ) {
			fail "Wrong profile filename "
					.(defined $profile_filename ? "'$profile_filename'" : "(undefined)");
		}
		$globals->{'profile_filename'} = $profile_filename;
		
		$globals->{'profile'} = read_profile($profile_filename)
			or fail $UserProfile::profile_error;
	
		is_ok Use_profile_general( $globals->{'profile'}, $options )
			or fail "Failed to use profile '$profile_filename'", $Error::Last;
	}
	
	# --
	
	if ( defined $options->{'verbose'} ) {
		if ( defined $options->{'verbose'} && $options->{'verbose'} =~ /^(\d+)$/ ) {
			$Misc::verbose_lvl = $1;
		} else {
			warning "Wrong -v parameter";
		}
	}
	if ( defined $options->{'list'} ) {
		unless ( defined $options->{'verbose'} ) { $Misc::verbose_lvl = 1; }
		# See actions on --list below
	}
	if ( $options->{'std-profile'} ) {
		$globals->{'std_profile'} = $options->{'std-profile'};
		# Should be checked when the Manifest is loaded.
	}
	if ( $options->{'arch'} ) {
		my $arch = $options->{'arch'};
		$host_info->{'architecture'} = guess_architecture($arch)
			or fail "Wrong architecture: '$arch'."
				." Allowed values are: ".(join ", ", @all_archs).".";
		$host_info->{'machine'} = $arch;
	}
	if ( $options->{'package-manager'} ) {
		my $package_manager = lc $options->{'package-manager'};
		( $package_manager eq 'rpm' || $package_manager eq 'dpkg' )
			or fail "Wrong package manager: '$package_manager'."
				." Supported package managers are 'rpm' and 'dpkg' (using 'alien').";
		$host_info->{'package_manager'} = $package_manager;
	}
	if ( $options->{'status'} ) {
		my $status = $options->{'status'};
		
		$globals->{'status'} = $status;
	} else {
		$globals->{'status'} = 'cert'; # Default status
	}
	if ( $options->{'batch'} ) {
		$globals->{'batch'} = $options->{'batch'};
	}
	if ( $options->{'download'} ) {
		$Download::enabled = $options->{'download'};
		if ( $options->{'download'} ne 1 ) {
			foreach my $s ( split /,/, $options->{'download'} ) {
				if ( $s =~ /^(http|ftp)$/i ) {
					$globals->{'dl_method'} = lc $1; # Used in Manifest::file_link();
				}
				if ( $s =~ /^(wget|curl)$/i ) {
					$Download::tool = lc $1;
				}
				else {
					fail "Can't understand the -D parameter. It should be -D[ftp|http|wget|curl] or comma-separated: -Dftp,wget.";
				}
			}
		}
	}
	if (   defined $options->{'proxy'}
		|| defined $options->{'http-proxy'}
		|| defined $options->{'ftp-proxy'}   )
	{
		local *parse_proxy_options = sub {
			my ($s, $proxy_protocol) = @_;
			
			local *set_proxy_option = sub {
				my ($key, $value) = @_;
				if ( !$proxy_protocol || $proxy_protocol eq "http" ) {
					$Download::options->{'http_'.$key} = $value;
				}
				if ( !$proxy_protocol || $proxy_protocol eq "ftp" ) {
					$Download::options->{'ftp_'.$key} = $value;
				}
			};
			
			# $s = "[user[:password]@]server[:port][,<auth>][,notunnel]"
			my ($proxy_user, $proxy_password);
			my ($proxy_host, $proxy_port);
			if ( $s =~ s/^(?:([A-Za-z0-9\-_.\@]+)(?::(.*))?\@)?([A-Za-z0-9\-_.]+)(?::(\d+))?// ) {
				$proxy_user = $1;
				$proxy_password = $2;
				$proxy_host = $3;
				$proxy_port = $4;
			}
			my ($proxy_auth, $proxy_notunnel);
			if ( $s =~ s/^,(\w+)// ) {
				if ( $1 eq 'notunnel' ) {
					$proxy_notunnel = 1;
				} else {
					$proxy_auth = $1;
					if ( $s =~ s/^,notunnel// ) {
						$proxy_notunnel = 1;
					}
				}
			}
			if ( $s ne "" ) {
				fail "Wrong --proxy option. Can't parse '$s'";
			}
			if ( !$proxy_host ) {
				fail "Wrong --proxy option. Expected: [http://][user[:password]\@]server[:port]";
			}
			set_proxy_option('proxy_host', $proxy_host);
			
			$proxy_port ||= 1080; # default proxy port
			set_proxy_option('proxy_port', $proxy_port);
			
			set_proxy_option('proxy_user', $proxy_user);
			set_proxy_option('proxy_password', $proxy_password);
			
			set_proxy_option('proxy_auth', $proxy_auth);
			set_proxy_option('proxy_notunnel', $proxy_notunnel);
		}; # end of local *parse_proxy_options()
		
		if ( defined $options->{'proxy'} ) {
			parse_proxy_options( $options->{'proxy'} );
		}
		if ( defined $options->{'http-proxy'} ) {
			parse_proxy_options( $options->{'http-proxy'}, 'http' );
		}
		if ( defined $options->{'ftp-proxy'} ) {
			parse_proxy_options( $options->{'ftp-proxy'}, 'ftp' );
		}
	}
	if ( $options->{'no-proxy'} ) {
		delete $Download::options->{'http_proxy_host'};
		delete $Download::options->{'ftp_proxy_host'};
		delete $ENV{$_} for ( qw/http_proxy ftp_proxy HTTP_PROXY FTP_PROXY HTTPS_PROXY ALL_PROXY/ );
	}
	if ( $options->{'mail-to'} ) {
		$globals->{'mailto'} = $options->{'mail-to'};
	}
	if ( $options->{'force-reinstall'} ) {
		$globals->{'force_reinstall'} = $options->{'force-reinstall'};
	}
	if ( $options->{'check-only'} ) {
		$globals->{'check_only'} = $options->{'check-only'};
	}
	if ( $options->{'ignore-check'} ) {
		$globals->{'ignore_check'} = $options->{'ignore-check'};
	}
	if ( $options->{'testrun-id'} ) {
		$globals->{'testrun_id'} = $options->{'testrun-id'};
	}
	if ( $options->{'webui'} ) {
		$globals->{'webui'} = $options->{'webui'};
	}
	if ( $options->{'post-cmd'} ) {
		$globals->{'post_cmd'} = $options->{'post-cmd'};
	}
	if ( $options->{'comment'} ) {
		$globals->{'run_comments'} = $options->{'comment'};
	}
	# ---
	
	if ( $options->{'help'} ) {
		print_help();
		exit 0;
	}
	if ( $options->{'update'} ) {
		is_ok download_setup() or fail $Error::Last;

		is_ok update()
		or do {
			complain $Error::Last;
			exit($Error::Last->{-ret} || 1);
		};
		if ( (!$options->{TESTS} || !@{$options->{TESTS}})
			&& !$options->{'cert'} )
		{
			exit 0;
		}
	}
	if ( $options->{'report'} ) {
		my $dir = $options->{'report'};
		if ( $dir !~ m{/} ) { $dir = $RESULTS_DIR."/".$dir; }
		
		rebuild_report($dir, $options->{'keep'});
		
		exit 0;
	}
	#---

	# The --cert option overrides some parameters
	if ( $options->{'cert'} ) {
		$globals->{'cert_mode'} = 1;

		$Download::enabled = 1;
		$options->{'ignore-unavailable'} = 0;
		$globals->{'status'} = 'cert';
		$options->{TESTS} = ['all', 'manual'];
		
		$globals->{'force_reinstall'} = 1;
	}

	if ( $Download::enabled ) {
		is_ok download_setup() or fail $Error::Last;
	}

	# Load the Manifest
	debug_inform "Loading manifest...";
	my $manifest = $globals->{'manifest'} = load_manifest();
	is_ok $manifest or fail "Failed to load manifest.", $Error::Last;
	if ( (!$manifest->{ALL}{STANDARDS} || !@{$manifest->{ALL}{STANDARDS}})
		|| $globals->{'cert_mode'} ) # If manifest update is required
	{
		# Refresh the manifest.
		if ( $Download::enabled ) {
			is_ok update()
				or fail $Error::Last;
			# Load the Manifest again
			$manifest = $globals->{'manifest'} = load_manifest();
			is_ok $manifest or fail "Failed to load manifest.", $Error::Last;
			if ( !$manifest->{ALL}{STANDARDS} || !@{$manifest->{ALL}{STANDARDS}} ) {
				fail "Manifest is empty.";
			}
		}
		else {
			fail "Manifest is empty. Please update the Manifest using the --update option.";
		}
	}
	debug_inform "...done.";

	get_profile_file_options($globals->{'profile'}, $manifest) if $globals->{'profile'};

	if ( $options->{'standard'} ) {
		my $selected = undef;
		foreach my $standard ( @{$manifest->{ALL}{STANDARDS}} ) {
			if ( (join"",split/ /,$options->{'standard'}) eq (join"",split/ /,$standard) ) {
				$selected = $standard;
				last;
			}
		}
		if ( $selected ) {
			$globals->{'standard'} = $selected;
		} else {
			fail "Wrong standard name: '".$options->{'standard'}."'.";
		}
	}
	
	host_check_and_info(); # get the host info
	$Packages::package_manager = $host_info->{'package_manager'};

	# Fix status autodetection
	# Should go before list_available_versions()
	if ( !$options->{'status'} ) {
		if ( !in_array($globals->{'status'}, @{$manifest->{ALL}{STATUSES}}) ) {
			if ( $globals->{'cert_mode'} ) {
				fail "No tests for certification was found in Manifest.";
			} else {
				$globals->{'status'} = $manifest->{ALL}{STATUSES}[0];
			}
		}
	}
	if ( !$globals->{'status'} ) {
		fail "No tests was found in Manifest.";
	}
	# Fix standard autodetection
	if ( !$options->{'standard'} ) {
		if ( !in_array($globals->{'standard'}, @{$manifest->{ALL}{STANDARDS}}) ) {
			($globals->{'standard'}) = (reverse sort @{$manifest->{ALL}{STANDARDS}}); # greatest standard
		}
	}
	if ( !$globals->{'standard'} ) {
		fail "No tests was found in Manifest.";
	}

	if ( $options->{'list'} ) {
		is_ok list_available_versions() or fail $Error::Last;
		exit 0;
	}
	
	# Print some settings
	my $command_line = "$0 @ARGV";
	inform "==========================================";
	inform $DC_BRANCH." Distribution Checker v. ".$DC_VERSION;
	inform format_time( $globals->{'start_time'} );
	inform "Command line: $command_line" if $command_line ne "";
	inform "Script directory is ".$FindBin::Bin;
	inform "Host name: ".$host_info->{'hostname'};
	inform "Host machine: ".$host_info->{'machine'};
	inform "Host architecture: ".$host_info->{'architecture'};
	inform "Host kernel: ".$host_info->{'kernel'};
	inform "Host OS: ".$host_info->{'OS'};
	inform "Host package manager: ".$host_info->{'package_manager'};
	inform "==========================================";

	inform "Target: ".$globals->{'standard'}.' ('.$globals->{'status'}.')';
	
	@tests_to_run = ();

	my $arch = $host_info->{'architecture'};
	my $standard = $globals->{'standard'};
	my $status = $globals->{'status'};
	$globals->{'std_profile'} = ($manifest->{STD_PROFILES}{$standard}{DEFAULT} or "");
	my @add = @{$options->{TESTS}} if $options->{TESTS};
	my @unavailable_errors = ();
	while ( @add ) {
		my $test_name = shift @add;
		($test_name, my $test_pref_ver) = split /:/, $test_name, 2;
		
		# Alias 'all'
		if ( $test_name eq 'all' ) {
			for my $testsuite ( sort keys %{$manifest->{TREE}{$standard}} ) {
				next if $manifest->{MANUAL}{$testsuite}; # All but manual
				if ( !glob_hash($manifest->{COMPOSITION}, $standard, $testsuite, $arch, $status ) ) {
					debug_inform "Test '$testsuite' isn't available for $standard, '$status', $arch.";
					next; # Skip unavailable
				}
				if ( $globals->{'std_profile'} ) {
					my $std_profile = $globals->{'std_profile'};
					if ( !glob_hash($manifest->{STD_PROFILES}, $standard, 'TESTS', $testsuite,$std_profile ) ) {
						# Filter by std_profile
						debug_inform "Test '$testsuite' isn't available for $standard, '$std_profile' profile.";
						next; # Skip
					}
				}
				push @add, $testsuite;
			}
			next;
		}
		# Alias 'manual'
		if ( $test_name eq 'manual' ) {
			for my $testsuite ( sort keys %{$manifest->{MANUAL}} ) {
				if ( !glob_hash($manifest, 'COMPOSITION', $standard, $testsuite, $arch, $status) ) {
					debug_inform "Test '$testsuite' isn't available for $standard, '$status', $arch.";
					next; 
				}
				if ( $globals->{'std_profile'} ) {
					my $std_profile = $globals->{'std_profile'};
					if ( !glob_hash($manifest->{STD_PROFILES}, $standard, 'TESTS', $testsuite, $std_profile) ) {
						# Filter by std_profile
						debug_inform "Test '$testsuite' isn't available for $standard, '$std_profile' profile.";
						next; # Skip
					}
				}
				push @add, $testsuite;
			}
			next;
		}
		# Check if the test is specified in the Manifest
		if ( !in_array( $test_name, @{$manifest->{ALL}{TESTSUITES}} ) ) {
			my $error = error "Test '$test_name' is unknown."
					." Use --list option to get the list of available tests.";
			push @unavailable_errors, $error;
			next;
		}
		if ( $globals->{'std_profile'} 
			&& !glob_hash($manifest->{STD_PROFILES}, $standard, 'TESTS', $test_name, $globals->{'std_profile'}) )
		{ # Filter by std_profile
			my $error = error "Test '$test_name' isn't available"
					.($globals->{'std_profile'} ? " for profile '".$globals->{'std_profile'}."'" : "")
					.". Use --list option to get the list of available tests.";
			push @unavailable_errors, $error;
			next;
		}
		
		# Init test info according to Manifest
		my $test_obj = new Test_common(NAME => $test_name);
		$test_obj->{TREE} = $manifest->{TREE}{$standard}{$test_name}
			or fail error("Test '$test_name' is unavailable for $standard",
				error "Tree entry isn't specified in the Manifest "
					." for test '$test_name', $standard.");
	
		# ->{DESCRIPTIONS}{$test_name}[]->{STANDARD|DISPLAYNAME|DESCRIPTION} = ".."
		my ($descr) = grep {version_filter($_->{STANDARD}, $standard)} @{$manifest->{'DESCRIPTIONS'}{$test_name}};
		if ( !$descr ) {
			fail "There is no description in the Manifest"
						." for test '$test_name', $standard.";
		}
		
		# Description & position in the TOC tree
		( $descr->{DISPLAYNAME} ) or warning "DISPLAYNAME for '$test_name' is not defined";
		$test_obj->{DISPLAYNAME} = $descr->{DISPLAYNAME};
		
		( $descr->{DESCRIPTION} ) or warning "DESCRIPTION for '$test_name' is not defined";
		$test_obj->{DESCRIPTION} = $descr->{DESCRIPTION};
		
		$test_obj->{VERSION_MNF} = $test_pref_ver if defined $test_pref_ver;
		
		# Select the test version, files for installation and set up the test options
		my $res = Choose_test_version( $test_obj );
		if ( !is_ok($res) || !defined $test_obj->{VERSION} ) {
			my $error = error "Test '$test_name' can't be run with the current settings.", $res;
			push @unavailable_errors, $error;
			next;
		}
		prepare_test_options($test_obj);
		
		push @tests_to_run, $test_obj;
	}
	if ( @unavailable_errors ) {
		my $error = join_errors(@unavailable_errors);
		if ( $options->{'ignore-unavailable'} ) {
			warning($error);
		} else {
			fail($error);
		}
	}
	if ( !@tests_to_run ) {
		fail "No tests to run (use 'all' to run all available tests).";
	}

	is_ok list_tests_to_run() or fail $Error::Last;

	if ( $options->{'not-run'} ) { exit 0; }
	
	# Initialization
	
	# TODO: what WebUI will do if dist-checker.pl fails here or earlier?
	
	# Complain if not root
	( $> == 0 ) or fail "Please run this script as root.\n";
	
	inform "Initialization...";
	
	if ( !defined $globals->{'testrun_id'} ) {
		my ($sec, $min, $hour, $mday, $mon, $year) = localtime( $globals->{'start_time'} );
		$globals->{'testrun_id'} = 
			 ( defined $host_info->{'architecture'} ? $host_info->{'architecture'} : "" )
			.'-'.(defined $host_info->{'hostname'} ? $host_info->{'hostname'} : "" )
			.'-'.sprintf( '%04d-%02d-%02d-%02dh-%02dm-%02ds',
					$year + 1900, $mon + 1, $mday, $hour, $min, $sec );
	}

	## Check the PtyShell tool
	if ( !-x $globals->{'ptyshell'} ) {
		return error "The ptyshell utility (".$globals->{'ptyshell'}.")"
				." doesn't exists or isn't executable.";
	}
	
	# Load test modules and create test objects
	my $error_load_modules = undef;
	my @module_load_errors = ();
	foreach my $test_obj ( @tests_to_run ) {
		# TODO: load all test modules for debug purposes.
		
		my $test_name = $test_obj->{NAME};
		
		my $module_file = main_module_file($manifest, $test_name, $test_obj->{VERSION}, $standard);
		if ( !is_ok($module_file) ) {
			my $error = error "Failed to prepare modules for '$test_name'.", $Error::Last;
			push @module_load_errors, $error;
			next;
		}
		my $class_name = load_test_module($module_file);
		if ( !is_ok($class_name) ) {
			my $error = error "Failed to load module '$module_file'. Please check for a new version of the Distribution Checker.", $Error::Last;
			push @module_load_errors, $error;
			next;
		}
		# create test object
		$test_obj = $class_name->new( %$test_obj );
	}
	if ( @module_load_errors ) {
		my $error = join_errors(@module_load_errors);
		if ( $options->{'ignore-unavailable'} ) {
			warning($error);
		} else {
			fail($error);
		}
	}
	$globals->{'verdict'} = 'incomplete' if grep !$_->{MANUAL}, @tests_to_run;
	$globals->{'verdict_man'} = 'incomplete' if grep $_->{MANUAL}, @tests_to_run;
	
	# Create needed dirs and initialize logging
	is_ok setup_dirs_and_logs() or return $Error::Last;
	
	inform "Initialization done.";
	inform "Result dir: '".$globals->{'result_dir'}."'" if $globals->{'result_dir'}; # The first line in log

	if ( defined $globals->{'profile_filename'} && $globals->{'result_dir'} ) {
		cmd( "cp -f ".shq($globals->{'profile_filename'})." ".shq($globals->{'result_dir'}."/profile") ) == 0
			or warning "Failed to copy profile: '".$globals->{'profile_filename'}."' to '".$globals->{'result_dir'}."'";
	}
	
	return 1;
}
#----------------------------------------------------------------------

# Print the tests to be run.
sub list_tests_to_run {
	
	my $packages = {};
	foreach my $test_obj ( @tests_to_run ) {
		check_files( values %{$test_obj->{FILES}} );
		
		if ( $test_obj->{REQUIREMENTS} ) {
			for my $req ( @{$test_obj->{REQUIREMENTS}} ) {
				my $file = $test_obj->{FILES}{$req->{PACKAGE}};
				if ( !$file || $file->{KIND} ne 'file' ) {
					$packages->{$req->{PACKAGE}} = 1;
				}
			}
		}
	}
	$packages = packages_info($packages);
	is_ok($packages) or return error $Error::Last;

	# The following subroutine splits decimal number by 3 digit for easier reading.
	# E.g. "1234567" => "12,345,678"
	local *split_num = sub{"".reverse+join",",reverse(shift)=~/.{1,3}/g};
	my $to_download = {};
	my $architecture = $host_info->{'architecture'};

	inform "Tests to run:";
	foreach my $test_obj ( @tests_to_run ) {
		my $test_name = $test_obj->{NAME};
		
		inform "  ".$test_obj->{NAME}." ".($test_obj->{VERSION} || 'undefined' );

		if ( !defined $test_obj->{REQUIREMENTS} || !@{$test_obj->{REQUIREMENTS}} ) {
			inform "     No files.";
			next;
		}

		# Print more detailed information about files to be installed
		for my $req ( @{$test_obj->{REQUIREMENTS}} ) {
			my $s = "";
			$s .= "     ".$req->{PACKAGE};
			$s .= " v. ".$req->{PACKAGE_VER} if $req->{PACKAGE_VER} && $req->{PACKAGE_VER} ne '*';

			my $file = $test_obj->{FILES}{$req->{PACKAGE}};
			my $file_key;
			$file_key = $file->{SRC}."/".$file->{FILENAME} if defined $file;
			
			$s .=  " (";
			if ( $file && !$file->{DOWNLOADED} ) {
				$s .=  "download ".split_num($file->{SIZE})." bytes";
				$to_download->{$file_key} = $file->{SIZE};
			}
			elsif ( $file && $file->{DOWNLOADED} == -1 ) {
				$s .=  "another version: ".split_num($file->{SIZE})." bytes";
				$to_download->{$file_key} = $file->{SIZE};
			}
			else {
				my $installed_ver = undef;
				if ( defined $packages->{$req->{PACKAGE}}{$architecture} ) {
					($installed_ver) = grep {version_filter($req->{PACKAGE_VER}, $_)} keys %{$packages->{$req->{PACKAGE}}{$architecture}};
				}
				if ( $installed_ver ) {
					if ( $installed_ver ne ($req->{PACKAGE_VER} or "") ) {
						$s .= $installed_ver." ";
					}
					$s .=  "installed";
				}
				elsif ( $file && $file->{DOWNLOADED} == 1 ) {
					$s .=  "downloaded";
				}
				else {
					$s .=  "not installed";
				}
			}
			$s .=  ")";
			inform $s;
		}
	}
	
	my $total_dl_size = 0;
	$total_dl_size += $_ for values %$to_download;
	inform "Total download size: ".split_num($total_dl_size)." bytes.";
}
#----------------------------------------------------------------------

# Checks whether the specified tool is available, terminating immediately
# if it is not.
sub require_tool {
	my ($tool_name) = @_;
	
	# Look if the shell knows this tool.
	unless ( does_shell_know($tool_name) ) {
		fail "Couldn't find a required tool: '$tool_name'."
		   ."\nYou could install it using the tools provided by your distro.";
	}
}
#----------------------------------------------------------------------

# This function creates an object that works with the test suite.
sub load_test_module {
	my ($file) = @_;
	
	debug_inform "Loading module: $file";

	my $dir = $globals->{'script_directory'}."/Tests";
	unshift @INC, $dir if !in_array($dir, @INC);
	
	# Link up the module
	eval {
		require $file;
	};
	if ($@) { # Any errors?
		return error $@;
	}
	
	my $class_name = $file;
	$class_name =~ s/\.pm$//;
	
	return $class_name;
}
#-----------------------------------------------------------------------

# Select test version, files for installation and set up the test options
sub Choose_test_version {
	my ($test_obj) = @_;

	my $test_name = $test_obj->name;

	my $manifest = $globals->{'manifest'};

	my $architecture = $host_info->{'architecture'};
	my $standard = $globals->{'standard'};
	my $std_profile = $globals->{'std_profile'};

	my $profile = $globals->{'profile'};
	my $profile_entry = undef;
	$profile_entry = $profile->{TEST_SUITES}{$test_name} if defined $profile;
	if ( defined $profile_entry ) { # User profile
		$test_obj->{VERSION_MNF} = $profile_entry->{VERSION_MNF} if defined $profile_entry->{VERSION_MNF};
		$test_obj->{STATUS}  = $profile_entry->{STATUS} if defined $profile_entry->{STATUS};
	}

	if ( !defined $test_obj->{STATUS} ) {
		$test_obj->{STATUS} = $globals->{'status'};
	}
	my $status = $test_obj->{STATUS};

	if ( !$test_obj->{VERSION_MNF} ) {
		$test_obj->{VERSION_MNF} = select_test_version_by_status($manifest, $standard, $test_name, $architecture, $status);

		if ( !defined $test_obj->{VERSION_MNF} ) {
			return error "$test_name test isn't supported for"
				." $standard, arch '$architecture', status '$status'.";
		}
	}

	inform "Checking packages for '$test_name' tests.";
	is_ok $test_obj->prepare_test_packages( CHECK_ONLY => 1 )
		or return $Error::Last;
	
	$test_obj->{VERSION} = $test_obj->{VERSION_MNF};
	if ( $test_obj->{VERSION} eq '*' ) {
		my $req = $test_obj->{REQUIREMENTS}[-1]
			or return error "Can't select test version for '$test_name' (no reqs),"
				." $standard, arch '$architecture', status '$status'.";
		
		$test_obj->{VERSION} = $test_obj->{PACKAGES}{$req->{PACKAGE}}
			or return error "Can't select test version for '$test_name' (no package),"
				." $standard, arch '$architecture', status '$status'.";
		$test_obj->{VERSION} =~ s/\.lsb\d*$//; # remove ".lsb4" at the end
	}

	return 1;
}

sub prepare_test_options {
	my ($test_obj) = @_;

	my $test_name = $test_obj->{NAME};

	my $manifest = $globals->{'manifest'};
	my $standard = $globals->{'standard'};
	my $profile_entry = $globals->{'profile'}{TEST_SUITES}{$test_name} if defined $globals->{'profile'};

	if ( defined $manifest->{OPTIONS}{$test_name} ) {
		my $version = $test_obj->{VERSION};
		my @options = grep { version_filter($_->{TEST_VER}, $version) 
				&& version_filter($_->{STANDARD}, $standard) } @{$manifest->{OPTIONS}{$test_name}};
		
		foreach my $opt ( @options ) {
			my $ID = $opt->{ID};
			
			( !defined $test_obj->{OPTIONS}{$ID} )
				or warning "Double option: '$ID'";
			
			# Copy the option with (a default value) from the Manifest
			$test_obj->{OPTIONS}{$ID} = copy( $opt );
		}
		
		# Get option values from the profile
		if ( defined $profile_entry ) {
			my $values = $profile_entry->{OPTIONS};
			if ( $values && %$values ) {
				foreach my $ID ( keys %$values ) {
					if ( !defined $test_obj->{OPTIONS}{$ID} ) {
						warning "Unused option for '$test_name': '$ID'";
						next;
					}
					$test_obj->{OPTIONS}{$ID}->{VALUE} = $values->{$ID};
				}
			}
			
			$values = $profile_entry->{AUTO_REPLIES}->{$test_obj->{STATUS}}->{$test_obj->{VERSION_MNF}};
			if ( $values ) {
				foreach my $ID ( keys %$values ) {
					if ( !defined $test_obj->{OPTIONS}{$ID} ) {
						warning "Unused option for '$test_name': '$ID'";
						next;
					}
					$test_obj->{OPTIONS}{$ID}->{VALUE} = $values->{$ID};
				}
			}
		}
	} # end of  if ( defined $manifest->{OPTIONS}{$test_name} )
	
	return 1;
}
#----------------------------------------------------------------------

sub select_test_version_by_status {
	my ($manifest, $standard, $test_name, $architecture, $status) = @_;
	
	# Manifest structure:
	# ->{COMPOSITION}{$standard}{$testsuite}{$arch}{$status}->{$test_ver}[]->{PACKAGE|PACKAGE_VER|LOCATION} = ".."
	
	my $test_vers = glob_hash($manifest->{COMPOSITION}, $standard, $test_name, $architecture, $status);
	( $test_vers ) or return undef;
	
	my $version_max = (sort { vercmp($a,$b) } keys %$test_vers)[-1];
	
	return $version_max;
}
#----------------------------------------------------------------------

# Do initial environment checks and read information about the host machine.
# Terminates the script if any of the checks fails.

sub host_check_and_info {
	# Check for basic utilities we will need.
	require_tool("uname"); # Included since LSB 1.0
	
	if ( !defined $host_info->{'OS'} ) {
		$host_info->{'OS'} = detect_OS();
	}
	
	if ( !defined $globals->{'standard'} ) {
		$globals->{'standard'} = detect_standard_version($host_info->{'OS'});
	}
	
	if ( ! does_shell_know("lsb_release") ) {
		my $lsb_pkg_name = 'lsb';
		if ( $DC_BRANCH eq 'Moblin' ) { $lsb_pkg_name = 'moblin-lsb'; }
		# This tool is required by LSB
		complain "Couldn't find the 'lsb_release' tool."
			." It is usually provided by the '$lsb_pkg_name' package. Please install it first.";
		fail() unless $globals->{'ignore_check'};
	}
	
	if ( $globals->{'standard'} =~ /^LSB/ ) {
		if ( capture ("lsb_release 2>&1") =~ /No LSB modules/ ) {
			complain "No LSB modules are available. Expected that lsb_release prints available LSB modules.";
			fail() unless $globals->{'ignore_check'};
		}
	}
	
	if ( $globals->{'mailto'} ) {
		if ( !does_shell_know("/usr/sbin/sendmail") ) {
			complain "Couldn't find '/usr/sbin/sendmail'. This tool is used for sending results by e-mail.";
			fail() unless $globals->{'ignore_check'};
			# else
			$globals->{'mailto'} = undef;
		}
	}
	
	my $tmp = "";
	
	if ( !defined $host_info->{'machine'} ) {
		# The machine type and architecture are discovered via uname.
		$tmp = trim(`uname -m`);
		if ( $tmp ne "" ) {
			$host_info->{'machine'} = $tmp;
		} else {
			complain "Failed to recognize the machine type."
				."\nUse the --arch option to specify the machine type manually on the command line";
			fail() unless $globals->{'ignore_check'};
			# else
			$host_info->{'machine'} = 'x86'; # fallback
		}
	}
	
	if ( !defined $host_info->{'hostname'} ) {
		# Get the host name.
		$tmp = trim(`uname -n`);
		if ( $tmp ne "" ) {
			$host_info->{'hostname'} = $tmp;
		} else {
			complain "Failed to obtain the host name."
					."\n'uname -n' command should print the host name.";
			fail() unless $globals->{'ignore_check'};
			# else
			$host_info->{'hostname'} = 'undefined'; # fallback
		}
	}
	
	if ( !defined $host_info->{'architecture'} ) {
		$host_info->{'architecture'} = detect_architecture()
			or fail "Failed to recognize the machine architecture."
				."\nSupported architectures are ".(join ", ", @all_archs)."."
				."\nUse the --arch option to specify the architecture manually on the command line"
				."\nYou can identify the architecture using the command 'uname -m'.";
	}
	
	if ( !defined $host_info->{'kernel'} ) {
		# Get the kernel version
		$tmp = trim(`uname -r -v`);
		$host_info->{'kernel'} = $tmp;
	}
	
	# Read the /etc/issue file, if present. Sometimes it is the only place
	# where an exact OS version can be found.
	if ( !defined $host_info->{'etc_issue'} && -f "/etc/issue" ) {
		$tmp = read_file("/etc/issue");
		if ( is_ok($tmp) ) {
			$tmp =~ s/\s+$//; # Remove trailing spaces
			$host_info->{'etc_issue'} = $tmp;
		}
	}
	
	if ( !defined $host_info->{'package_manager'} ) {
		$host_info->{'package_manager'} = guess_package_manager( $host_info->{'OS'} )
			or fail "Couldn’t recognize the package manager"
					." for this distro (".$host_info->{'OS'}.")."
					."\nUse the -pm option to specify the package manager manually on the command line"
					."\nSupported package managers are 'rpm' and 'dpkg' (using 'alien').";
	}
	
	# rpm is required even if the package manager is dpkg
	require_tool("rpm");
	if ( $host_info->{'package_manager'} eq "dpkg") {
		# Check for alien and dpkg
		require_tool("alien"); # Alien is used to install rpm packages
		require_tool("dpkg");
	}
}
#----------------------------------------------------------------------

sub setup_dirs_and_logs {
	
	# Default temporary directory
	$globals->{'temp_dir'} = $DEFAULT_TEMP_DIR;
	is_ok create_empty_directory( $globals->{'temp_dir'} )
		or return error "Failed to create temp dir", $Error::Last;
	
	return 1 if $globals->{'check_only'};
	
	if ( ! -d $RESULTS_DIR ) {
		is_ok create_empty_directory( $RESULTS_DIR )
			or return error "Failed to create result dir: ".$RESULTS_DIR;
	}
	
	# Results subdir
	$globals->{'result_dir'} = $RESULTS_DIR."/".$globals->{'testrun_id'};
	if ( ! -d $globals->{'result_dir'} ) {
		# Create a subdir in the result directory
		mkdir $globals->{'result_dir'}
			or return error "Failed to create dir '".$globals->{'result_dir'}."': $!";
		is_ok append_string_to_file( '!'.$RESULTS_DIR."/HISTORY", $globals->{'testrun_id'}."\n" ) # '!' tells that flock should be used
			or return error "Failed to write to the HISTORY file", $Error::Last;
	}
	$Report::result_dir = $globals->{'result_dir'};
	
	$globals->{'test_result_dir'} = $globals->{'result_dir'}."/results";
	if ( ! -d $globals->{'test_result_dir'} ) {
		# Create directory in the result repository
		mkdir $globals->{'test_result_dir'}
			or return error "Failed to create dir '".$globals->{'test_result_dir'}."': $!";
	}
	
	## Setup logs

	is_ok Misc::setup_logging() or return $Error::Last;
	
	return;
}
#----------------------------------------------------------------------

sub Benchmark {
	
	my $i;
	my $res;

	$SIG{ALRM} = sub { $res = $i; }; # Used to stop benchmarking after N second (alarm function)

	# Benchmark calculations
	my $r = 20;
	$i = 1; $res = undef; alarm 1;
	while ( !defined $res ) {  $i++;
		$r = length($r*$i*$i*$i); # arithmetic operations
	}
	$host_info->{'bench_c'} = $res/1000;
		
	# Benchmark process spawning
	$i = 1; $res = undef; alarm 2;
	while ( !defined $res ) {  $i++;
		`cat /etc/issue`; # spawn processes
	}
	$host_info->{'bench_p'} = $res;

	$SIG{ALRM} = 'DEFAULT';
	
	# Print benchmark results
	inform "Benchmark C: ".$host_info->{'bench_c'};
	inform "Benchmark P: ".$host_info->{'bench_p'};
	
	return 1;
}
#----------------------------------------------------------------------

sub write_runconfig {

	my $content = "";
	
	local *add_var = sub {
		my ( $name, $value ) = @_;
		
		# check
		if ( !defined $value ) {
			warning "$name value is undefined";
			$value = "";
		}
		
		$value =~ s{[\n\x0D]+}{\x81}g;
		
		$content .= $name.": ".$value."\n";
		
		return 1;
	};
	
	add_var( 'STANDARD', $globals->{'standard'} );
	add_var( 'CERT', $globals->{'cert_mode'} ) if $globals->{'cert_mode'};
	add_var( 'RUN_COMMENTS', $globals->{'run_comments'} ) if $globals->{'run_comments'};
	
	# Save the $host_info
	foreach my $key ( keys %$host_info ) {
		add_var( 'HOST_'.$key, $host_info->{$key} );
	}
	
	add_var( 'TESTER_NAME', $globals->{'tester_name'} ) if $globals->{'tester_name'};
	add_var( 'ORGANIZATION', $globals->{'tester_org'} ) if $globals->{'tester_org'};
	
	foreach my $test_obj ( @tests_to_run ) {
		
		$content .= "\n[".$test_obj->{'NAME'}."]\n";

		add_var( 'VERSION_MNF', $test_obj->{'VERSION_MNF'} ) if $test_obj->{'VERSION_MNF'} ne $test_obj->{'VERSION'};
		add_var( 'VERSION', $test_obj->{'VERSION'} );
		add_var( 'STATUS', $test_obj->{'STATUS'} );
		add_var( 'TREE', $test_obj->{'TREE'} );
		add_var( 'DISPLAYNAME', $test_obj->{'DISPLAYNAME'} );
		add_var( 'STATE', $test_obj->{'STATE'} );
		add_var( 'VERDICT', $test_obj->{'VERDICT'} ) if $test_obj->{'VERDICT'};
	}
	
	is_ok write_string_as_file( $globals->{'result_dir'}."/runconfig", $content )
		or return $Error::Last;
	
	return;
}

# Write the INFO file
sub write_INFO {
	my $INFO = "";
	
	$INFO .= "Verdict: ".Report::verdict_text($globals->{'verdict'})."\n" if $globals->{'verdict'};
	$INFO .= "Verdict-manual: ".Report::verdict_text($globals->{'verdict_man'})."\n" if $globals->{'verdict_man'};
	$INFO .= "Certification\n" if $globals->{'cert_mode'};
	
	$INFO .= $globals->{'standard'}." tests:\n" if $globals->{'standard'};
	
	foreach my $test_obj ( @tests_to_run ) {
		next if $test_obj->{MANUAL};
		$INFO .= " ".$test_obj->{NAME};
		$INFO .= " - ".$test_obj->{VERDICT} if $test_obj->{VERDICT};
		$INFO .= ";\n";
	}
	if ( grep $_->{MANUAL}, @tests_to_run ) {
		$INFO .= " Manual tests";
		$INFO .= " - ".$globals->{'verdict_man'} if $globals->{'verdict_man'};
		$INFO .= ";\n";
	}
	
	$INFO .= "\n";
	
	$INFO .= $globals->{'run_comments'} if $globals->{'run_comments'};
	
	is_ok write_string_as_file( $globals->{'result_dir'}."/INFO", $INFO )
		or return $Error::Last;
	
	return;
}
#-----------------------------------------------------------------------

# Write the status file. See TestStatus.pm for details.
sub write_status {
	my $content = "";
	
	$content .= "PID = ".$globals->{'PID'}."\n";
	$content .= "RESULT_DIR = ".$globals->{'testrun_id'}."\n" if defined $globals->{'testrun_id'}; # undef if --check-only
	$content .= "CERTIFICATION = ".($globals->{'cert_mode'} ? 1 : 0)."\n";
	$content .= "STATUS = ".($globals->{'done'} ? $globals->{'done'} : 'Running')."\n";
	
	$content .= "CURRENT_TIME = ".time()."\n";
	
	$content .= "START_TIME = ".$globals->{'start_time'}."\n";
	
	# Tests to run:
	foreach my $test_obj ( @tests_to_run ) {
		
		$content .= "\n[".$test_obj->{NAME}."]\n";
		
		$content .= "NAME = ".($test_obj->{DISPLAYNAME} or $test_obj->{NAME})."\n";
		
		my $status = $test_obj->{STATE};
		if ( $status eq 'Finished' ) {
			$status = 'No verdict'; # in case there are no $test_obj->{VERDICT} yet
			
			if ( $test_obj->{VERDICT} ) {
				if ( $test_obj->{VERDICT} eq 'failed' ) { $status = 'Failed'; }
				elsif ( $test_obj->{VERDICT} eq 'warning' ) { $status = 'Warnings'; }
				elsif ( $test_obj->{VERDICT} eq 'passed' ) { $status = 'Passed'; }
				elsif ( $test_obj->{VERDICT} eq 'run_problems' ) { $status = 'Crashed'; }
				elsif ( $test_obj->{VERDICT} eq 'incomplete' ) { $status = 'Incomplete'; }
				elsif ( $test_obj->{VERDICT} eq 'skipped' ) { $status = 'Skipped'; }
			}
		}
		# STATUS values: Not started|Preparing|Running|Failed|Warnings|Passed|Incomplete|Skipped|Crashed
		$content .= "STATUS = ".$status."\n";
		
		$content .= "START_TIME = ".$test_obj->{PREPARE_TIME}."\n" if defined $test_obj->{PREPARE_TIME};
		$content .= "STOP_TIME = ".$test_obj->{FINISH_TIME}."\n" if defined $test_obj->{FINISH_TIME};
		
		$content .= "PREPARE_PERCENT = ".int($test_obj->{PREPARE_PERCENT})."\n" if defined $test_obj->{PREPARE_PERCENT};
		$content .= "CURRENT_PERCENT = ".$test_obj->{PERCENT}."\n" if defined $test_obj->{PERCENT};
		$content .= "ESTIMATE_DURATION = ".$test_obj->{ESTIM_TIME}."\n" if defined $test_obj->{ESTIM_TIME};
	}
	
	# write the file using flock
	return write_string_as_file( '!'.$globals->{'status_file'}, $content );
}

#-----------------------------------------------------------------------

sub Cleanup_all {
	my ($ret_code) = @_;
	my $ok = ($ret_code == 0);
	
	inform "Cleanup...";
	
	my $recompile_report = 0;
	foreach my $test_obj ( @tests_to_run ) {
		next if !$test_obj->{STARTED};
		next if $test_obj->{CLEANUP_OK};
		
		debug_inform "...cleanup ".$test_obj->{NAME};
		
		# Check the subshell is closed
		$test_obj->Check_subshell_off();
		
		if ( $test_obj->{STATE} eq 'Running' || $test_obj->{STATE} eq 'Preparing' ) {
			$test_obj->state('Crashed');
		}
		
		$test_obj->Report(); # make a report on crashed test
		$recompile_report = 1;
		
		$test_obj->Cleanup();
	}
	if ( $recompile_report ) {
		CompileReport();
	}
	
	if ( $ok && !DEBUG ) {
		cmd( "rm -rf ".shq($globals->{'tmp_dir'}) ) if $globals->{'tmp_dir'};
	}
	
	if ( DEBUG && @all_warnings ) {
		inform "# All warnings of this run (for DEBUG purposes):";
		foreach my $warning ( @all_warnings ) {
			myprint "!!! Warning:\n".$warning->tostring()."--------------------\n";
		}
	}
	
	if ( !$ok ) {
		my $readme_file = dirname($globals->{'script_directory'})."/README";
		my $verbose_log_file = $globals->{'result_dir'}."/verbose_log";
		
		inform <<HEREDOC
#--------------------------------------------------------------
# There were errors.
# You could see the verbose log in the file 
# '$verbose_log_file'.
# The most common problems are described in the Troubleshooting
# section of the README file
# '$readme_file'
# If you believe that there is a bug in the Distribution Checker,
#  please report it to [linux\@ispras.ru].
#--------------------------------------------------------------
HEREDOC
	}
	else {
		inform "Finished OK.";
	}
}
#----------------------------------------------------------------------

sub Use_profile_general {
	my ($profile, $options) = @_;
	
	# GENERAL
	my $general = $profile->{'GENERAL'};
	if ( defined $general ) {
		
		# "ARCHITECTURE" => "ia32"
		if ( defined $general->{'ARCHITECTURE'} ) {
			$host_info->{'architecture'} = guess_architecture( $general->{'ARCHITECTURE'} );
		}
	
		# "AUTOMATIC_TESTS" => "1"
		# unused
		
		# "CERTIFY" => "1"
		$options->{'cert'} = $general->{'CERTIFY'} if defined $general->{'CERTIFY'};
		
		# "STD_PROFILE" => "core,c++,desktop"
		$options->{'std-profile'} = $general->{'STD_PROFILE'} if defined $general->{'STD_PROFILE'} && $general->{'STD_PROFILE'} ne "no";
		
		# STANDARD
		$options->{'standard'} = $general->{'STD_VERSION'} if defined $general->{'STD_VERSION'};
		
		# MANUAL_TESTS
		# not used here
		
		# USE_INTERNET
		if ( defined $general->{'USE_INTERNET'} && $general->{'USE_INTERNET'} ) {
			$options->{'download'} = 1;
		}
		
		# VERBOSE_LEVEL
		$Misc::verbose_lvl = $general->{'VERBOSE_LEVEL'} if defined $general->{'VERBOSE_LEVEL'};
		
		# NAME, ORGANIZATION
		$globals->{'tester_name'} = $general->{'NAME'} if $general->{'NAME'};
		$globals->{'tester_org'} = $general->{'ORGANIZATION'} if $general->{'ORGANIZATION'};
		# COMMENTS
		$globals->{'run_comments'} = $general->{'COMMENTS'} if $general->{'COMMENTS'};
		
		if ( $general->{'EMAIL'} && $general->{'SEND_EMAIL'} ) {
			$globals->{'mailto'} = $general->{'EMAIL'};
		}
		
	} # End of {GENERAL}
	
	# TEST_SUITES
	if ( defined $profile->{TEST_SUITES_SORT} ) {
		foreach my $testsuite ( @{$profile->{TEST_SUITES_SORT}} ) {
			next if !$profile->{TEST_SUITES}{$testsuite}{RUN}; # Not selected
			
			push @{$options->{TESTS}}, $testsuite;
		}
	}

	return 1;
}

sub get_profile_file_options {
	my ($profile, $manifest) = @_;

	( defined $profile->{FILES} ) or return 1;

	# $profile->{FILES}->{<file>}->{DOWNLOAD} == <dl-option>
	#                            ->{INSTALL} == <inst-option>
	foreach my $key ( keys %{$profile->{FILES}} ) {
		my $file_options = $profile->{FILES}{$key};
		my $file = $manifest->{FILES}{$key};
		
		if ( !defined $file ) {
			warning "File '$key' isn't defined in Manifest";
			next;
		}

		# <dl-option> is one of the following:
		#   0 - download, if necessary;  1 - force download;   2 - do not download
		if ( defined $file_options->{DOWNLOAD} ) {
			if ( $file_options->{DOWNLOAD} == 1 ) {
				# Re-download this file
				$file->{OPTIONS}{RELOAD} = 1;
			}
			elsif ( $file_options->{DOWNLOAD} == 2 ) {
				# Do not download this file
				$file->{OPTIONS}{KEEP} = 1;
			}
		}

		# <inst-option> is one of the following:
		#   0 - install, if necessary;  1 - force reinstall;  2 - do not deinstall
		if ( defined $file_options->{INSTALL} ) {
			if ( $file_options->{INSTALL} == 1 ) {
				# forced reinstallation
				$file->{OPTIONS}{REINSTALL} = 1;
			}
			elsif ( $file_options->{INSTALL} == 2 ) {
				my $package_name = $file->{PACKAGE};
				my $package_ver = $file->{VERSION};
				$manifest->{PACKAGES}{$package_name}{KEEP}{$package_ver} = 1;
			}
		}
	}

	return 1;
}
#----------------------------------------------------------------------

sub list_available_versions {
	
	my $manifest = $globals->{'manifest'} or return error "Manifest isn't loaded.";
	
	my $architecture = $host_info->{'architecture'};
	my $standard = $globals->{'standard'};
	
	print "\n";

	if ( !$manifest->{TREE} || !%{$manifest->{TREE}} ) {
		print "No information about the tests available.\n";
		print "Please update the test list using the --update option.\n";
		print "\n";
		exit 0;
	}

	print "All architectures: ".join(", ", map "'$_'", @{$manifest->{ALL}{ARCHITECTURES}} ).". ('-a' option.)\n";
	print "All standards: ".join(", ", map "'$_'", @{$manifest->{ALL}{STANDARDS}} ).". ('-s' option.)\n";
	print "All statuses: ".join(", ", map "'$_'", @{$manifest->{ALL}{STATUSES}} ).". ('-S' option.)\n";
	if ( $manifest->{STD_PROFILES}{$standard} && %{$manifest->{STD_PROFILES}{$standard}{DISPLAY_NAME}} ) {
		print "All profiles for $standard: ".join(", ", map "'$_'", sort keys %{$manifest->{STD_PROFILES}{$standard}{DISPLAY_NAME}} ).". ('-p' option.)\n";
	}
	print "All tests: ".join(", ", @{$manifest->{ALL}{TESTSUITES}} ).".\n";	
	
	print "\n\n";
	
	print "Available test versions for\n";
	print " Architecture: ".$architecture."\n";
	print " Standard: ".$standard."\n";
	
	print "\n";
	
	# Get information about installed packages
	
	my $packages = {};
	foreach my $package_name ( keys %{$manifest->{PACKAGES}} ) {
		my $found;
		for my $file_key ( @{$manifest->{PACKAGES}{$package_name}{FILES}} ) {
			if ( $manifest->{FILES}{$file_key}{KIND} ne 'file' ) {
				$found = 1; last;
			}
		}
		next if !$found;
		$packages->{$package_name} = 1;
	}
	if ( $manifest->{COMPOSITION}{$standard} ) {
		for my $test_name ( keys %{$manifest->{COMPOSITION}{$standard}} ) {
			my $composition = glob_hash($manifest->{COMPOSITION}, $standard, $test_name, $architecture) or next;
			foreach my $status ( sort keys %$composition ) {
			foreach my $test_ver ( sort keys %{$composition->{$status}} ) {
				my $requirements = $composition->{$status}{$test_ver} or next;
				for my $req ( @$requirements ) {
					if ( !$req->{LOCATION} ) {
						$packages->{$req->{PACKAGE}} = 1;
					}
				}
			}}
		}
	}
	$packages = packages_info($packages);
	is_ok($packages) or return $Error::Last;
	
	check_files( values %{$manifest->{FILES}} );
	
	local *split_num = sub{"".reverse+join",",reverse(shift)=~/.{1,3}/g};
	
	foreach my $test_name ( sort keys %{$manifest->{TREE}{$standard}} ) {
		
		print "-------------------------------------------\n";
		
		# ->{DESCRIPTIONS}{$testsuite}[]->{STANDARD|DISPLAYNAME|DESCRIPTION} = ".."
		my ($descr) = grep {version_filter($_->{STANDARD}, $standard)} @{$manifest->{'DESCRIPTIONS'}{$test_name}};
		if ( $descr ) {
			print $test_name.": ".$descr->{DISPLAYNAME}."\n";
			print ": ".$descr->{DESCRIPTION}."\n" if $descr->{DESCRIPTION};
		} else {
			print "$test_name: this test isn't described in the Manifest.\n";
		}
		
		# ->{COMPOSITION}{$standard}{$testsuite}{$arch}{$status}{$test_ver}[]->{PACKAGE|PACKAGE_VER|LOCATION} = ".."
		my $composition = glob_hash($manifest->{COMPOSITION}, $standard, $test_name, $architecture);
		if ( !$composition ) {
			print "\n";
			print "  Not available for: $architecture, $standard.\n";
			next;
		}
		
		# ->{STD_PROFILES}{$standard}{TESTS}{$testsuite}{$profile} = 1
		my $prof_hash = $manifest->{STD_PROFILES}{$standard}{TESTS}{$test_name};
		if ( $prof_hash && %$prof_hash ) {
			print "(Profiles: ".join(' | ', sort grep $prof_hash->{$_}, keys %$prof_hash).")\n"; 
		}
		
		foreach my $status ( sort keys %$composition ) {
		foreach my $test_ver ( sort keys %{$composition->{$status}} ) {
			print "\n";
			print "  Version ".$test_ver." ($status)\n";

			if ( $test_ver ne '*' ) {
				my $module_file = main_module_file($manifest, $test_name, $test_ver, $standard);
				if ( !is_ok($module_file) || !-f $globals->{'script_directory'}."/Tests/".$module_file ) {
					print "    No integration module.\n";
				}
			}

			my $requirements = $composition->{$status}{$test_ver} or next;

			for my $req ( @$requirements ) {
				print "     ";
				print $req->{PACKAGE};
				print " ".$req->{PACKAGE_VER} if $req->{PACKAGE_VER};
				# TODO: -S local

				my $file;
				if ( $req->{LOCATION} ) {
					my $found_files = Manifest::select_file($manifest, $architecture, $Packages::package_manager, $req);
					if ( is_err($found_files) || !$found_files || !@$found_files ) {
						print " (No file in Manifest!)\n";
						next;
					} else {
						$file = $found_files->[0]; # Take any one
					}
				}

				print " (";
				if ( $file && !$file->{DOWNLOADED} ) {
					print "download ".split_num($file->{SIZE})." bytes";
				}
				elsif ( $file && $file->{DOWNLOADED} == -1 ) {
					print "re-download: ".split_num($file->{SIZE})." bytes";
				}
				else {
					my $installed_ver = undef;
					if ( defined $packages->{$req->{PACKAGE}}{$architecture} ) {
						($installed_ver) = grep {version_filter($req->{PACKAGE_VER}, $_)} keys %{$packages->{$req->{PACKAGE}}{$architecture}}
					}
					if ( $installed_ver ) {
						if ( $installed_ver ne ($req->{PACKAGE_VER} or "") ) {
							print $installed_ver." ";
						}
						print "installed";
					}
					elsif ( $file && $file->{DOWNLOADED} == 1 ) {
						print "downloaded";
					}
					else {
						print "not installed";
					}
				}
				print ")";
				print "\n";
			}
		}}
	} continue {
		print "\n";
	}
}
#-----------------------------------------------------------------------

# Restore global variables using information from special files
#  in a specified result directory.
# This function is used when re-building report.
sub restore_env {
	my ($result_dir) = @_;
	
	$globals->{'result_dir'} = $result_dir;
	$globals->{'testrun_id'} = extract_filename($result_dir);
	$globals->{'test_result_dir'} = $result_dir."/results";
	
	# Restore runconfig
	my $runconfig = read_config( $result_dir."/runconfig" );
	is_ok($runconfig) or return error "Failed to read runconfig", $Error::Last;
	
	$globals->{'standard'} = $runconfig->{STANDARD} if $runconfig->{STANDARD};
	$globals->{'cert_mode'} = $runconfig->{CERT} if $runconfig->{CERT};
	$globals->{'run_comments'} = $runconfig->{RUN_COMMENTS} if $runconfig->{RUN_COMMENTS};
	
	my %tests = ();
	
	if ( $runconfig->{SECTIONS} ) {
		foreach my $test_name ( keys %{$runconfig->{SECTIONS}} ) {
			my $data = $runconfig->{SECTIONS}{$test_name};
			
			my $test_obj = $tests{$test_name};
			if ( !$test_obj ) {
				$test_obj = $tests{$test_name} = {NAME => $test_name};
				push @tests_to_run, $test_obj;
			}
			for my $key (keys %$data) {
				$test_obj->{$key} = $data->{$key};
			}
		}
	}
	
	# Restore .info data
	my @info_files = glob($result_dir."/results/*.info");
	foreach my $info_file ( @info_files ) {
		my $test_info = read_config( $info_file );
		my $test_name = $test_info->{NAME} or next;
		
		my $test_obj = $tests{$test_name};
		if ( !$test_obj ) {
			$test_obj = $tests{$test_name} = {NAME => $test_name};
			push @tests_to_run, $test_obj;
		}
		for my $key (keys %$test_info) {
			$test_obj->{$key} = $test_info->{$key};
		}
	}
}

sub rebuild_report {
	my ($dir, $keep) = @_;
	
	# Restore saved environment
	is_ok restore_env($dir) or fail $Error::Last;
	
	# Load Manifest
	debug_inform "Loading manifest...";
	my $manifest = $globals->{'manifest'} = load_manifest('MODULES' => 1);
	is_ok $manifest or fail "Failed to load manifest.", $Error::Last;
	
	# Load problem database
	$Report::result_dir = $globals->{'result_dir'};
	$Report::cert_mode = $globals->{'cert_mode'}; # Restored by restore_env($dir) call above.
	inform "Loading problem database...";
	is_ok Report::problem_db2_load( $PROBLEM_DB_FILE )
		or warning "Failed to load the problem database.", $Error::Last;
	
	( defined $globals->{'standard'} ) or fail "Wrong data stored";
	
	# Generate test reports
	foreach my $test_obj ( @tests_to_run ) {
		my $test_name = $test_obj->{NAME};
		if ( $keep && $test_obj->{VERDICT} && !$test_obj->{MANUAL} ) {
			debug_inform "skip $test_name: ".Report::verdict_text($test_obj->{VERDICT});
			next; # Don't rebuilt existing reports if '--keep' key was specified (except for manual tests)
		}
		
		# Load test modules and create test objects
		if ( !$test_obj->{VERSION} ) {
			warning("No VERSION stored for $test_name");
			next;
		}
		if ( !$test_obj->{VERSION_MNF} ) {
			$test_obj->{VERSION_MNF} = $test_obj->{VERSION};
		}
		# Load test module and create test object
		my $module_file = main_module_file($manifest, $test_name, $test_obj->{VERSION}, $globals->{'standard'});
		if ( !is_ok($module_file) ) {
			warning "Failed to prepare modules for '$test_name'. Please check for a new version of the Distribution Checker.", $Error::Last;
			next;
		}
		my $class_name = load_test_module($module_file);
		if ( !is_ok($class_name) ) {
			warning "Failed to load module '$module_file'.", $Error::Last;
			next;
		}
		$test_obj = $class_name->new( %$test_obj ); # create test object
		
		$test_obj->Report();
		inform $test_obj->{NAME}.": ".Report::verdict_text($test_obj->{VERDICT}) if $test_obj->{VERDICT};
	}
	
	is_ok CompileReport() or fail $Error::Last;
	
	# Make a tarball with the results
	$globals->{'results_tarball'} = $globals->{'result_dir'}."/".$globals->{'testrun_id'}.".tgz";
		
	cmd( "rm -f ".shq($globals->{'results_tarball'}) ); # Remove the old tarball
	
	my $tmp_file = extract_filename($globals->{'results_tarball'});
	cmd( "cd ".shq(extract_dir($globals->{'result_dir'}))
		." && tar czf ".shq($tmp_file)." ".shq( extract_filename($globals->{'result_dir'}) ).' --force-local'
		." && mv ".shq($tmp_file)." ".shq($globals->{'results_tarball'})
		);

	inform "Report: ".$globals->{'result_dir'};
	inform "Verdict: ".Report::verdict_text($globals->{'verdict'}) if $globals->{'verdict'};
}

sub CompileReport {
	inform "Making report...";

	my ($verdict, $verdict_man) = Report::Compile();
	if ( !is_ok($verdict) ) {
		complain "Failed to compile the report", $Error::Last;
		$verdict = 'run_problems';
	}
	
	if ( $verdict ) {
		if ( $verdict !~ /^(passed|warning|incomplete)$/ ) {
			$verdict = 'failed';
		}
		$globals->{'verdict'} = $verdict;
	}
	if ( $verdict_man ) {
		if ( $verdict_man !~ /^(incomplete|passed|skipped)$/ ) {
			$verdict_man = 'failed';
		}
		$globals->{'verdict_man'} = $verdict_man;
	}
	
	write_INFO();
}
#-----------------------------------------------------------------------

sub send_results {
	my ( $tarball_file ) = @_;
	
	( $globals->{'mailto'} ) or return error "Mailto parameter isn't defined.";
	( $tarball_file ) or return error "Results tarball missed";
	
	# Send the results by e-mail if requested.
	inform "Sending the results by e-mail...\n";

	# Use sendmail utility
	sendmail (
		TO      => $globals->{'mailto'},
		SUBJECT => "Distribution Check results for ".$host_info->{'hostname'},
		TEXT    => "Please see the attached archive for the test results.",
		ATTACHMENT => $tarball_file,
	);
}
#-----------------------------------------------------------------------

# Syncronize Manifest with the server
#
# $manif_file - Manifest file name with path.
sub update_manifest {
	my ($manif_file, $manif_new_file) = @_;
	
	my $manifest2 = manifest_parse( $manif_new_file );
	if ( !is_ok($manifest2) ) {
		if ( $Error::Last->{-data} ) {
			# Check for manifest version
			if ( $Error::Last->{-data}{MANIFEST}{VERSION} ) {
				if ( vercmp($Error::Last->{-data}{MANIFEST}{VERSION}, MANIFEST_FORMAT_VER) > 0 ) {
					$Error::Last = error "Incompatible Manifest format (".$Error::Last->{-data}{MANIFEST}{VERSION}."). Please update the Distribution Checker.";
				}
				elsif ( vercmp($Error::Last->{-data}{MANIFEST}{VERSION}, MANIFEST_FORMAT_VER) < 0 ) {
					$Error::Last = error "Your Manifest format (".MANIFEST_FORMAT_VER.") is newer than available (".$Error::Last->{-data}{MANIFEST}{VERSION}.").";
				}
			}
		}
		return error "Failed to parse manifest file '$manif_new_file'", $Error::Last;
	}

	$manifest2->{MANIFEST}{TIMESTAMP} = time();

	#is_ok manifest_check($manifest2)
		#or return error("Manifest check failed", $Error::Last->{-arr} ? $Error::Last->{-arr}[0] : $Error::Last, -ret => 2);

	if ( -f $manif_file ) {
		# Backup the Manifest file
		cmd( "cp -f ".shq($manif_file)." ".shq($manif_file.".old") ) == 0
			or return error "Failed to backup Manifest file";
	}

	# Write a new Manifest
	is_ok write_string_as_file( "!".$manif_file, print_manifest($manifest2) )
		or return error "Failed to write the Manifest file '$manif_file'", $Error::Last, -ret => 2;
	
	inform "Manifest has been written to: '$manif_file'\n";
	
	return 1;
}

# Update problem_db, manifests and test modules.
sub update {
	
	my @errors = ();

	inform "Checking for updates";

	# Update problem_db
	debug_inform("Updating problem database...");
	is_ok download_file($PROBLEM_DB_LINK . ".md5", $PROBLEM_DB_FILE . ".md5")
		or push @errors, error "Failed to update problem database checksum", $Error::Last;
	is_ok download_file($PROBLEM_DB_LINK, $PROBLEM_DB_FILE, MD5SUM => 1)
		or push @errors, error "Failed to update problem database", $Error::Last;
	
	my $ret_code; # return code, used by WebUI. It has specific value if there is option to overwrite a manifest file when failed to merge it.
	
	# Update manifests
	debug_inform("Updating manifest files...");
	my @files = ();
	push @files, $MANIFEST_DIR."/Manifest";
	push @files, glob($MANIFEST_DIR."/*.mnf");

	for my $manif_file ( @files ) {
		debug_inform("Updating $manif_file ...");
		
		my $url;
		if ( -f $manif_file ) {
			my $manifest = manifest_parse( $manif_file );
			if ( !is_ok($manifest) ) {
				# The error returned by manifest_parse() should contain
				# the part of the Manifest it has managed to parse.
				$manifest = $Error::Last->{-data};
			}
			# Take the Manifest URL from the existing Manifest header
			if ( $manifest && $manifest->{MANIFEST}{URL} ) {
				$url = $manifest->{MANIFEST}{URL};
			}
			$manifest = undef;
		}
		if ( !$url && $manif_file =~ m{/Manifest$} ) {
			$url = $DEFAULT_MANIFEST_URL;
		}
		if ( !$url ) {
			warning "No URL for updating '$manif_file'. Please update it manually.";
			next;
		}
		
		my $manif_new_file = $manif_file.".new";
		if ( -f $manif_new_file ) { # Delete this file if it is on the way
			cmd( "rm -f ".shq($manif_new_file) );
			if ( -f $manif_new_file ) {
				push @errors, error "Failed to remove '$manif_new_file'", error $last_cmd_output;
				next;
			}
		}
		
		# Download new Manifest file
		is_ok download_file($url, $manif_new_file)
		or do {
			push @errors, error "Failed to download a new Manifest file", $Error::Last;
			cmd( "rm -f ".shq($manif_new_file) );
			next;
		};
		# check if the file was really downloaded
		if ( !-f $manif_new_file || -z $manif_new_file) {
			push @errors, error "Failed to download a new Manifest file";
			next;
		}
		
		my $ret;
		is_ok update_manifest($manif_file, $manif_new_file)
		or do {
			push @errors, error "Failed to update '$manif_file'", $Error::Last;
			$ret = $Error::Last->{-ret};
		};
		if ( $ret && $ret eq 2 ) {
			$ret_code = 2;
			# Return code 2 means that the local Manifest file should be overwrited with new Manifest file.
			# Do not delete Manifest.new.
			push @errors, error "You probably should overwrite '$manif_file' with '$manif_new_file'";
		} else {
			cmd( "rm -f ".shq($manif_new_file) );
		}
	}
	
	# Update test modules
	debug_inform("Updating test modules...");
	
	my $manifest = load_manifest();
	is_ok($manifest) or fail "Failed to load manifest.", $Error::Last;
	
	# Check latest Distribution Checker version
	if ( defined $manifest->{MANIFEST}{DC_VERSION_AVAILABLE} ) {
		if ( $DC_VERSION lt $manifest->{MANIFEST}{DC_VERSION_AVAILABLE} ) {
			inform "";
			inform "A newer version of $DC_BRANCH Distribution Checker is available: ".$manifest->{MANIFEST}{DC_VERSION_AVAILABLE};
			inform "";
		}
	}
	
	my $modules = {};
	foreach my $test_name ( sort keys %{$manifest->{MODULES}} ) {
		foreach my $module ( @{$manifest->{MODULES}{$test_name}} ) {
			my $module_file = $module->{FILE};
			if ( $modules->{$module_file} && $modules->{$module_file} ne $module->{FROM} ) {
				warning "Conflicting sources for '$module_file': '".$modules->{$module_file}."' ne '".$module->{FROM}."'.";
			}
			$modules->{$module_file} = $module->{FROM};
		}
	}
	foreach my $module_file ( sort keys %$modules ) {
		my $from = $modules->{$module_file};
		my $fullname = $globals->{'script_directory'}."/Tests/".$module_file;
		
		if ( !$from ) {
			#debug_inform("No source for updating '$module_file' is specified. It's not an error.");
			next;
		}
		debug_inform("Updating '$module_file'...");
		
		# backup
		cmd("cp -f ".shq($fullname)." ".shq($fullname.".old"));
		
		if ( $from =~ /^\// ) {
			if ( -f $from ) {
				cmd("cp -f ".shq($from)." ".shq($fullname));
				( -f $fullname ) or push @errors, error("Can't update $module_file", error $last_cmd_output);
			} else {
				push @errors, error "There is no file '$from'";
			}
		}
		elsif ( $from =~ /^(https?|ftps?):/ ) {
			is_ok download_file($from, $fullname)
				or push @errors, error("Can't update $module_file", error "Failed to download '$from'", $Error::Last);
		}
		else {
			warning "Unsupported update source for $module_file: '$from'";
			die if DEBUG; # DBG:
		}
	}
	
	# Done
	
	if ( @errors ) {
		my $err = join_errors(@errors);
		$err->{-ret} = $ret_code if $ret_code;
		return $err;
	}
	
	return 1;
}

#============================================================================
# The Main Script
#============================================================================

# Initialization
is_ok Setup()
	or fail "Initialization failed.", $Error::Last;

# Write the status file.
unless ( $globals->{'check_only'} ) {
	is_ok write_status()
		or fail "Can't write status file", $Error::Last;
	
	print_stderr "Started.\n" if $globals->{'webui'};
	# Since this point the WEB-UI shows the console output.
}
	
$_END = sub {
	if ( $need_cleanup ) {
		Cleanup_all( $? );
	}
	
	$globals->{'done'} ||= 'Terminated';
	write_status();
	
	print_stderr "Finished.\n" if $globals->{'webui'};
};

# Run test checks
{
	my $there_are_errors = 0;

	foreach my $test_obj ( @tests_to_run ) {
		my $test_name = $test_obj->{NAME};

		inform "Checking $test_name...";
		
		my $res = $test_obj->check();
		if ( !is_ok($res) || !$res ) {
			complain "Preliminary check failed for the ".$test_obj->{DISPLAYNAME}.".", $Error::Last;
			$there_are_errors = 1;
		}
	}

	fail() if $there_are_errors && !$globals->{'ignore_check'};
}

if ( !$ENV{TERM} ) {
	fail("Tests should be started from a terminal.");
}

if ( $globals->{'check_only'} ) {
	inform "Done.";
	exit 0;
}

$Packages::cache = undef; # disable packages cache

# Setup interrupt handlers
$SIG{INT} = $SIG{TERM} = sub { 
	local $SIG{INT} = local $SIG{TERM} = 'IGNORE'; 
	
	fail "Caught a terminate signal!";
};

write_status();

# TODO: Stop the ntpd???

unless ( DEBUG && @tests_to_run == 1 ) { # DBG:
	inform "Benchmarking...";
	Benchmark();
	inform "...done.";
}

# Prepare the report module
{
	$Report::result_dir = $globals->{'result_dir'};
	$Report::cert_mode = $globals->{'cert_mode'};
	inform "Loading problem database...";
	is_ok Report::problem_db2_load( $PROBLEM_DB_FILE )
		or warning "Failed to load the problem database."
			." Please update the problem database using option '--refresh-prdb'.", $Error::Last;
	inform "...done.";
}

# Write the run configuration  [after the Benchmark]
{
	write_status();
	# Save the system configuration
	is_ok write_runconfig()
		or fail "Failed to save configuration.", $Error::Last;
	# Write the info file
	is_ok write_INFO()
		or fail "Failed to write INFO file.", $Error::Last;
	
	# Save the test configuration
	foreach my $test_obj ( @tests_to_run ) {
		my $test_name = $test_obj->{NAME};
		is_ok $test_obj->save_info()
			or fail "Failed to save configuration for test '$test_name'", $Error::Last;
	}
}

## RUN THE TESTS ##

# Stop the running test if captured the Interrupt signal.
$SIG{INT} = sub {
	local $SIG{INT} = 'IGNORE';
	if ( $globals->{'interrupt_request'} ) { # Ctrl-C pushed twice
		kill 'TERM', $$; # send the TERM signal to itself.
		return;
	}
	$globals->{'interrupt_request'} = 1;
	
	inform "Caught an interrupt signal!";
	
	my $subshell_stopped = 0;
	foreach my $test_obj ( @tests_to_run ) {
		next if !$test_obj->{STARTED};
		
		# Force the subshell to be closed
		if ( $test_obj->Check_subshell_off() ) {
			$subshell_stopped = 1;
		}
	}
	if ( !$subshell_stopped ) {
		kill 'TERM', $$; # send the TERM signal to itself.
		return;
	}
};

write_status();

unless ( defined $globals->{'batch'} && !$globals->{'batch'} ) {
	if ( $globals->{'batch'} ) { inform "Batch downloading"; }
	# Batch download
	foreach my $test_obj ( @tests_to_run ) {
		my $test_name = $test_obj->{NAME};
		if ( $test_obj->{STATUS} eq 'snapshot' || $globals->{'batch'} ) {
			debug_inform "Downloading files for '$test_name' tests.";
			if ( is_ok $test_obj->prepare_test_packages( NOT_INSTALL => 1 ) ) {
				$test_obj->{DOWNLOADED} = 1;
			} else {
				complain "Failed to download files for '$test_name' test", $Error::Last;
			}
		}
	}
}

$need_cleanup = 1; # Used in END{}

# THE MAIN CYCLE
foreach my $test_obj ( @tests_to_run ) {
	my $test_name = $test_obj->{NAME};
	
	inform "==========================================";
	
	$globals->{'interrupt_request'} = 0;
	
	is_ok $test_obj->Go()
		or complain $Error::Last;
	
	$globals->{'interrupt_request'} = 0; # TODO: use in no-subshell app-bat tests
	
	# Print the test verdict
	inform $test_obj->name." verdict: ".Report::verdict_text($test_obj->{VERDICT}) if $test_obj->{VERDICT};
	
	write_runconfig();
	
	# Compile the report
	CompileReport();
}
#-----------------

# Finish
inform "==========================================";

# Make a tarball with the results
{
	$globals->{'results_tarball'} = $globals->{'result_dir'}."/".$globals->{'testrun_id'}.".tgz";
	
	my $tmp_file = $globals->{'temp_dir'}."/".extract_filename($globals->{'results_tarball'});
	cmd( "cd ".shq(extract_dir($globals->{'result_dir'}))
		." && tar czf ".shq($tmp_file)." ".shq( extract_filename($globals->{'result_dir'}) ).' --force-local'
		." && mv ".shq($tmp_file)." ".shq($globals->{'results_tarball'})
		);
}

inform "Report: ".$globals->{'result_dir'};
inform "Verdict: ".Report::verdict_text($globals->{'verdict'}) if $globals->{'verdict'};

if ( $globals->{'mailto'} ) {
	# Send results by e-mail
	is_ok send_results( $globals->{'results_tarball'} )
		or complain $Error::Last;
}

if ( $globals->{'post_cmd'} ) {
	# Execute user-defined command to the result tarball.
	cmd( $globals->{'post_cmd'}." ".shq($globals->{'results_tarball'}) );
}

$globals->{'done'} = 'Finished';

exit 0;

END { goto &$_END if $_END; }