On Jan 25, 2012, at 6:02 AM,
miked@osl.iu.edu wrote:
> Author: miked
> Date:
2012-01-25 06:02:47 EST (Wed, 25 Jan 2012)
> New Revision: 1433
> URL:
https://svn.open-mpi.org/trac/mtt/changeset/1433
>
> Log:
> * Introduce mtt.break_threshold_timeout_and_fail ini param to specify % of overall failed tests to trigger stop execution.
>
>
> Text files modified:
> trunk/lib/MTT/Reporter/TextFile.pm | 9 ++++-
> trunk/lib/MTT/Test/Run.pm | 30 ++++++++++++++++----
> trunk/lib/MTT/Test/RunEngine.pm | 60 +++++++++++++++++++++++++++++++--------
> trunk/lib/MTT/Util.pm | 6 ++++
> trunk/lib/MTT/Values.pm | 3 +
> 5 files changed, 86 insertions(+), 22 deletions(-)
>
> Modified: trunk/lib/MTT/Reporter/TextFile.pm
> ==============================================================================
> --- trunk/lib/MTT/Reporter/TextFile.pm (original)
> +++ trunk/lib/MTT/Reporter/TextFile.pm
2012-01-25 06:02:47 EST (Wed, 25 Jan 2012)
> @@ -230,8 +230,13 @@
> my $filename = "All_phase-summary.txt";
> my $file = "$dirname/" . MTT::Files::make_safe_filename("$filename");
>
> - my $body = join("\n", ($summary_header, $table->render, $perf_stat, $summary_footer));
> -
> + my $body;
> + if ($MTT::Globals::Internals->{is_stopped_on_break_threshold}){
> + $body = join("\n", ($summary_header, $table->render, $perf_stat, $MTT::Globals::Internals->{stopped_on_break_threshold_message}, $summary_footer));
> + }
> + else{
> + $body = join("\n", ($summary_header, $table->render, $perf_stat, $summary_footer));
> + }
> print $body;
> _output_results($file, $body);
>
>
> Modified: trunk/lib/MTT/Test/Run.pm
> ==============================================================================
> --- trunk/lib/MTT/Test/Run.pm (original)
> +++ trunk/lib/MTT/Test/Run.pm
2012-01-25 06:02:47 EST (Wed, 25 Jan 2012)
> @@ -64,8 +64,11 @@
> #--------------------------------------------------------------------------
>
> sub Run {
> - my ($ini, $ini_full, $install_dir, $runs_data_dir, $force) = @_;
> + my ($ini, $ini_full, $install_dir, $runs_data_dir, $force, $count_total_tests_number) = @_;
>
> + if ($count_total_tests_number ne "yes"){
> + Run($ini, $ini_full, $install_dir, $runs_data_dir, $force,"yes");
> + }
> # Save the environment
> my %ENV_SAVE = %ENV;
>
> @@ -188,7 +191,7 @@
> _do_run($ini, $section, $test_build,
> $mpi_get, $mpi_install,
> $install_dir, $runs_data_dir,
> - $force);
> + $force,$count_total_tests_number);
> delete $MTT::Globals::Internals->{mpi_get_name};
> delete $MTT::Globals::Internals->{mpi_install_name};
> delete $MTT::Globals::Internals->{test_get_name};
> @@ -196,12 +199,24 @@
> delete $MTT::Globals::Internals->{test_run_name};
> %ENV = %ENV_SAVE;
> }
> + last
> + if ($MTT::Globals::Internals->{is_stopped_on_break_threshold});
> }
> + last
> + if ($MTT::Globals::Internals->{is_stopped_on_break_threshold});
> }
> + last
> + if ($MTT::Globals::Internals->{is_stopped_on_break_threshold});
> }
> + last
> + if ($MTT::Globals::Internals->{is_stopped_on_break_threshold});
> }
> + last
> + if ($MTT::Globals::Internals->{is_stopped_on_break_threshold});
> }
> }
> + last
> + if ($MTT::Globals::Internals->{is_stopped_on_break_threshold});
> }
>
> Verbose("*** Run test phase complete\n");
> @@ -211,7 +226,7 @@
>
> sub _do_run {
> my ($ini, $section, $test_build, $mpi_get, $mpi_install, $install_dir,
> - $runs_data_dir, $force) = @_;
> + $runs_data_dir, $force, $count_total_tests_number) = @_;
>
> # Simple section name
> my $simple_section = GetSimpleSection($section);
> @@ -528,9 +543,13 @@
> # If we got a list of tests to run, invoke the run engine to
> # actually run them.
> if ($ret && $ret->{test_result}) {
> - MTT::Test::RunEngine::RunEngine($ini, $section, $install_dir,
> + my $return_value;
> + $return_value = MTT::Test::RunEngine::RunEngine($ini, $section, $install_dir,
> $runs_data_dir, $mpi_details,
> - $test_build, $force, $ret);
> + $test_build, $force, $ret, $count_total_tests_number);
> + if ($count_total_tests_number eq "yes"){
> + $MTT::Globals::Internals->{total_tests_counter} += $return_value;
> + }
> }
>
> # Unload any loaded environment modules
> @@ -545,7 +564,6 @@
> # means reverting to an env snapshot.
> MTT::EnvImporter::unload(reverse @env_importers);
> }
> -
> }
>
> #--------------------------------------------------------------------------
>
> Modified: trunk/lib/MTT/Test/RunEngine.pm
> ==============================================================================
> --- trunk/lib/MTT/Test/RunEngine.pm (original)
> +++ trunk/lib/MTT/Test/RunEngine.pm
2012-01-25 06:02:47 EST (Wed, 25 Jan 2012)
> @@ -38,10 +38,14 @@
> my $section;
> my $mpi_details_name;
> my $test_run_full_name;
> +my $break_threshold;
>
> # Keep track of how many tests have passed, failed, skipped, and timed out
> my $test_results_count;
>
> +#same as test_results_count but global over all envokations of RunEngine
> +my $test_results_count_global;
> +
> # Submit results after each run or after *all* the runs
> my $report_after_each_result = 0;
> my $report_results_count = 0;
> @@ -56,7 +60,7 @@
> my ($install_dir, $runs_data_dir, $mpi_details, $test_build, $force, $ret);
>
> # Make sure though, that the $ini remains a global
> - ($ini, $section, $install_dir, $runs_data_dir, $mpi_details, $test_build, $force, $ret) = @_;
> + ($ini, $section, $install_dir, $runs_data_dir, $mpi_details, $test_build, $force, $ret, my $count_total_tests_number) = @_;
>
> my $test_results;
> $group_reports = MTT::Values::Value($ini, "mtt", "submit_group_results");
> @@ -81,12 +85,16 @@
> my $variants_count_total =
> $test_count_total * $np_count_total * $argv_count_total * $exec_count_total;
>
> + if ($count_total_tests_number eq "yes"){
> + return $variants_count_total;
> + }
> # Set some thresholds for an early exit
> - my $break_threshold;
> + $break_threshold = undef;
> $break_threshold->{MTT::Values::PASS} = Value($ini, $section, "break_threshold_pass");
> $break_threshold->{MTT::Values::FAIL} = Value($ini, $section, "break_threshold_fail");
> $break_threshold->{MTT::Values::TIMED_OUT} = Value($ini, $section, "break_threshold_timeout");
> $break_threshold->{MTT::Values::SKIPPED} = Value($ini, $section, "break_threshold_skipped");
> + $break_threshold->{MTT::Values::TIMED_OUT_OR_FAIL} = Value($ini, "mtt", "break_threshold_timeout_and_fail");
>
> # This boolean value defaults to 0, and allows the user to submit results
> # after each test to ensure at least *some* results are submitted (in case
> @@ -208,25 +216,29 @@
> $run->{wdir} = $wdir;
> }
> # Just one np, or an array of np values?
> +
> if (ref($all_np) eq "") {
> - $test_results->{$all_np} =
> - _run_one_np($install_dir, $run, $mpi_details, $all_np, $force);
> + $test_results->{$all_np} = _run_one_np($install_dir, $run, $mpi_details, $all_np, $force);
> } else {
> foreach my $this_np (@$all_np) {
> # See if we're supposed to terminate.
> last
> - if (MTT::Util::time_to_terminate());
> + if (MTT::Util::time_to_terminate());
> +
> + $test_results->{$all_np} = _run_one_np($install_dir, $run, $mpi_details, $this_np,$force);
>
> - $test_results->{$this_np} =
> - _run_one_np($install_dir, $run, $mpi_details, $this_np,
> - $force);
> + last
> + if ($MTT::Globals::Internals->{is_stopped_on_break_threshold});
> }
> }
> +
> + last
> + if ($MTT::Globals::Internals->{is_stopped_on_break_threshold});
> ++$test_count;
>
> # Write out the "to be saved" test run results
> MTT::Test::SaveRuns($runs_data_dir);
> -
> +
> $MTT::Test::Run::mpi_details = $save_run_mpi_details;
>
> # Output a progress bar
> @@ -247,6 +259,7 @@
>
> MTT::Reporter::QueueSubmit();
> }
> +
> }
>
> sub _run_one_np {
> @@ -290,16 +303,30 @@
> foreach my $e (@$execs) {
> # See if we're supposed to terminate.
> last
> - if (MTT::Util::time_to_terminate());
> + if (MTT::Util::time_to_terminate());
> +
> _run_one_test($install_dir, $run, $mpi_details, $e, $name,
> - $variant++, $force);
> + $variant++, $force);
> +
> + last
> + if (MTT::Util::check_break_threshold(
> + $test_results_count_global,
> + $break_threshold,
> + $MTT::Globals::Internals->{total_tests_counter})
> + );
> }
> }
> -
> + last
> + if (MTT::Util::check_break_threshold(
> + $test_results_count_global,
> + $break_threshold,
> + $MTT::Globals::Internals->{total_tests_counter})
> + );
> +
> $MTT::Test::Run::test_argv = undef;
> }
> }
> -
> +
> $MTT::Test::Run::test_np = undef;
> }
>
> @@ -457,6 +484,13 @@
> $test_results_count->{$report->{test_result}}++
> if (exists($report->{test_result}));
>
> + $test_results_count_global->{$report->{test_result}}++
> + if (exists($report->{test_result}));
> +
> + $test_results_count_global->{MTT::Values::TIMED_OUT_OR_FAIL}++
> + if (exists($report->{test_result}) &&
> + (MTT::Values::FAIL == $report->{test_result} || MTT::Values::TIMED_OUT == $report->{test_result}));
> +
> # If there is an after_each step, run it
> $ENV{MTT_TEST_RUN_RESULT_MESSAGE} =
> (MTT::Values::PASS == $report->{test_result} ? "passed" :
>
> Modified: trunk/lib/MTT/Util.pm
> ==============================================================================
> --- trunk/lib/MTT/Util.pm (original)
> +++ trunk/lib/MTT/Util.pm
2012-01-25 06:02:47 EST (Wed, 25 Jan 2012)
> @@ -205,6 +205,12 @@
>
> if (($count->{$result} / $total) > $threshold->{$result}) {
> Verbose("--> Threshold ($per) exceeded for \"$result_label\": $count->{$result} out of $total.\n");
> + $MTT::Globals::Internals->{is_stopped_on_break_threshold} = "true";
> + $MTT::Globals::Internals->{stopped_on_break_threshold_message} = "--> Threshold ($per) exceeded for \"$result_label\": $count->{$result} out of $total.\n";
> + print STDOUT "--> Threshold ($per) exceeded for \"$result_label\": $count->{$result} out of $total.\n";
> + if ($MTT::Globals::Internals->{is_stopped_on_break_threshold}){
> + print STDOUT "0xdeadbeef: it works";
> + }
> return 1;
> }
> }
>
> Modified: trunk/lib/MTT/Values.pm
> ==============================================================================
> --- trunk/lib/MTT/Values.pm (original)
> +++ trunk/lib/MTT/Values.pm
2012-01-25 06:02:47 EST (Wed, 25 Jan 2012)
> @@ -45,6 +45,7 @@
> PASS => 1,
> SKIPPED => 2,
> TIMED_OUT => 3,
> + TIMED_OUT_OR_FAIL =>4,
> };
>
> # Map to human-readable English labels
> @@ -53,7 +54,7 @@
> $result_messages->{MTT::Values::PASS} = "pass";
> $result_messages->{MTT::Values::TIMED_OUT} = "timeout";
> $result_messages->{MTT::Values::SKIPPED} = "skipped";
> -
> +$result_messages->{MTT::Values::TIMED_OUT_OR_FAIL} = "timeout_and_fail";
> # current $ini and $section parameters (we use it in funclets)
> our $evaluate_string_ini;
> our $evaluate_string_section;
> _______________________________________________
> mtt-svn mailing list
>
mtt-svn@open-mpi.org
>
http://www.open-mpi.org/mailman/listinfo.cgi/mtt-svn