Open MPI logo

MTT Devel Mailing List Archives

  |   Home   |   Support   |   FAQ   |   all MTT Devel mailing list

Subject: Re: [MTT devel] [MTT svn] svn:mtt-svn r1433
From: Mike Dubman (mike.ompi_at_[hidden])
Date: 2012-01-26 02:12:34


done
credit goes to Valentin as a creator of this feature!

On Wed, Jan 25, 2012 at 3:11 PM, Jeff Squyres <jsquyres_at_[hidden]> wrote:

> Yummy -- thanks Mike!
>
> Can you add this field (and any others you have added) to the wiki?
>
> https://svn.open-mpi.org/trac/mtt/wiki/MTTINIFileFields
>
> This is our *only* documentation; it's important to keep it up to date.
>
> Thanks!
>
>
> On Jan 25, 2012, at 6:02 AM, miked_at_[hidden] wrote:
>
> > Author: miked
> > Date: 2012-01-25 06:02:47 EST (Wed, 25 Jan 2012)
> > New Revision: 1433
> > URL: https://svn.open-mpi.org/trac/mtt/changeset/1433
> >
> > Log:
> > * Introduce mtt.break_threshold_timeout_and_fail ini param to specify %
> of overall failed tests to trigger stop execution.
> >
> >
> > Text files modified:
> > trunk/lib/MTT/Reporter/TextFile.pm | 9 ++++-
> > trunk/lib/MTT/Test/Run.pm | 30 ++++++++++++++++----
> > trunk/lib/MTT/Test/RunEngine.pm | 60
> +++++++++++++++++++++++++++++++--------
> > trunk/lib/MTT/Util.pm | 6 ++++
> > trunk/lib/MTT/Values.pm | 3 +
> > 5 files changed, 86 insertions(+), 22 deletions(-)
> >
> > Modified: trunk/lib/MTT/Reporter/TextFile.pm
> >
> ==============================================================================
> > --- trunk/lib/MTT/Reporter/TextFile.pm (original)
> > +++ trunk/lib/MTT/Reporter/TextFile.pm 2012-01-25 06:02:47 EST
> (Wed, 25 Jan 2012)
> > @@ -230,8 +230,13 @@
> > my $filename = "All_phase-summary.txt";
> > my $file = "$dirname/" . MTT::Files::make_safe_filename("$filename");
> >
> > - my $body = join("\n", ($summary_header, $table->render, $perf_stat,
> $summary_footer));
> > -
> > + my $body;
> > + if ($MTT::Globals::Internals->{is_stopped_on_break_threshold}){
> > + $body = join("\n", ($summary_header, $table->render,
> $perf_stat, $MTT::Globals::Internals->{stopped_on_break_threshold_message},
> $summary_footer));
> > + }
> > + else{
> > + $body = join("\n", ($summary_header, $table->render,
> $perf_stat, $summary_footer));
> > + }
> > print $body;
> > _output_results($file, $body);
> >
> >
> > Modified: trunk/lib/MTT/Test/Run.pm
> >
> ==============================================================================
> > --- trunk/lib/MTT/Test/Run.pm (original)
> > +++ trunk/lib/MTT/Test/Run.pm 2012-01-25 06:02:47 EST (Wed, 25 Jan 2012)
> > @@ -64,8 +64,11 @@
> >
> #--------------------------------------------------------------------------
> >
> > sub Run {
> > - my ($ini, $ini_full, $install_dir, $runs_data_dir, $force) = @_;
> > + my ($ini, $ini_full, $install_dir, $runs_data_dir, $force,
> $count_total_tests_number) = @_;
> >
> > + if ($count_total_tests_number ne "yes"){
> > + Run($ini, $ini_full, $install_dir, $runs_data_dir,
> $force,"yes");
> > + }
> > # Save the environment
> > my %ENV_SAVE = %ENV;
> >
> > @@ -188,7 +191,7 @@
> > _do_run($ini, $section, $test_build,
> > $mpi_get, $mpi_install,
> > $install_dir, $runs_data_dir,
> > - $force);
> > +
> $force,$count_total_tests_number);
> > delete
> $MTT::Globals::Internals->{mpi_get_name};
> > delete
> $MTT::Globals::Internals->{mpi_install_name};
> > delete
> $MTT::Globals::Internals->{test_get_name};
> > @@ -196,12 +199,24 @@
> > delete
> $MTT::Globals::Internals->{test_run_name};
> > %ENV = %ENV_SAVE;
> > }
> > + last
> > + if
> ($MTT::Globals::Internals->{is_stopped_on_break_threshold});
> > }
> > + last
> > + if
> ($MTT::Globals::Internals->{is_stopped_on_break_threshold});
> > }
> > + last
> > + if
> ($MTT::Globals::Internals->{is_stopped_on_break_threshold});
> > }
> > + last
> > + if
> ($MTT::Globals::Internals->{is_stopped_on_break_threshold});
> > }
> > + last
> > + if
> ($MTT::Globals::Internals->{is_stopped_on_break_threshold});
> > }
> > }
> > + last
> > + if ($MTT::Globals::Internals->{is_stopped_on_break_threshold});
> > }
> >
> > Verbose("*** Run test phase complete\n");
> > @@ -211,7 +226,7 @@
> >
> > sub _do_run {
> > my ($ini, $section, $test_build, $mpi_get, $mpi_install,
> $install_dir,
> > - $runs_data_dir, $force) = @_;
> > + $runs_data_dir, $force, $count_total_tests_number) = @_;
> >
> > # Simple section name
> > my $simple_section = GetSimpleSection($section);
> > @@ -528,9 +543,13 @@
> > # If we got a list of tests to run, invoke the run engine to
> > # actually run them.
> > if ($ret && $ret->{test_result}) {
> > - MTT::Test::RunEngine::RunEngine($ini, $section, $install_dir,
> > + my $return_value;
> > + $return_value = MTT::Test::RunEngine::RunEngine($ini, $section,
> $install_dir,
> > $runs_data_dir, $mpi_details,
> > - $test_build, $force, $ret);
> > + $test_build, $force, $ret,
> $count_total_tests_number);
> > + if ($count_total_tests_number eq "yes"){
> > + $MTT::Globals::Internals->{total_tests_counter} +=
> $return_value;
> > + }
> > }
> >
> > # Unload any loaded environment modules
> > @@ -545,7 +564,6 @@
> > # means reverting to an env snapshot.
> > MTT::EnvImporter::unload(reverse @env_importers);
> > }
> > -
> > }
> >
> >
> #--------------------------------------------------------------------------
> >
> > Modified: trunk/lib/MTT/Test/RunEngine.pm
> >
> ==============================================================================
> > --- trunk/lib/MTT/Test/RunEngine.pm (original)
> > +++ trunk/lib/MTT/Test/RunEngine.pm 2012-01-25 06:02:47 EST (Wed, 25
> Jan 2012)
> > @@ -38,10 +38,14 @@
> > my $section;
> > my $mpi_details_name;
> > my $test_run_full_name;
> > +my $break_threshold;
> >
> > # Keep track of how many tests have passed, failed, skipped, and timed
> out
> > my $test_results_count;
> >
> > +#same as test_results_count but global over all envokations of RunEngine
> > +my $test_results_count_global;
> > +
> > # Submit results after each run or after *all* the runs
> > my $report_after_each_result = 0;
> > my $report_results_count = 0;
> > @@ -56,7 +60,7 @@
> > my ($install_dir, $runs_data_dir, $mpi_details, $test_build, $force,
> $ret);
> >
> > # Make sure though, that the $ini remains a global
> > - ($ini, $section, $install_dir, $runs_data_dir, $mpi_details,
> $test_build, $force, $ret) = @_;
> > + ($ini, $section, $install_dir, $runs_data_dir, $mpi_details,
> $test_build, $force, $ret, my $count_total_tests_number) = @_;
> >
> > my $test_results;
> > $group_reports = MTT::Values::Value($ini, "mtt",
> "submit_group_results");
> > @@ -81,12 +85,16 @@
> > my $variants_count_total =
> > $test_count_total * $np_count_total * $argv_count_total *
> $exec_count_total;
> >
> > + if ($count_total_tests_number eq "yes"){
> > + return $variants_count_total;
> > + }
> > # Set some thresholds for an early exit
> > - my $break_threshold;
> > + $break_threshold = undef;
> > $break_threshold->{MTT::Values::PASS} = Value($ini, $section,
> "break_threshold_pass");
> > $break_threshold->{MTT::Values::FAIL} = Value($ini, $section,
> "break_threshold_fail");
> > $break_threshold->{MTT::Values::TIMED_OUT} = Value($ini, $section,
> "break_threshold_timeout");
> > $break_threshold->{MTT::Values::SKIPPED} = Value($ini, $section,
> "break_threshold_skipped");
> > + $break_threshold->{MTT::Values::TIMED_OUT_OR_FAIL} = Value($ini,
> "mtt", "break_threshold_timeout_and_fail");
> >
> > # This boolean value defaults to 0, and allows the user to submit
> results
> > # after each test to ensure at least *some* results are submitted
> (in case
> > @@ -208,25 +216,29 @@
> > $run->{wdir} = $wdir;
> > }
> > # Just one np, or an array of np values?
> > +
> > if (ref($all_np) eq "") {
> > - $test_results->{$all_np} =
> > - _run_one_np($install_dir, $run, $mpi_details, $all_np,
> $force);
> > + $test_results->{$all_np} = _run_one_np($install_dir, $run,
> $mpi_details, $all_np, $force);
> > } else {
> > foreach my $this_np (@$all_np) {
> > # See if we're supposed to terminate.
> > last
> > - if (MTT::Util::time_to_terminate());
> > + if (MTT::Util::time_to_terminate());
> > +
> > + $test_results->{$all_np} = _run_one_np($install_dir,
> $run, $mpi_details, $this_np,$force);
> >
> > - $test_results->{$this_np} =
> > - _run_one_np($install_dir, $run, $mpi_details,
> $this_np,
> > - $force);
> > + last
> > + if
> ($MTT::Globals::Internals->{is_stopped_on_break_threshold});
> > }
> > }
> > +
> > + last
> > + if
> ($MTT::Globals::Internals->{is_stopped_on_break_threshold});
> > ++$test_count;
> >
> > # Write out the "to be saved" test run results
> > MTT::Test::SaveRuns($runs_data_dir);
> > -
> > +
> > $MTT::Test::Run::mpi_details = $save_run_mpi_details;
> >
> > # Output a progress bar
> > @@ -247,6 +259,7 @@
> >
> > MTT::Reporter::QueueSubmit();
> > }
> > +
> > }
> >
> > sub _run_one_np {
> > @@ -290,16 +303,30 @@
> > foreach my $e (@$execs) {
> > # See if we're supposed to terminate.
> > last
> > - if (MTT::Util::time_to_terminate());
> > + if (MTT::Util::time_to_terminate());
> > +
> > _run_one_test($install_dir, $run, $mpi_details, $e,
> $name,
> > - $variant++, $force);
> > + $variant++, $force);
> > +
> > + last
> > + if (MTT::Util::check_break_threshold(
> > + $test_results_count_global,
> > + $break_threshold,
> > +
> $MTT::Globals::Internals->{total_tests_counter})
> > + );
> > }
> > }
> > -
> > + last
> > + if (MTT::Util::check_break_threshold(
> > + $test_results_count_global,
> > + $break_threshold,
> > + $MTT::Globals::Internals->{total_tests_counter})
> > + );
> > +
> > $MTT::Test::Run::test_argv = undef;
> > }
> > }
> > -
> > +
> > $MTT::Test::Run::test_np = undef;
> > }
> >
> > @@ -457,6 +484,13 @@
> > $test_results_count->{$report->{test_result}}++
> > if (exists($report->{test_result}));
> >
> > + $test_results_count_global->{$report->{test_result}}++
> > + if (exists($report->{test_result}));
> > +
> > + $test_results_count_global->{MTT::Values::TIMED_OUT_OR_FAIL}++
> > + if (exists($report->{test_result}) &&
> > + (MTT::Values::FAIL == $report->{test_result} ||
> MTT::Values::TIMED_OUT == $report->{test_result}));
> > +
> > # If there is an after_each step, run it
> > $ENV{MTT_TEST_RUN_RESULT_MESSAGE} =
> > (MTT::Values::PASS == $report->{test_result} ? "passed" :
> >
> > Modified: trunk/lib/MTT/Util.pm
> >
> ==============================================================================
> > --- trunk/lib/MTT/Util.pm (original)
> > +++ trunk/lib/MTT/Util.pm 2012-01-25 06:02:47 EST (Wed, 25 Jan 2012)
> > @@ -205,6 +205,12 @@
> >
> > if (($count->{$result} / $total) > $threshold->{$result}) {
> > Verbose("--> Threshold ($per) exceeded for
> \"$result_label\": $count->{$result} out of $total.\n");
> > + $MTT::Globals::Internals->{is_stopped_on_break_threshold} =
> "true";
> > +
> $MTT::Globals::Internals->{stopped_on_break_threshold_message} = "-->
> Threshold ($per) exceeded for \"$result_label\": $count->{$result} out of
> $total.\n";
> > + print STDOUT "--> Threshold ($per) exceeded for
> \"$result_label\": $count->{$result} out of $total.\n";
> > + if
> ($MTT::Globals::Internals->{is_stopped_on_break_threshold}){
> > + print STDOUT "0xdeadbeef: it works";
> > + }
> > return 1;
> > }
> > }
> >
> > Modified: trunk/lib/MTT/Values.pm
> >
> ==============================================================================
> > --- trunk/lib/MTT/Values.pm (original)
> > +++ trunk/lib/MTT/Values.pm 2012-01-25 06:02:47 EST (Wed, 25 Jan 2012)
> > @@ -45,6 +45,7 @@
> > PASS => 1,
> > SKIPPED => 2,
> > TIMED_OUT => 3,
> > + TIMED_OUT_OR_FAIL =>4,
> > };
> >
> > # Map to human-readable English labels
> > @@ -53,7 +54,7 @@
> > $result_messages->{MTT::Values::PASS} = "pass";
> > $result_messages->{MTT::Values::TIMED_OUT} = "timeout";
> > $result_messages->{MTT::Values::SKIPPED} = "skipped";
> > -
> > +$result_messages->{MTT::Values::TIMED_OUT_OR_FAIL} = "timeout_and_fail";
> > # current $ini and $section parameters (we use it in funclets)
> > our $evaluate_string_ini;
> > our $evaluate_string_section;
> > _______________________________________________
> > mtt-svn mailing list
> > mtt-svn_at_[hidden]
> > http://www.open-mpi.org/mailman/listinfo.cgi/mtt-svn
>
>
> --
> Jeff Squyres
> jsquyres_at_[hidden]
> For corporate legal information go to:
> http://www.cisco.com/web/about/doing_business/legal/cri/
>
>
> _______________________________________________
> mtt-devel mailing list
> mtt-devel_at_[hidden]
> http://www.open-mpi.org/mailman/listinfo.cgi/mtt-devel
>