Open MPI logo

MTT Devel Mailing List Archives

  |   Home   |   Support   |   FAQ   |   all MTT Devel mailing list

Subject: Re: [MTT devel] [MTT svn] svn:mtt-svn r1440
From: Jeffrey Squyres (jsquyres_at_[hidden])
Date: 2012-03-02 11:36:54


Sweet!

Can you document this on the wiki in the listing of all the INI fields?

On Mar 2, 2012, at 10:27 AM, miked_at_[hidden] wrote:

> Author: miked
> Date: 2012-03-02 10:27:34 EST (Fri, 02 Mar 2012)
> New Revision: 1440
> URL: https://svn.open-mpi.org/trac/mtt/changeset/1440
>
> Log:
> Now it is possible to shuffle tests in mtt. The parameter that enables this feature is:
> shuffle_tests=
> it can be assigned a comma separated list of allowed values: sections, tests, execs, nps, args, all.
>
> Notations:
> sections - shuffle test_run sections found in mtt
> tests - shuffle test executables if any (we have single executable per test_run)
> execs - shuffle exec command line present in main [MTT] section
> nps - shuffle runs with different nps
> args - shuffle executable arguments
> all - enable all the above.
>
>
> Text files modified:
> trunk/lib/MTT/Globals.pm | 65 +++++++++++----------------------------
> trunk/lib/MTT/Reporter/Email.pm | 6 ++-
> trunk/lib/MTT/Reporter/TextFile.pm | 2
> trunk/lib/MTT/Test/Run.pm | 7 +++
> trunk/lib/MTT/Test/RunEngine.pm | 61 +++++++++++++++++++++++++++++--------
> trunk/lib/MTT/Util.pm | 14 ++++++-
> 6 files changed, 89 insertions(+), 66 deletions(-)
>
> Modified: trunk/lib/MTT/Globals.pm
> ==============================================================================
> --- trunk/lib/MTT/Globals.pm (original)
> +++ trunk/lib/MTT/Globals.pm 2012-03-02 10:27:34 EST (Fri, 02 Mar 2012)
> @@ -80,8 +80,7 @@
>
> save_intermediate_report => undef,
> save_intermediate_report_enable => undef,
> - ini_value_run_for => undef,
> - start_time => localtime,
> + shuffle_tests => undef,
> description => undef,
> };
>
> @@ -181,50 +180,24 @@
> $Values->{save_intermediate_report_enable} = $val;
> }
>
> -# $val = MTT::Values::Value($ini, "MTT", "finish_at");
> -# if (defined($val)) {
> -# #finish_at format: hh:mm[dd/MM]
> -# $val =~ m/(\d\d:\d\d)(\[(\d\d)\/(\d\d)\])*/;
> -# my $stop_time = $1.":00";
> -# my $stop_day = $3;
> -# my $stop_month =$4;
> -# my @timeData = localtime(time);
> -# #@timedata : [0]secs [1]minutes [2]hours [3]days [4]month-1
> -# my $finish_time = MTT::Util::parse_time_to_seconds($stop_time);
> -#
> -#
> -# my $local_time=$timeData[2].":".$timeData[1].":".$timeData[0];
> -# my $current_time = MTT::Util::parse_time_to_seconds($local_time);
> -#
> -#
> -#
> -# my $secs_left;
> -# if ($current_time < $finish_time){
> -# $secs_left = $finish_time-$current_time;
> -# } else {
> -# $secs_left = $finish_time+24*3600-$current_time;
> -# }
> -# $Values->{ini_value_run_for} = $secs_left;
> -#
> -# if (!$stop_month){
> -# if ($stop_day){
> -# if ($stop_day < $timeData[3]){
> -# MTT::Messages::Warning("Stop date is less than start date: disabling stop_at feature");
> -# $Values->{ini_value_run_for} = undef;
> -# }else if ($stop_day == $timeData[3]){
> -# if ($finish_time < $current_time){
> -# MTT::Messages::Warning("The stop is less than start time: disabling stop_at feature");
> -# }
> -# }else{
> -# if ($finish_time > $current_time){
> -# $secs_left += 24*3600;
> -# }
> -# }
> -# }
> -# }
> -# printf "secs_left = ",$secs_left, "hours_left = ",$secs_left/3600,"\n";
> -#
> -# }
> +
> + $val = MTT::Values::Value($ini, "MTT", "shuffle_tests");
> + if (defined($val)) {
> + my @shuffles = grep length, split(/\s*,\s*/,$val);
> + my @allowed_shuffles = ('sections', 'tests', 'execs', 'nps', 'args', 'all');
> + my %allowed_map = map { $_ => 1} @allowed_shuffles;
> + foreach my $sh (@shuffles){
> + if (!exists($allowed_map{$sh})){
> + MTT::Messages::Error("<$sh> is not allowed shuffle_tests value. Allowed are:", join(',',@allowed_shuffles),".");
> + }
> + $Values->{shuffle_tests}->{$sh} = 1;
> + }
> + if ($Values->{shuffle_tests}->{all}){
> + foreach my $allow (@allowed_shuffles){
> + $Values->{shuffle_tests}->{$allow} = 1;
> + }
> + }
> + }
> }
>
> #--------------------------------------------------------------------------
>
> Modified: trunk/lib/MTT/Reporter/Email.pm
> ==============================================================================
> --- trunk/lib/MTT/Reporter/Email.pm (original)
> +++ trunk/lib/MTT/Reporter/Email.pm 2012-03-02 10:27:34 EST (Fri, 02 Mar 2012)
> @@ -148,8 +148,10 @@
> $to = Value( $Ini, $Section, "email_to" );
> }
> }
> - my $body = $subject."\n".$footer."\n";
> - MTT::Mail::Send($subject, $to, $from, $body );
> + my $body = $footer."\n";
> + if ($to){
> + MTT::Mail::Send($subject, $to, $from, $body );
> + }
> }
>
> 1;
>
> Modified: trunk/lib/MTT/Reporter/TextFile.pm
> ==============================================================================
> --- trunk/lib/MTT/Reporter/TextFile.pm (original)
> +++ trunk/lib/MTT/Reporter/TextFile.pm 2012-03-02 10:27:34 EST (Fri, 02 Mar 2012)
> @@ -142,7 +142,7 @@
> sub Flush{
> my ($info, $entries) = @_;
> my @results_to_flush = @results;
> - push(@results_to_flush, $entries);
> + push(@results_to_flush, $entries) if $entries;
> _summary_report(\@results_to_flush, "yes")
> if (@results_to_flush);
>
>
> Modified: trunk/lib/MTT/Test/Run.pm
> ==============================================================================
> --- trunk/lib/MTT/Test/Run.pm (original)
> +++ trunk/lib/MTT/Test/Run.pm 2012-03-02 10:27:34 EST (Fri, 02 Mar 2012)
> @@ -77,7 +77,12 @@
>
> # Go through all the sections in the ini file looking for section
> # names that begin with "Test run:"
> - foreach my $section ($ini->Sections()) {
> + my @sects = $ini->Sections();
> + if ($MTT::Globals::Values->{shuffle_tests}->{sections}){
> + MTT::Util::shuffle(\@sects);
> + }
> +
> + foreach my $section (@sects) {
> # See if we're supposed to terminate. Only check in the
> # outtermost and innermost loops (even though we *could* check
> # at every loop level); that's good enough.
>
> Modified: trunk/lib/MTT/Test/RunEngine.pm
> ==============================================================================
> --- trunk/lib/MTT/Test/RunEngine.pm (original)
> +++ trunk/lib/MTT/Test/RunEngine.pm 2012-03-02 10:27:34 EST (Fri, 02 Mar 2012)
> @@ -39,7 +39,7 @@
> my $mpi_details_name;
> my $test_run_full_name;
> my $break_threshold;
> -
> +my $shuffle;
> # Keep track of how many tests have passed, failed, skipped, and timed out
> my $test_results_count;
>
> @@ -52,7 +52,6 @@
> my $report_after_n_results;
> my $prev_section_name = undef;
> my $group_reports = 0;
> -my $tests_performed = 0;
>
> #--------------------------------------------------------------------------
>
> @@ -96,6 +95,10 @@
> $break_threshold->{MTT::Values::TIMED_OUT} = Value($ini, $section, "break_threshold_timeout");
> $break_threshold->{MTT::Values::SKIPPED} = Value($ini, $section, "break_threshold_skipped");
> $break_threshold->{MTT::Values::TIMED_OUT_OR_FAIL} = Value($ini, "mtt", "break_threshold_timeout_and_fail");
> +
> + $shuffle = undef;
> + _register_shuffles($ini,$section);
> +
> my $reports_per_job = Value($ini, "mtt", "reports_per_job");
> # This boolean value defaults to 0, and allows the user to submit results
> # after each test to ensure at least *some* results are submitted (in case
> @@ -167,6 +170,10 @@
> $verbose_out = 0;
> my $test_count = 0;
> my $printed = 0;
> + if ($shuffle->{tests}){
> + MTT::Util::shuffle(\@{$ret->{tests}});
> + }
> +
> foreach my $run (@{$ret->{tests}}) {
>
> # See if we're supposed to terminate.
> @@ -221,6 +228,9 @@
> if (ref($all_np) eq "") {
> $test_results->{$all_np} = _run_one_np($install_dir, $run, $mpi_details, $all_np, $force);
> } else {
> + if ($shuffle->{nps}){
> + MTT::Util::shuffle($all_np);
> + }
> foreach my $this_np (@$all_np) {
> # See if we're supposed to terminate.
> last
> @@ -288,13 +298,19 @@
> if (ref($all_argv) eq "") {
> $all_argv = [$all_argv];
> }
> -
> + if ($shuffle->{args}){
> + MTT::Util::shuffle($all_argv);
> + }
> foreach my $this_argv (@$all_argv) {
> $MTT::Test::Run::test_argv = $this_argv;
>
> # Get all the exec's for this one np
> my $execs = MTT::Values::EvaluateString($mpi_details->{exec}, $ini, $mpi_details_name);
> -
> + if (ref($execs) ne "" && $shuffle->{execs}){
> + MTT::Util::shuffle($execs);
> + }
> +
> +
> # If we just got one, run it. Otherwise, loop over running them.
> if (ref($execs) eq "") {
> _run_one_test($install_dir, $run, $mpi_details, $execs, $name, 1,
> @@ -497,14 +513,7 @@
> if (exists($report->{test_result}) &&
> (MTT::Values::FAIL == $report->{test_result} || MTT::Values::TIMED_OUT == $report->{test_result}));
>
> - $tests_performed++;
>
> - #print "\t\t--> Test #",$tests_performed," done\n";
> -
> -
> - #MTT::Reporter::Flush();
> - #MTT::Reporter::TextFile::deSubmit();
> -
> # If there is an after_each step, run it
> $ENV{MTT_TEST_RUN_RESULT_MESSAGE} =
> (MTT::Values::PASS == $report->{test_result} ? "passed" :
> @@ -558,6 +567,32 @@
> }
> }
> }
> -
> -
> +sub _register_shuffles{
> + my ($ini,$section) = @_;
> +
> + my $val = MTT::Values::Value($ini, $section, "shuffle_tests");
> + if (defined($val)) {
> + my @shuffles = grep length, split(/\s*,\s*/,$val);
> + my @allowed_shuffles = ('tests', 'execs', 'nps', 'args', 'all', 'none');
> + my %allowed_map = map { $_ => 1} @allowed_shuffles;
> + foreach my $sh (@shuffles){
> + if (!exists($allowed_map{$sh})){
> + MTT::Messages::Error("<$sh> is not allowed section specific shuffle_tests value. Allowed are:", join(',',@allowed_shuffles),".");
> + }
> + $shuffle->{$sh} = 1;
> + }
> + if ($shuffle->{none}){
> + $shuffle = undef;
> + }
> + if ($shuffle->{all}){
> + foreach my $allow (@allowed_shuffles){
> + $shuffle->{$allow} = 1;
> + }
> + }
> +
> + }
> + else{
> + $shuffle = $MTT::Globals::Values->{shuffle_tests};
> + }
> +}
> 1;
>
> Modified: trunk/lib/MTT/Util.pm
> ==============================================================================
> --- trunk/lib/MTT/Util.pm (original)
> +++ trunk/lib/MTT/Util.pm 2012-03-02 10:27:34 EST (Fri, 02 Mar 2012)
> @@ -208,9 +208,6 @@
> $MTT::Globals::Internals->{is_stopped_on_break_threshold} = "true";
> $MTT::Globals::Internals->{stopped_on_break_threshold_message} = "--> Threshold ($per) exceeded for \"$result_label\": $count->{$result} out of $total.\n";
> print STDOUT "--> Threshold ($per) exceeded for \"$result_label\": $count->{$result} out of $total.\n";
> - if ($MTT::Globals::Internals->{is_stopped_on_break_threshold}){
> - print STDOUT "0xdeadbeef: it works";
> - }
> return 1;
> }
> }
> @@ -395,4 +392,15 @@
> }
> }
>
> +#--------------------------------------------------------------------------
> +sub shuffle{
> + # Shuffle an array given via reference randomly
> + my $array = shift;
> + my $i;
> + for ($i = @$array; --$i; ) {
> + my $j = int rand ($i+1);
> + next if $i == $j;
> + @$array[$i,$j] = @$array[$j,$i];
> + }
> +}
> 1;
> _______________________________________________
> mtt-svn mailing list
> mtt-svn_at_[hidden]
> http://www.open-mpi.org/mailman/listinfo.cgi/mtt-svn

-- 
Jeff Squyres
jsquyres_at_[hidden]
For corporate legal information go to: http://www.cisco.com/web/about/doing_business/legal/cri/