From: Jeff Squyres (jsquyres_at_[hidden])
Date: 2006-11-30 09:30:36
On Nov 30, 2006, at 9:29 AM, Ethan Mallove wrote:
> Sweet! I guess the only ''drawback'' is the meta-data is
> only accessable to perl, but who cares.
That was my thought.
> I bet this fix will
> make #118 easier.
>
> http://svn.open-mpi.org/trac/mtt/ticket/118
> "Slim down result submissions to the bare essentials"
Hrm. Maybe. The actual data is the same -- all I did was change the
format in which it was saved to disk.
>
> On Thu, Nov/30/2006 07:55:14AM, jsquyres_at_[hidden] wrote:
>> Author: jsquyres
>> Date: 2006-11-30 07:55:07 EST (Thu, 30 Nov 2006)
>> New Revision: 461
>>
>> Removed:
>> trunk/lib/XML/
>> Modified:
>> trunk/lib/MTT/Files.pm
>> trunk/lib/MTT/MPI.pm
>> trunk/lib/MTT/Test.pm
>> trunk/lib/MTT/Test/RunEngine.pm
>>
>> Log:
>> Fixes #144
>>
>> This commit is a bunch of things to improve the meta data handling
>> and
>> storage in the MTT client:
>>
>> * Remove the use of XML::Simple and use Data::Dumper instead. This
>> allows us to remove the costly/slow (and error-prone when
>> coding up
>> new data layers) data transformation for each of the datasets that
>> were saved out to disk. The output data is about the same size,
>> but we avoid the [potentially] huge memory transformation before
>> the write, and for large data sets (e.g., Test run data), can save
>> quite a bit of time (e.g., ~3 seconds with Data::Dumper vs. ~10
>> seconds with XML::Simple).
>> * Split the Test run data up into multiple files instead of one huge
>> file. This allows us to write only the new data after tests have
>> run rather than having to write ''all'' the data ''every'' time.
>> It makes the reading and writing of the test run data a little
>> more
>> complex, but the performance win is worth it.
>> * Write the Test Run data [slightly] less frequently -- i.e., write
>> out a [small] batch of new Test Run data periodically rather than
>> writing after ''every'' test.
>>
>>
>> Modified: trunk/lib/MTT/Files.pm
>> =====================================================================
>> =========
>> --- trunk/lib/MTT/Files.pm (original)
>> +++ trunk/lib/MTT/Files.pm 2006-11-30 07:55:07 EST (Thu, 30 Nov 2006)
>> @@ -40,8 +40,15 @@
>>
>>
>> #--------------------------------------------------------------------
>> ------
>>
>> -sub mkdir {
>> +sub safe_mkdir {
>> my ($dir) = @_;
>> + MTT::Files::mkdir($dir, 1);
>> +}
>> +
>> +#-------------------------------------------------------------------
>> -------
>> +
>> +sub mkdir {
>> + my ($dir, $safe) = @_;
>>
>> my $c = cwd();
>> Debug("Making dir: $dir (cwd: $c)\n");
>> @@ -58,6 +65,9 @@
>> foreach my $p (@parts) {
>> next if (! $p);
>>
>> + $p = make_safe_filename($p)
>> + if ($safe);
>> +
>> $str .= "$p";
>> if (! -d $str) {
>> Debug("$str does not exist -- creating\n");
>> @@ -402,6 +412,8 @@
>> return 1;
>> }
>>
>> +#-------------------------------------------------------------------
>> -------
>> +
>> # Copy infile or stdin to a unique file in /tmp
>> sub copyfile {
>>
>> @@ -432,4 +444,64 @@
>> return $outfile;
>> }
>>
>> +#-------------------------------------------------------------------
>> -------
>> +
>> +sub load_dumpfile {
>> + my ($filename, $data) = @_;
>> +
>> + # Check that the file is there
>> + return
>> + if (! -r $filename);
>> +
>> + # Get the file size
>> + my ($dev, $ino, $mode, $nlink, $uid, $gid, $rdev, $size,
>> + $atime, $mtime, $ctime, $blksize, $blocks) = stat
>> ($filename);
>> +
>> + # Read it in
>> + open IN, $filename;
>> + my $tmp;
>> + read(IN, $tmp, $size);
>> + close IN;
>> +
>> + # It's in Dumper format. How many $VARx's are there?
>> + return
>> + if (! $tmp =~ m/\$VAR[0-9]+/g);
>> + my $count = 0;
>> + ++$count
>> + while ($tmp =~ m/\$VAR[0-9]+/g);
>> +
>> + # We know how many $VARx's there are. Build up a string to
>> eval.
>> + my $str;
>> + my $var_num = 1;
>> + while ($var_num <= $count) {
>> + $str .= "my \$VAR$var_num;\n";
>> + ++$var_num;
>> + }
>> + $str .= "eval \$tmp;\n";
>> + my $var_num = 1;
>> + while ($var_num <= $count) {
>> + $str .= "\$\$data->{VAR$var_num} = \$VAR$var_num;\n";
>> + ++$var_num;
>> + }
>> + eval $str;
>> +}
>> +
>> +#-------------------------------------------------------------------
>> -------
>> +
>> +sub save_dumpfile {
>> + my ($filename) = @_;
>> + shift;
>> +
>> + # Serialize
>> + my $d = new Data::Dumper([@_]);
>> + $d->Purity(1)->Indent(1);
>> +
>> + open FILE, ">$filename.new";
>> + print FILE $d->Dump;
>> + close FILE;
>> +
>> + # Atomically move it onto the old file
>> + rename("$filename.new", $filename);
>> +}
>> +
>> 1;
>>
>> Modified: trunk/lib/MTT/MPI.pm
>> =====================================================================
>> =========
>> --- trunk/lib/MTT/MPI.pm (original)
>> +++ trunk/lib/MTT/MPI.pm 2006-11-30 07:55:07 EST (Thu, 30 Nov 2006)
>> @@ -15,10 +15,9 @@
>> package MTT::MPI;
>>
>> use strict;
>> -use XML::Simple qw(:strict);
>> use MTT::MPI::Get;
>> use MTT::MPI::Install;
>> -use Data::Dumper;
>> +use MTT::Files;
>>
>>
>> #--------------------------------------------------------------------
>> ------
>>
>> @@ -31,16 +30,10 @@
>>
>> #--------------------------------------------------------------------
>> ------
>>
>> # Filename where list of MPI sources is kept
>> -my $sources_data_filename = "mpi_sources.xml";
>> -
>> -# XML options for the MPI sources
>> -my $sources_xs;
>> +my $sources_data_filename = "mpi_sources.dump";
>>
>> # Filename where list of MPI installs is kept
>> -my $installs_data_filename = "mpi_installs.xml";
>> -
>> -# XML options for the MPI installs
>> -my $installs_xs;
>> +my $installs_data_filename = "mpi_installs.dump";
>>
>>
>> #--------------------------------------------------------------------
>> ------
>>
>> @@ -60,65 +53,17 @@
>>
>>
>> #--------------------------------------------------------------------
>> ------
>>
>> -sub _scrub {
>> - my $h = shift;
>> -
>> - foreach my $k (keys(%$h)) {
>> - if (ref($h->{$k}) eq "") {
>> - # Remove bad characters
>> - $h->{$k} =~ s/\000/<NULL>/g;
>> - } else {
>> - _scrub($h->{$k});
>> - }
>> - }
>> -}
>> -
>> -
>> #--------------------------------------------------------------------
>> ------
>> -
>> -sub _setup_sources_xml {
>> - $sources_xs = new XML::Simple(KeyAttr => { mpi_get =>
>> "simple_section_name",
>> - mpi_version =>
>> "version",
>> - },
>> - ForceArray => [ "mpi_get",
>> - "mpi_version",
>> - ],
>> - AttrIndent => 1,
>> - RootName => "mpi_sources",
>> - );
>> -}
>> -
>> -
>> #--------------------------------------------------------------------
>> ------
>> -
>> +use Data::Dumper;
>> sub LoadSources {
>> my ($dir) = @_;
>>
>> # Explicitly delete anything that was there
>> $MTT::MPI::sources = undef;
>>
>> - _setup_sources_xml()
>> - if (!$sources_xs);
>> -
>> # If the file exists, read it in
>> - if (-f "$dir/$sources_data_filename") {
>> - my $in = $sources_xs->XMLin("$dir/$sources_data_filename");
>> -
>> - # Now transform this to the form suitable for
>> - # $MTT::MPI::sources (see comment in SaveSources)
>> -
>> - # For each MPI get section
>> - foreach my $mpi_get_key (keys(%{$in->{mpi_get}})) {
>> - my $mpi_get = $in->{mpi_get}->{$mpi_get_key};
>> -
>> - # For each version
>> - foreach my $mpi_version_key (keys(%{$mpi_get->
>> {mpi_version}})) {
>> - my $mpi_version = $mpi_get->{mpi_version}->
>> {$mpi_version_key};
>> -
>> - $MTT::MPI::sources->{$mpi_get_key}->
>> {$mpi_version_key} = $mpi_version;
>> - $MTT::MPI::sources->{$mpi_get_key}->
>> {$mpi_version_key}->{simple_section_name} = $mpi_get_key;
>> - $MTT::MPI::sources->{$mpi_get_key}->
>> {$mpi_version_key}->{version} = $mpi_version_key;
>> - }
>> - }
>> - }
>> + my $data;
>> + MTT::Files::load_dumpfile("$dir/$sources_data_filename", \
>> $data);
>> + $MTT::MPI::sources = $data->{VAR1};
>> }
>>
>>
>> #--------------------------------------------------------------------
>> ------
>> @@ -126,54 +71,8 @@
>> sub SaveSources {
>> my ($dir) = @_;
>>
>> - _setup_sources_xml()
>> - if (!$sources_xs);
>> -
>> - # Transform $MTT::MPI::sources to something XML::Simple can
>> write
>> - # into valid XML (since our values can [and will] contain :'s,
>> - # which are the namespace identifiers in XML)
>> - my $transformed;
>> -
>> - # For each MPI get section
>> - foreach my $mpi_get_key (keys(%$MTT::MPI::sources)) {
>> - my $mpi_get = $MTT::MPI::sources->{$mpi_get_key};
>> -
>> - # For each version
>> - foreach my $mpi_version_key (keys(%$mpi_get)) {
>> - my $mpi_version = $mpi_get->{$mpi_version_key};
>> -
>> - # Deep copy and scrub
>> - my $h;
>> - %$h = %$mpi_version;
>> - _scrub($h);
>> -
>> - $transformed->{mpi_get}->{$mpi_get_key}->
>> {mpi_version}->{$mpi_version_key} = $h;
>> - }
>> - }
>> -
>> - # Write out the file
>> - my $xml = $sources_xs->XMLout($transformed);
>> - my $file = "$dir/$sources_data_filename";
>> - open(FILE, ">$file.new");
>> - print FILE $xml;
>> - close(FILE);
>> - system("mv $file.new $file");
>> -}
>> -
>> -
>> #--------------------------------------------------------------------
>> ------
>> -
>> -sub _setup_installs_xml {
>> - $installs_xs = new XML::Simple(KeyAttr => { mpi_get =>
>> "simple_section_name",
>> - mpi_version =>
>> "version",
>> - mpi_install =>
>> "simple_section_name",
>> - },
>> - ForceArray => [ "mpi_get",
>> - "mpi_version",
>> - "mpi_install",
>> - ],
>> - AttrIndent => 1,
>> - RootName => "mpi_installs",
>> - );
>> + MTT::Files::save_dumpfile("$dir/$sources_data_filename",
>> + $MTT::MPI::sources);
>> }
>>
>>
>> #--------------------------------------------------------------------
>> ------
>> @@ -184,33 +83,10 @@
>> # Explicitly delete anything that was there
>> $MTT::MPI::installs = undef;
>>
>> - _setup_installs_xml()
>> - if (!$installs_xs);
>> -
>> # If the file exists, read it in
>> - if (-f "$dir/$installs_data_filename") {
>> - my $in = $installs_xs->XMLin("$dir/
>> $installs_data_filename");
>> -
>> - # Now transform this to the form suitable for
>> - # $MTT::MPI::installs (see comment in SaveSources). Wow.
>> -
>> - # For each MPI get section
>> - foreach my $mpi_get_key (keys(%{$in->{mpi_get}})) {
>> - my $mpi_get = $in->{mpi_get}->{$mpi_get_key};
>> -
>> - # For each version of each MPI get
>> - foreach my $mpi_version_key (keys(%{$mpi_get->
>> {mpi_version}})) {
>> - my $mpi_version = $mpi_get->{mpi_version}->
>> {$mpi_version_key};
>> -
>> - # For each MPI install section
>> - foreach my $mpi_install_key (keys(%{$mpi_version->
>> {mpi_install}})) {
>> - $MTT::MPI::installs->{$mpi_get_key}->
>> {$mpi_version_key}->{$mpi_install_key} =
>> - $in->{mpi_get}->{$mpi_get_key}->
>> {mpi_version}->{$mpi_version_key}->{mpi_install}->{$mpi_install_key};
>> - $MTT::MPI::installs->{$mpi_get_key}->
>> {$mpi_version_key}->{$mpi_install_key}->{simple_section_name} =
>> $mpi_install_key;
>> - }
>> - }
>> - }
>> - }
>> + my $data;
>> + MTT::Files::load_dumpfile("$dir/$installs_data_filename", \
>> $data);
>> + $MTT::MPI::installs = $data->{VAR1};
>> }
>>
>>
>> #--------------------------------------------------------------------
>> ------
>> @@ -218,42 +94,8 @@
>> sub SaveInstalls {
>> my ($dir) = @_;
>>
>> - _setup_installs_xml()
>> - if (!$installs_xs);
>> -
>> - # Transform $MTT::MPI::installs to something XML::Simple can
>> write
>> - # into valid XML (see comment in SaveSources). Wow.
>> - my $transformed;
>> -
>> - # For each MPI get section
>> - foreach my $mpi_get_key (keys(%{$MTT::MPI::installs})) {
>> - my $mpi_get = $MTT::MPI::installs->{$mpi_get_key};
>> -
>> - # For each version of that get
>> - foreach my $mpi_version_key (keys(%{$mpi_get})) {
>> - my $mpi_version = $mpi_get->{$mpi_version_key};
>> -
>> - # For each MPI install action
>> - foreach my $mpi_install_key (keys(%{$mpi_version})) {
>> - my $mpi_install = $mpi_version->{$mpi_install_key};
>> -
>> - # Deep copy and scrub
>> - my $h;
>> - %$h = %$mpi_install;
>> - _scrub($h);
>> -
>> - $transformed->{mpi_get}->{$mpi_get_key}->
>> {mpi_version}->{$mpi_version_key}->{mpi_install}->
>> {$mpi_install_key} = $h;
>> - }
>> - }
>> - }
>> -
>> - # Write out the file
>> - my $xml = $installs_xs->XMLout($transformed);
>> - my $file = "$dir/$installs_data_filename";
>> - open(FILE, ">$file.new");
>> - print FILE $xml;
>> - close(FILE);
>> - system("mv $file.new $file");
>> + MTT::Files::save_dumpfile("$dir/$installs_data_filename",
>> + $MTT::MPI::installs);
>> }
>>
>> 1;
>>
>> Modified: trunk/lib/MTT/Test.pm
>> =====================================================================
>> =========
>> --- trunk/lib/MTT/Test.pm (original)
>> +++ trunk/lib/MTT/Test.pm 2006-11-30 07:55:07 EST (Thu, 30 Nov 2006)
>> @@ -13,11 +13,11 @@
>> package MTT::Test;
>>
>> use strict;
>> +use File::Find;
>> use MTT::Test::Get;
>> use MTT::Test::Build;
>> use MTT::Test::Run;
>> -use XML::Simple;
>> -use Data::Dumper;
>> +use MTT::Files;
>>
>>
>> #--------------------------------------------------------------------
>> ------
>>
>> @@ -29,26 +29,24 @@
>>
>> # Exported run tests handle
>> our $runs;
>> +our $runs_to_be_saved;
>>
>>
>> #--------------------------------------------------------------------
>> ------
>>
>> # Filename where list of test sources information is kept
>> -my $sources_data_filename = "test_sources.xml";
>> -
>> -# XML options for the test builds
>> -my $sources_xs;
>> +my $sources_data_filename = "test_sources.dump";
>>
>> # Filename where list of test build information is kept
>> -my $builds_data_filename = "test_builds.xml";
>> +my $builds_data_filename = "test_builds.dump";
>>
>> -# XML options for the test builds
>> -my $builds_xs;
>> +# Subdir where test runs are kept
>> +my $runs_subdir = "test_runs";
>>
>> # Filename where list of test run information is kept
>> -my $runs_data_filename = "test_runs.xml";
>> +my $runs_data_filename = "test_runs.dump";
>>
>> -# XML options for the test runs
>> -my $runs_xs;
>> +# Helper variable for when we're loading test run data
>> +my $load_run_file_start_dir;
>>
>>
>> #--------------------------------------------------------------------
>> ------
>>
>> @@ -76,57 +74,16 @@
>>
>>
>> #--------------------------------------------------------------------
>> ------
>>
>> -sub _scrub {
>> - my $h = shift;
>> -
>> - foreach my $k (keys(%$h)) {
>> - if (ref($h->{$k}) eq "") {
>> - # Remove bad characters
>> - $h->{$k} =~ s/\000/<NULL>/g;
>> - } else {
>> - _scrub($h->{$k});
>> - }
>> - }
>> -}
>> -
>> -
>> #--------------------------------------------------------------------
>> ------
>> -
>> -sub _setup_sources_xml {
>> - $sources_xs = new XML::Simple(KeyAttr => { test_get =>
>> "simple_section_name",
>> - },
>> - ForceArray => [ "test_get",
>> - ],
>> - AttrIndent => 1,
>> - RootName => "test_sources",
>> - );
>> -}
>> -
>> -
>> #--------------------------------------------------------------------
>> ------
>> -
>> sub LoadSources {
>> my ($dir) = @_;
>>
>> # Explicitly delete anything that was there
>> $MTT::Test::sources = undef;
>>
>> - _setup_sources_xml()
>> - if (!$sources_xs);
>> -
>> # If the file exists, read it in
>> - if (-f "$dir/$sources_data_filename") {
>> - my $in = $sources_xs->XMLin("$dir/$sources_data_filename");
>> -
>> - # Now transform this to the form suitable for
>> - # $MTT::Test::sources (see comment in SaveSources)
>> -
>> - # For each Test get section
>> - foreach my $test_get_key (keys(%{$in->{test_get}})) {
>> - my $test_get = $in->{test_get}->{$test_get_key};
>> -
>> - $MTT::Test::sources->{$test_get_key} = $test_get;
>> - $MTT::Test::sources->{$test_get_key}->
>> {simple_section_name} = $test_get_key;
>> - }
>> - }
>> + my $data;
>> + MTT::Files::load_dumpfile("$dir/$sources_data_filename", \
>> $data);
>> + $MTT::Test::sources = $data->{VAR1};
>> }
>>
>>
>> #--------------------------------------------------------------------
>> ------
>> @@ -134,51 +91,8 @@
>> sub SaveSources {
>> my ($dir) = @_;
>>
>> - _setup_sources_xml()
>> - if (!$sources_xs);
>> -
>> - # Transform $MTT::Test::sources to something XML::Simple can
>> write
>> - # into valid XML (since our values can [and will] contain :'s,
>> - # which are the namespace identifiers in XML)
>> - my $transformed;
>> -
>> - # For each Test get section
>> - foreach my $test_get_key (keys(%$MTT::Test::sources)) {
>> - my $test_get = $MTT::Test::sources->{$test_get_key};
>> -
>> - # Deep copy and scrub
>> - my $h;
>> - %$h = %$test_get;
>> - _scrub($h);
>> -
>> - $transformed->{test_get}->{$test_get_key} = $h;
>> - }
>> -
>> - # Write out the file
>> - my $xml = $sources_xs->XMLout($transformed);
>> - my $file = "$dir/$sources_data_filename";
>> - open(FILE, ">$file.new");
>> - print FILE $xml;
>> - close(FILE);
>> - system("mv $file.new $file");
>> -}
>> -
>> -
>> #--------------------------------------------------------------------
>> ------
>> -
>> -sub _setup_builds_xml {
>> - $builds_xs = new XML::Simple(KeyAttr => { mpi_get =>
>> "simple_section_name",
>> - mpi_version =>
>> "version",
>> - mpi_install =>
>> "simple_section_name",
>> - test_build =>
>> "simple_section_name",
>> - },
>> - ForceArray => [ "mpi_get",
>> - "mpi_version",
>> - "mpi_install",
>> - "test_build",
>> - ],
>> - AttrIndent => 1,
>> - RootName => "test_builds",
>> - );
>> + MTT::Files::save_dumpfile("$dir/$sources_data_filename",
>> + $MTT::Test::sources);
>> }
>>
>>
>> #--------------------------------------------------------------------
>> ------
>> @@ -189,39 +103,10 @@
>> # Explicitly delete anything that was there
>> $MTT::Test::builds = undef;
>>
>> - _setup_builds_xml()
>> - if (!$builds_xs);
>> -
>> # If the file exists, read it in
>> - if (-f "$dir/$builds_data_filename") {
>> - my $in = $builds_xs->XMLin("$dir/$builds_data_filename");
>> -
>> - # Now transform this to the form suitable for
>> - # $MTT::Test::builds (see comment in SaveSources). Wow.
>> - # For each MPI get section
>> - foreach my $mpi_get_key (keys(%{$in->{mpi_get}})) {
>> - my $mpi_get = $in->{mpi_get}->{$mpi_get_key};
>> -
>> - # For each version of that MPI
>> - foreach my $mpi_version_key (keys(%{$mpi_get->
>> {mpi_version}})) {
>> - my $mpi_version = $mpi_get->{mpi_version}->
>> {$mpi_version_key};
>> -
>> - # For each MPI install section
>> - foreach my $mpi_install_key (keys(%{$mpi_version->
>> {mpi_install}})) {
>> - my $mpi_install = $mpi_version->{mpi_install}-
>> >{$mpi_install_key};
>> -
>> - # For each test build section
>> - foreach my $test_build_key (keys(%
>> {$mpi_install->{test_build}})) {
>> - my $test_build = $mpi_install->
>> {test_build}->{$test_build_key};
>> -
>> - $MTT::Test::builds->{$mpi_get_key}->
>> {$mpi_version_key}->{$mpi_install_key}->{$test_build_key} =
>> - $in->{mpi_get}->{$mpi_get_key}->
>> {mpi_version}->{$mpi_version_key}->{mpi_install}->
>> {$mpi_install_key}->{test_build}->{$test_build_key};
>> - $MTT::Test::builds->{$mpi_get_key}->
>> {$mpi_version_key}->{$mpi_install_key}->{$test_build_key}->
>> {simple_section_name} = $test_build_key;
>> - }
>> - }
>> - }
>> - }
>> - }
>> + my $data;
>> + MTT::Files::load_dumpfile("$dir/$builds_data_filename", \$data);
>> + $MTT::Test::builds = $data->{VAR1};
>> }
>>
>>
>> #--------------------------------------------------------------------
>> ------
>> @@ -229,135 +114,48 @@
>> sub SaveBuilds {
>> my ($dir) = @_;
>>
>> - _setup_builds_xml()
>> - if (!$builds_xs);
>> -
>> - # Transform $MTT::Test::builds to something XML::Simple can
>> write
>> - # into valid XML (see comment in SaveSources). Wow.
>> - my $transformed;
>> -
>> - # For each MPI get section
>> - foreach my $mpi_get_key (keys(%{$MTT::Test::builds})) {
>> - my $mpi_get = $MTT::Test::builds->{$mpi_get_key};
>> -
>> - # For each version of that MPI
>> - foreach my $mpi_version_key (keys(%{$mpi_get})) {
>> - my $mpi_version = $mpi_get->{$mpi_version_key};
>> -
>> - # For each MPI install section
>> - foreach my $mpi_install_key (keys(%{$mpi_version})) {
>> - my $mpi_install = $mpi_version->{$mpi_install_key};
>> -
>> - # For each test build section
>> - foreach my $test_build_key (keys(%{$mpi_install})) {
>> - my $test_build = $mpi_install->
>> {$test_build_key};
>> -
>> - # Deep copy and scrub
>> - my $h;
>> - %$h = %$test_build;
>> - _scrub($h);
>> -
>> - $transformed->{mpi_get}->{$mpi_get_key}->
>> {mpi_version}->{$mpi_version_key}->{mpi_install}->
>> {$mpi_install_key}->{test_build}->{$test_build_key} = $h;
>> - }
>> - }
>> - }
>> - }
>> -
>> - # Write out the file
>> - my $xml = $builds_xs->XMLout($transformed);
>> - my $file = "$dir/$builds_data_filename";
>> - open(FILE, ">$file.new");
>> - print FILE $xml;
>> - close(FILE);
>> - system("mv $file.new $file");
>> -}
>> -
>> -
>> #--------------------------------------------------------------------
>> ------
>> -
>> -sub _setup_runs_xml {
>> - $runs_xs = new XML::Simple(KeyAttr => { mpi_get =>
>> "simple_section_name",
>> - mpi_version =>
>> "version",
>> - mpi_install =>
>> "simple_section_name",
>> - test_build =>
>> "simple_section_name",
>> - test_run =>
>> "simple_section_name",
>> - test_name => "name",
>> - test_np => "nprocs",
>> - test_cmd => "argv",
>> - },
>> - ForceArray => [ "mpi_get",
>> - "mpi_version",
>> - "mpi_install",
>> - "test_build",
>> - "test_run",
>> - "test_name",
>> - "test_np",
>> - "test_cmd",
>> - ],
>> - AttrIndent => 1,
>> - RootName => "test_runs",
>> - );
>> + MTT::Files::save_dumpfile("$dir/$builds_data_filename",
>> + $MTT::Test::builds);
>> }
>>
>>
>> #--------------------------------------------------------------------
>> ------
>>
>> +use Data::Dumper;
>> sub LoadRuns {
>> my ($dir) = @_;
>>
>> # Explicitly delete anything that was there
>> $MTT::Test::runs = undef;
>>
>> - _setup_runs_xml()
>> - if (!$runs_xs);
>> -
>> - # If the file exists, read it in
>> - if (-f "$dir/$runs_data_filename") {
>> - my $in = $runs_xs->XMLin("$dir/$runs_data_filename");
>> -
>> - # Now transform this to the form suitable for
>> - # $MTT::Test::runs (see comment in SaveSources). Wow.
>> - # For each MPI get section
>> - foreach my $mpi_get_key (keys(%{$in->{mpi_get}})) {
>> - my $mpi_get = $in->{mpi_get}->{$mpi_get_key};
>> -
>> - # For each MPI source
>> - foreach my $mpi_version_key (keys(%{$mpi_get->
>> {mpi_version}})) {
>> - my $mpi_version = $mpi_get->{mpi_version}->
>> {$mpi_version_key};
>> -
>> - # For each MPI install section
>> - foreach my $mpi_install_key (keys(%{$mpi_version->
>> {mpi_install}})) {
>> - my $mpi_install = $mpi_version->{mpi_install}-
>> >{$mpi_install_key};
>> -
>> - # For each test build section
>> - foreach my $test_build_key (keys(%
>> {$mpi_install->{test_build}})) {
>> - my $test_build = $mpi_install->
>> {test_build}->{$test_build_key};
>> -
>> - # For each test run section
>> - foreach my $test_run_key (keys(%
>> {$test_build->{test_run}})) {
>> - my $test_run = $test_build->
>> {test_run}->{$test_run_key};
>> -
>> - # For each test name
>> - foreach my $test_name_key (keys(%
>> {$test_run->{test_name}})) {
>> - my $test_name = $test_run->
>> {test_name}->{$test_name_key};
>> -
>> - # For each np
>> - foreach my $test_np_key (keys(%
>> {$test_name->{test_np}})) {
>> - my $test_np = $test_name->
>> {test_np}->{$test_np_key};
>> -
>> - # For each test command
>> - foreach my $test_cmd_key (keys
>> (%{$test_np->{test_cmd}})) {
>> - my $test_cmd = $test_np->
>> {test_cmd}->{$test_cmd_key};
>> -
>> - $MTT::Test::runs->
>> {$mpi_get_key}->{$mpi_version_key}->{$mpi_install_key}->
>> {$test_build_key}->{$test_run_key}->{$test_name_key}->
>> {$test_np_key}->{$test_cmd_key} =
>> - $in->{mpi_get}->
>> {$mpi_get_key}->{mpi_version}->{$mpi_version_key}->{mpi_install}->
>> {$mpi_install_key}->{test_build}->{$test_build_key}->{test_run}->
>> {$test_run_key}->{test_name}->{$test_name_key}->{test_np}->
>> {$test_np_key}->{test_cmd}->{$test_cmd_key};
>> - }
>> - }
>> - }
>> - }
>> - }
>> - }
>> - }
>> - }
>> - }
>> + # See SaveRuns, below, for an explanation. We traverse
>> + # directories looking for dump files and read them into the
>> + # appropriate section of the $MTT::Test::runs hash.
>> +
>> + $load_run_file_start_dir = "$dir/$runs_subdir";
>> + find(\&load_run_file, $load_run_file_start_dir)
>> + if (-d $load_run_file_start_dir);
>> +}
>> +
>> +sub load_run_file {
>> + # We only want files named $runs_data_filename
>> + return 0
>> + if (! -f $_ || $runs_data_filename ne $_);
>> +
>> + # Read in the file
>> + my $data;
>> + MTT::Files::load_dumpfile($File::Find::name, \$data);
>> +
>> + # Put the loaded data in the hash in the right place. Per
>> + # SaveRuns(), below, we look at the key values in $data->
>> {VAR1} to
>> + # know where to put $data->{VAR2} in the $MTT::Test::runs hash.
>> + my $str = "\$MTT::Test::runs";
>> + my $k = 1;
>> + do {
>> + $str .= "->{\"" . $data->{VAR1}->{$k} . "\"}";
>> + ++$k;
>> + } while (exists($data->{VAR1}->{$k}));
>> + $str .= " = \$data->{VAR2}";
>> + eval $str;
>> }
>>
>>
>> #--------------------------------------------------------------------
>> ------
>> @@ -365,15 +163,25 @@
>> sub SaveRuns {
>> my ($dir) = @_;
>>
>> - _setup_runs_xml()
>> - if (!$runs_xs);
>> + # Because test run data can get very, very large, we break it up
>> + # and store it in lots of smaller files so that we can write out
>> + # to disk in small portions.
>> +
>> + # Test runs are stored in the hash with this order of keys:
>> +
>> + # mpi_get_simple_section_name
>> + # mpi_version
>> + # mpi_install_simple_section_name
>> + # test_build_simple_section_name
>> + # test_run_simple_section_name
>> + # test_name
>> + # np
>> + # command
>>
>> - # Transform $MTT::Test::runs to something XML::Simple can write
>> - # into valid XML (see comment in SaveSources). Wow.
>> - my $transformed;
>> + # We save from test_name and down in a single file.
>>
>> # For each MPI get section
>> - foreach my $mpi_get_key (keys(%{$MTT::Test::runs})) {
>> + foreach my $mpi_get_key (keys(%
>> {$MTT::Test::runs_to_be_saved})) {
>> my $mpi_get = $MTT::Test::runs->{$mpi_get_key};
>>
>> # For each source of that MPI
>> @@ -395,23 +203,22 @@
>> # For each test name
>> foreach my $test_name_key (keys(%
>> {$test_run})) {
>> my $test_name = $test_run->
>> {$test_name_key};
>> -
>> - # For each np
>> - foreach my $test_np_key (keys(%
>> {$test_name})) {
>> - my $test_np = $test_name->
>> {$test_np_key};
>> -
>> - # For each cmd
>> - foreach my $test_cmd_key (keys(%
>> {$test_np})) {
>> - my $test_cmd = $test_np->
>> {$test_cmd_key};
>> -
>> - # Deep copy and scrub
>> - my $h;
>> - %$h = %$test_cmd;
>> - _scrub($h);
>> -
>> - $transformed->{mpi_get}->
>> {$mpi_get_key}->{mpi_version}->{$mpi_version_key}->{mpi_install}->
>> {$mpi_install_key}->{test_build}->{$test_build_key}->{test_run}->
>> {$test_run_key}->{test_name}->{$test_name_key}->{test_np}->
>> {$test_np_key}->{test_cmd}->{$test_cmd_key} = $h;
>> - }
>> - }
>> +
>> + my $file = MTT::Files::safe_mkdir
>> ("$dir/$runs_subdir/$mpi_get_key/$mpi_version_key/$mpi_install_key/
>> $test_build_key/$test_run_key/$test_name_key") . "/
>> $runs_data_filename";
>> +
>> + # We need to save two items in test run
>> + # data files -- the actual data and
>> where
>> + # it belongs in the $MTT::Test::runs
>> hash.
>> + my $hashname = {
>> + 1 => $mpi_get_key,
>> + 2 => $mpi_version_key,
>> + 3 => $mpi_install_key,
>> + 4 => $test_build_key,
>> + 5 => $test_run_key,
>> + 6 => $test_name_key,
>> + };
>> + MTT::Files::save_dumpfile($file,
>> $hashname,
>> + $test_name);
>> }
>> }
>> }
>> @@ -419,13 +226,8 @@
>> }
>> }
>>
>> - # Write out the file
>> - my $xml = $runs_xs->XMLout($transformed);
>> - my $file = "$dir/$runs_data_filename";
>> - open(FILE, ">$file.new");
>> - print FILE $xml;
>> - close(FILE);
>> - system("mv $file.new $file");
>> + # Explicitly reset the test runs to be saved
>> + $MTT::Test::runs_to_be_saved = undef;
>> }
>>
>> 1;
>>
>> Modified: trunk/lib/MTT/Test/RunEngine.pm
>> =====================================================================
>> =========
>> --- trunk/lib/MTT/Test/RunEngine.pm (original)
>> +++ trunk/lib/MTT/Test/RunEngine.pm 2006-11-30 07:55:07 EST (Thu,
>> 30 Nov 2006)
>> @@ -67,6 +67,9 @@
>> }
>> }
>> ++$count;
>> +
>> + # Write out the "to be saved" test run results
>> + MTT::Test::SaveRuns($top_dir);
>>
>> # Output a progress bar
>> if ($verbose_out > 50) {
>> @@ -189,8 +192,21 @@
>> # Assume that the Analyze module will output one line
>> ++$verbose_out;
>>
>> - $MTT::Test::runs->{$mpi_details->
>> {mpi_get_simple_section_name}}->{$mpi_details->{version}}->
>> {$mpi_details->{mpi_install_simple_section_name}}->{$run->
>> {test_build_simple_section_name}}->{$run->{simple_section_name}}->
>> {$name}->{$MTT::Test::Run::test_np}->{$cmd} = $report;
>> - MTT::Test::SaveRuns($top_dir);
>> + # For Test Runs data, we have two datasets: the "to be saved"
>> set
>> + # and the "all results" set. The "to be saved" set is a
>> + # relatively small set of data that is written out to disk
>> + # periodically (i.e., augmenting what has already been written
>> + # out). The "all results" set is everything that has
>> occurred so
>> + # far. We do this because the "all results" set can get *very*
>> + # large, so we don't want to write out the whole thing every
>> time
>> + # we save the results to disk.
>> +
>> + # So save this new result in both the "to be saved" and "all
>> + # results" sets. We'll write out the "to be saved" results
>> + # shortly.
>> +
>> + $MTT::Test::runs_to_be_saved->{$mpi_details->
>> {mpi_get_simple_section_name}}->{$mpi_details->{version}}->
>> {$mpi_details->{mpi_install_simple_section_name}}->{$run->
>> {test_build_simple_section_name}}->{$run->{simple_section_name}}->
>> {$name}->{$MTT::Test::Run::test_np}->{$cmd} =
>> + $MTT::Test::runs->{$mpi_details->
>> {mpi_get_simple_section_name}}->{$mpi_details->{version}}->
>> {$mpi_details->{mpi_install_simple_section_name}}->{$run->
>> {test_build_simple_section_name}}->{$run->{simple_section_name}}->
>> {$name}->{$MTT::Test::Run::test_np}->{$cmd} = $report;
>> MTT::Reporter::QueueAdd("Test Run", $run->
>> {simple_section_name}, $report);
>>
>> # If there is an after_each step, run it
>> _______________________________________________
>> mtt-svn mailing list
>> mtt-svn_at_[hidden]
>> http://www.open-mpi.org/mailman/listinfo.cgi/mtt-svn
>
> --
> -Ethan
> _______________________________________________
> mtt-users mailing list
> mtt-users_at_[hidden]
> http://www.open-mpi.org/mailman/listinfo.cgi/mtt-users
-- Jeff Squyres Server Virtualization Business Unit Cisco Systems