From: Ethan Mallove (ethan.mallove_at_[hidden])
Date: 2006-11-30 09:29:01
Sweet! I guess the only ''drawback'' is the meta-data is
only accessable to perl, but who cares. I bet this fix will
make #118 easier.
http://svn.open-mpi.org/trac/mtt/ticket/118
"Slim down result submissions to the bare essentials"
On Thu, Nov/30/2006 07:55:14AM, jsquyres_at_[hidden] wrote:
> Author: jsquyres
> Date: 2006-11-30 07:55:07 EST (Thu, 30 Nov 2006)
> New Revision: 461
>
> Removed:
> trunk/lib/XML/
> Modified:
> trunk/lib/MTT/Files.pm
> trunk/lib/MTT/MPI.pm
> trunk/lib/MTT/Test.pm
> trunk/lib/MTT/Test/RunEngine.pm
>
> Log:
> Fixes #144
>
> This commit is a bunch of things to improve the meta data handling and
> storage in the MTT client:
>
> * Remove the use of XML::Simple and use Data::Dumper instead. This
> allows us to remove the costly/slow (and error-prone when coding up
> new data layers) data transformation for each of the datasets that
> were saved out to disk. The output data is about the same size,
> but we avoid the [potentially] huge memory transformation before
> the write, and for large data sets (e.g., Test run data), can save
> quite a bit of time (e.g., ~3 seconds with Data::Dumper vs. ~10
> seconds with XML::Simple).
> * Split the Test run data up into multiple files instead of one huge
> file. This allows us to write only the new data after tests have
> run rather than having to write ''all'' the data ''every'' time.
> It makes the reading and writing of the test run data a little more
> complex, but the performance win is worth it.
> * Write the Test Run data [slightly] less frequently -- i.e., write
> out a [small] batch of new Test Run data periodically rather than
> writing after ''every'' test.
>
>
> Modified: trunk/lib/MTT/Files.pm
> ==============================================================================
> --- trunk/lib/MTT/Files.pm (original)
> +++ trunk/lib/MTT/Files.pm 2006-11-30 07:55:07 EST (Thu, 30 Nov 2006)
> @@ -40,8 +40,15 @@
>
> #--------------------------------------------------------------------------
>
> -sub mkdir {
> +sub safe_mkdir {
> my ($dir) = @_;
> + MTT::Files::mkdir($dir, 1);
> +}
> +
> +#--------------------------------------------------------------------------
> +
> +sub mkdir {
> + my ($dir, $safe) = @_;
>
> my $c = cwd();
> Debug("Making dir: $dir (cwd: $c)\n");
> @@ -58,6 +65,9 @@
> foreach my $p (@parts) {
> next if (! $p);
>
> + $p = make_safe_filename($p)
> + if ($safe);
> +
> $str .= "$p";
> if (! -d $str) {
> Debug("$str does not exist -- creating\n");
> @@ -402,6 +412,8 @@
> return 1;
> }
>
> +#--------------------------------------------------------------------------
> +
> # Copy infile or stdin to a unique file in /tmp
> sub copyfile {
>
> @@ -432,4 +444,64 @@
> return $outfile;
> }
>
> +#--------------------------------------------------------------------------
> +
> +sub load_dumpfile {
> + my ($filename, $data) = @_;
> +
> + # Check that the file is there
> + return
> + if (! -r $filename);
> +
> + # Get the file size
> + my ($dev, $ino, $mode, $nlink, $uid, $gid, $rdev, $size,
> + $atime, $mtime, $ctime, $blksize, $blocks) = stat($filename);
> +
> + # Read it in
> + open IN, $filename;
> + my $tmp;
> + read(IN, $tmp, $size);
> + close IN;
> +
> + # It's in Dumper format. How many $VARx's are there?
> + return
> + if (! $tmp =~ m/\$VAR[0-9]+/g);
> + my $count = 0;
> + ++$count
> + while ($tmp =~ m/\$VAR[0-9]+/g);
> +
> + # We know how many $VARx's there are. Build up a string to eval.
> + my $str;
> + my $var_num = 1;
> + while ($var_num <= $count) {
> + $str .= "my \$VAR$var_num;\n";
> + ++$var_num;
> + }
> + $str .= "eval \$tmp;\n";
> + my $var_num = 1;
> + while ($var_num <= $count) {
> + $str .= "\$\$data->{VAR$var_num} = \$VAR$var_num;\n";
> + ++$var_num;
> + }
> + eval $str;
> +}
> +
> +#--------------------------------------------------------------------------
> +
> +sub save_dumpfile {
> + my ($filename) = @_;
> + shift;
> +
> + # Serialize
> + my $d = new Data::Dumper([@_]);
> + $d->Purity(1)->Indent(1);
> +
> + open FILE, ">$filename.new";
> + print FILE $d->Dump;
> + close FILE;
> +
> + # Atomically move it onto the old file
> + rename("$filename.new", $filename);
> +}
> +
> 1;
>
> Modified: trunk/lib/MTT/MPI.pm
> ==============================================================================
> --- trunk/lib/MTT/MPI.pm (original)
> +++ trunk/lib/MTT/MPI.pm 2006-11-30 07:55:07 EST (Thu, 30 Nov 2006)
> @@ -15,10 +15,9 @@
> package MTT::MPI;
>
> use strict;
> -use XML::Simple qw(:strict);
> use MTT::MPI::Get;
> use MTT::MPI::Install;
> -use Data::Dumper;
> +use MTT::Files;
>
> #--------------------------------------------------------------------------
>
> @@ -31,16 +30,10 @@
> #--------------------------------------------------------------------------
>
> # Filename where list of MPI sources is kept
> -my $sources_data_filename = "mpi_sources.xml";
> -
> -# XML options for the MPI sources
> -my $sources_xs;
> +my $sources_data_filename = "mpi_sources.dump";
>
> # Filename where list of MPI installs is kept
> -my $installs_data_filename = "mpi_installs.xml";
> -
> -# XML options for the MPI installs
> -my $installs_xs;
> +my $installs_data_filename = "mpi_installs.dump";
>
> #--------------------------------------------------------------------------
>
> @@ -60,65 +53,17 @@
>
> #--------------------------------------------------------------------------
>
> -sub _scrub {
> - my $h = shift;
> -
> - foreach my $k (keys(%$h)) {
> - if (ref($h->{$k}) eq "") {
> - # Remove bad characters
> - $h->{$k} =~ s/\000/<NULL>/g;
> - } else {
> - _scrub($h->{$k});
> - }
> - }
> -}
> -
> -#--------------------------------------------------------------------------
> -
> -sub _setup_sources_xml {
> - $sources_xs = new XML::Simple(KeyAttr => { mpi_get => "simple_section_name",
> - mpi_version => "version",
> - },
> - ForceArray => [ "mpi_get",
> - "mpi_version",
> - ],
> - AttrIndent => 1,
> - RootName => "mpi_sources",
> - );
> -}
> -
> -#--------------------------------------------------------------------------
> -
> +use Data::Dumper;
> sub LoadSources {
> my ($dir) = @_;
>
> # Explicitly delete anything that was there
> $MTT::MPI::sources = undef;
>
> - _setup_sources_xml()
> - if (!$sources_xs);
> -
> # If the file exists, read it in
> - if (-f "$dir/$sources_data_filename") {
> - my $in = $sources_xs->XMLin("$dir/$sources_data_filename");
> -
> - # Now transform this to the form suitable for
> - # $MTT::MPI::sources (see comment in SaveSources)
> -
> - # For each MPI get section
> - foreach my $mpi_get_key (keys(%{$in->{mpi_get}})) {
> - my $mpi_get = $in->{mpi_get}->{$mpi_get_key};
> -
> - # For each version
> - foreach my $mpi_version_key (keys(%{$mpi_get->{mpi_version}})) {
> - my $mpi_version = $mpi_get->{mpi_version}->{$mpi_version_key};
> -
> - $MTT::MPI::sources->{$mpi_get_key}->{$mpi_version_key} = $mpi_version;
> - $MTT::MPI::sources->{$mpi_get_key}->{$mpi_version_key}->{simple_section_name} = $mpi_get_key;
> - $MTT::MPI::sources->{$mpi_get_key}->{$mpi_version_key}->{version} = $mpi_version_key;
> - }
> - }
> - }
> + my $data;
> + MTT::Files::load_dumpfile("$dir/$sources_data_filename", \$data);
> + $MTT::MPI::sources = $data->{VAR1};
> }
>
> #--------------------------------------------------------------------------
> @@ -126,54 +71,8 @@
> sub SaveSources {
> my ($dir) = @_;
>
> - _setup_sources_xml()
> - if (!$sources_xs);
> -
> - # Transform $MTT::MPI::sources to something XML::Simple can write
> - # into valid XML (since our values can [and will] contain :'s,
> - # which are the namespace identifiers in XML)
> - my $transformed;
> -
> - # For each MPI get section
> - foreach my $mpi_get_key (keys(%$MTT::MPI::sources)) {
> - my $mpi_get = $MTT::MPI::sources->{$mpi_get_key};
> -
> - # For each version
> - foreach my $mpi_version_key (keys(%$mpi_get)) {
> - my $mpi_version = $mpi_get->{$mpi_version_key};
> -
> - # Deep copy and scrub
> - my $h;
> - %$h = %$mpi_version;
> - _scrub($h);
> -
> - $transformed->{mpi_get}->{$mpi_get_key}->{mpi_version}->{$mpi_version_key} = $h;
> - }
> - }
> -
> - # Write out the file
> - my $xml = $sources_xs->XMLout($transformed);
> - my $file = "$dir/$sources_data_filename";
> - open(FILE, ">$file.new");
> - print FILE $xml;
> - close(FILE);
> - system("mv $file.new $file");
> -}
> -
> -#--------------------------------------------------------------------------
> -
> -sub _setup_installs_xml {
> - $installs_xs = new XML::Simple(KeyAttr => { mpi_get => "simple_section_name",
> - mpi_version => "version",
> - mpi_install => "simple_section_name",
> - },
> - ForceArray => [ "mpi_get",
> - "mpi_version",
> - "mpi_install",
> - ],
> - AttrIndent => 1,
> - RootName => "mpi_installs",
> - );
> + MTT::Files::save_dumpfile("$dir/$sources_data_filename",
> + $MTT::MPI::sources);
> }
>
> #--------------------------------------------------------------------------
> @@ -184,33 +83,10 @@
> # Explicitly delete anything that was there
> $MTT::MPI::installs = undef;
>
> - _setup_installs_xml()
> - if (!$installs_xs);
> -
> # If the file exists, read it in
> - if (-f "$dir/$installs_data_filename") {
> - my $in = $installs_xs->XMLin("$dir/$installs_data_filename");
> -
> - # Now transform this to the form suitable for
> - # $MTT::MPI::installs (see comment in SaveSources). Wow.
> -
> - # For each MPI get section
> - foreach my $mpi_get_key (keys(%{$in->{mpi_get}})) {
> - my $mpi_get = $in->{mpi_get}->{$mpi_get_key};
> -
> - # For each version of each MPI get
> - foreach my $mpi_version_key (keys(%{$mpi_get->{mpi_version}})) {
> - my $mpi_version = $mpi_get->{mpi_version}->{$mpi_version_key};
> -
> - # For each MPI install section
> - foreach my $mpi_install_key (keys(%{$mpi_version->{mpi_install}})) {
> - $MTT::MPI::installs->{$mpi_get_key}->{$mpi_version_key}->{$mpi_install_key} =
> - $in->{mpi_get}->{$mpi_get_key}->{mpi_version}->{$mpi_version_key}->{mpi_install}->{$mpi_install_key};
> - $MTT::MPI::installs->{$mpi_get_key}->{$mpi_version_key}->{$mpi_install_key}->{simple_section_name} = $mpi_install_key;
> - }
> - }
> - }
> - }
> + my $data;
> + MTT::Files::load_dumpfile("$dir/$installs_data_filename", \$data);
> + $MTT::MPI::installs = $data->{VAR1};
> }
>
> #--------------------------------------------------------------------------
> @@ -218,42 +94,8 @@
> sub SaveInstalls {
> my ($dir) = @_;
>
> - _setup_installs_xml()
> - if (!$installs_xs);
> -
> - # Transform $MTT::MPI::installs to something XML::Simple can write
> - # into valid XML (see comment in SaveSources). Wow.
> - my $transformed;
> -
> - # For each MPI get section
> - foreach my $mpi_get_key (keys(%{$MTT::MPI::installs})) {
> - my $mpi_get = $MTT::MPI::installs->{$mpi_get_key};
> -
> - # For each version of that get
> - foreach my $mpi_version_key (keys(%{$mpi_get})) {
> - my $mpi_version = $mpi_get->{$mpi_version_key};
> -
> - # For each MPI install action
> - foreach my $mpi_install_key (keys(%{$mpi_version})) {
> - my $mpi_install = $mpi_version->{$mpi_install_key};
> -
> - # Deep copy and scrub
> - my $h;
> - %$h = %$mpi_install;
> - _scrub($h);
> -
> - $transformed->{mpi_get}->{$mpi_get_key}->{mpi_version}->{$mpi_version_key}->{mpi_install}->{$mpi_install_key} = $h;
> - }
> - }
> - }
> -
> - # Write out the file
> - my $xml = $installs_xs->XMLout($transformed);
> - my $file = "$dir/$installs_data_filename";
> - open(FILE, ">$file.new");
> - print FILE $xml;
> - close(FILE);
> - system("mv $file.new $file");
> + MTT::Files::save_dumpfile("$dir/$installs_data_filename",
> + $MTT::MPI::installs);
> }
>
> 1;
>
> Modified: trunk/lib/MTT/Test.pm
> ==============================================================================
> --- trunk/lib/MTT/Test.pm (original)
> +++ trunk/lib/MTT/Test.pm 2006-11-30 07:55:07 EST (Thu, 30 Nov 2006)
> @@ -13,11 +13,11 @@
> package MTT::Test;
>
> use strict;
> +use File::Find;
> use MTT::Test::Get;
> use MTT::Test::Build;
> use MTT::Test::Run;
> -use XML::Simple;
> -use Data::Dumper;
> +use MTT::Files;
>
> #--------------------------------------------------------------------------
>
> @@ -29,26 +29,24 @@
>
> # Exported run tests handle
> our $runs;
> +our $runs_to_be_saved;
>
> #--------------------------------------------------------------------------
>
> # Filename where list of test sources information is kept
> -my $sources_data_filename = "test_sources.xml";
> -
> -# XML options for the test builds
> -my $sources_xs;
> +my $sources_data_filename = "test_sources.dump";
>
> # Filename where list of test build information is kept
> -my $builds_data_filename = "test_builds.xml";
> +my $builds_data_filename = "test_builds.dump";
>
> -# XML options for the test builds
> -my $builds_xs;
> +# Subdir where test runs are kept
> +my $runs_subdir = "test_runs";
>
> # Filename where list of test run information is kept
> -my $runs_data_filename = "test_runs.xml";
> +my $runs_data_filename = "test_runs.dump";
>
> -# XML options for the test runs
> -my $runs_xs;
> +# Helper variable for when we're loading test run data
> +my $load_run_file_start_dir;
>
> #--------------------------------------------------------------------------
>
> @@ -76,57 +74,16 @@
>
> #--------------------------------------------------------------------------
>
> -sub _scrub {
> - my $h = shift;
> -
> - foreach my $k (keys(%$h)) {
> - if (ref($h->{$k}) eq "") {
> - # Remove bad characters
> - $h->{$k} =~ s/\000/<NULL>/g;
> - } else {
> - _scrub($h->{$k});
> - }
> - }
> -}
> -
> -#--------------------------------------------------------------------------
> -
> -sub _setup_sources_xml {
> - $sources_xs = new XML::Simple(KeyAttr => { test_get => "simple_section_name",
> - },
> - ForceArray => [ "test_get",
> - ],
> - AttrIndent => 1,
> - RootName => "test_sources",
> - );
> -}
> -
> -#--------------------------------------------------------------------------
> -
> sub LoadSources {
> my ($dir) = @_;
>
> # Explicitly delete anything that was there
> $MTT::Test::sources = undef;
>
> - _setup_sources_xml()
> - if (!$sources_xs);
> -
> # If the file exists, read it in
> - if (-f "$dir/$sources_data_filename") {
> - my $in = $sources_xs->XMLin("$dir/$sources_data_filename");
> -
> - # Now transform this to the form suitable for
> - # $MTT::Test::sources (see comment in SaveSources)
> -
> - # For each Test get section
> - foreach my $test_get_key (keys(%{$in->{test_get}})) {
> - my $test_get = $in->{test_get}->{$test_get_key};
> -
> - $MTT::Test::sources->{$test_get_key} = $test_get;
> - $MTT::Test::sources->{$test_get_key}->{simple_section_name} = $test_get_key;
> - }
> - }
> + my $data;
> + MTT::Files::load_dumpfile("$dir/$sources_data_filename", \$data);
> + $MTT::Test::sources = $data->{VAR1};
> }
>
> #--------------------------------------------------------------------------
> @@ -134,51 +91,8 @@
> sub SaveSources {
> my ($dir) = @_;
>
> - _setup_sources_xml()
> - if (!$sources_xs);
> -
> - # Transform $MTT::Test::sources to something XML::Simple can write
> - # into valid XML (since our values can [and will] contain :'s,
> - # which are the namespace identifiers in XML)
> - my $transformed;
> -
> - # For each Test get section
> - foreach my $test_get_key (keys(%$MTT::Test::sources)) {
> - my $test_get = $MTT::Test::sources->{$test_get_key};
> -
> - # Deep copy and scrub
> - my $h;
> - %$h = %$test_get;
> - _scrub($h);
> -
> - $transformed->{test_get}->{$test_get_key} = $h;
> - }
> -
> - # Write out the file
> - my $xml = $sources_xs->XMLout($transformed);
> - my $file = "$dir/$sources_data_filename";
> - open(FILE, ">$file.new");
> - print FILE $xml;
> - close(FILE);
> - system("mv $file.new $file");
> -}
> -
> -#--------------------------------------------------------------------------
> -
> -sub _setup_builds_xml {
> - $builds_xs = new XML::Simple(KeyAttr => { mpi_get => "simple_section_name",
> - mpi_version => "version",
> - mpi_install => "simple_section_name",
> - test_build => "simple_section_name",
> - },
> - ForceArray => [ "mpi_get",
> - "mpi_version",
> - "mpi_install",
> - "test_build",
> - ],
> - AttrIndent => 1,
> - RootName => "test_builds",
> - );
> + MTT::Files::save_dumpfile("$dir/$sources_data_filename",
> + $MTT::Test::sources);
> }
>
> #--------------------------------------------------------------------------
> @@ -189,39 +103,10 @@
> # Explicitly delete anything that was there
> $MTT::Test::builds = undef;
>
> - _setup_builds_xml()
> - if (!$builds_xs);
> -
> # If the file exists, read it in
> - if (-f "$dir/$builds_data_filename") {
> - my $in = $builds_xs->XMLin("$dir/$builds_data_filename");
> -
> - # Now transform this to the form suitable for
> - # $MTT::Test::builds (see comment in SaveSources). Wow.
> - # For each MPI get section
> - foreach my $mpi_get_key (keys(%{$in->{mpi_get}})) {
> - my $mpi_get = $in->{mpi_get}->{$mpi_get_key};
> -
> - # For each version of that MPI
> - foreach my $mpi_version_key (keys(%{$mpi_get->{mpi_version}})) {
> - my $mpi_version = $mpi_get->{mpi_version}->{$mpi_version_key};
> -
> - # For each MPI install section
> - foreach my $mpi_install_key (keys(%{$mpi_version->{mpi_install}})) {
> - my $mpi_install = $mpi_version->{mpi_install}->{$mpi_install_key};
> -
> - # For each test build section
> - foreach my $test_build_key (keys(%{$mpi_install->{test_build}})) {
> - my $test_build = $mpi_install->{test_build}->{$test_build_key};
> -
> - $MTT::Test::builds->{$mpi_get_key}->{$mpi_version_key}->{$mpi_install_key}->{$test_build_key} =
> - $in->{mpi_get}->{$mpi_get_key}->{mpi_version}->{$mpi_version_key}->{mpi_install}->{$mpi_install_key}->{test_build}->{$test_build_key};
> - $MTT::Test::builds->{$mpi_get_key}->{$mpi_version_key}->{$mpi_install_key}->{$test_build_key}->{simple_section_name} = $test_build_key;
> - }
> - }
> - }
> - }
> - }
> + my $data;
> + MTT::Files::load_dumpfile("$dir/$builds_data_filename", \$data);
> + $MTT::Test::builds = $data->{VAR1};
> }
>
> #--------------------------------------------------------------------------
> @@ -229,135 +114,48 @@
> sub SaveBuilds {
> my ($dir) = @_;
>
> - _setup_builds_xml()
> - if (!$builds_xs);
> -
> - # Transform $MTT::Test::builds to something XML::Simple can write
> - # into valid XML (see comment in SaveSources). Wow.
> - my $transformed;
> -
> - # For each MPI get section
> - foreach my $mpi_get_key (keys(%{$MTT::Test::builds})) {
> - my $mpi_get = $MTT::Test::builds->{$mpi_get_key};
> -
> - # For each version of that MPI
> - foreach my $mpi_version_key (keys(%{$mpi_get})) {
> - my $mpi_version = $mpi_get->{$mpi_version_key};
> -
> - # For each MPI install section
> - foreach my $mpi_install_key (keys(%{$mpi_version})) {
> - my $mpi_install = $mpi_version->{$mpi_install_key};
> -
> - # For each test build section
> - foreach my $test_build_key (keys(%{$mpi_install})) {
> - my $test_build = $mpi_install->{$test_build_key};
> -
> - # Deep copy and scrub
> - my $h;
> - %$h = %$test_build;
> - _scrub($h);
> -
> - $transformed->{mpi_get}->{$mpi_get_key}->{mpi_version}->{$mpi_version_key}->{mpi_install}->{$mpi_install_key}->{test_build}->{$test_build_key} = $h;
> - }
> - }
> - }
> - }
> -
> - # Write out the file
> - my $xml = $builds_xs->XMLout($transformed);
> - my $file = "$dir/$builds_data_filename";
> - open(FILE, ">$file.new");
> - print FILE $xml;
> - close(FILE);
> - system("mv $file.new $file");
> -}
> -
> -#--------------------------------------------------------------------------
> -
> -sub _setup_runs_xml {
> - $runs_xs = new XML::Simple(KeyAttr => { mpi_get => "simple_section_name",
> - mpi_version => "version",
> - mpi_install => "simple_section_name",
> - test_build => "simple_section_name",
> - test_run => "simple_section_name",
> - test_name => "name",
> - test_np => "nprocs",
> - test_cmd => "argv",
> - },
> - ForceArray => [ "mpi_get",
> - "mpi_version",
> - "mpi_install",
> - "test_build",
> - "test_run",
> - "test_name",
> - "test_np",
> - "test_cmd",
> - ],
> - AttrIndent => 1,
> - RootName => "test_runs",
> - );
> + MTT::Files::save_dumpfile("$dir/$builds_data_filename",
> + $MTT::Test::builds);
> }
>
> #--------------------------------------------------------------------------
>
> +use Data::Dumper;
> sub LoadRuns {
> my ($dir) = @_;
>
> # Explicitly delete anything that was there
> $MTT::Test::runs = undef;
>
> - _setup_runs_xml()
> - if (!$runs_xs);
> -
> - # If the file exists, read it in
> - if (-f "$dir/$runs_data_filename") {
> - my $in = $runs_xs->XMLin("$dir/$runs_data_filename");
> -
> - # Now transform this to the form suitable for
> - # $MTT::Test::runs (see comment in SaveSources). Wow.
> - # For each MPI get section
> - foreach my $mpi_get_key (keys(%{$in->{mpi_get}})) {
> - my $mpi_get = $in->{mpi_get}->{$mpi_get_key};
> -
> - # For each MPI source
> - foreach my $mpi_version_key (keys(%{$mpi_get->{mpi_version}})) {
> - my $mpi_version = $mpi_get->{mpi_version}->{$mpi_version_key};
> -
> - # For each MPI install section
> - foreach my $mpi_install_key (keys(%{$mpi_version->{mpi_install}})) {
> - my $mpi_install = $mpi_version->{mpi_install}->{$mpi_install_key};
> -
> - # For each test build section
> - foreach my $test_build_key (keys(%{$mpi_install->{test_build}})) {
> - my $test_build = $mpi_install->{test_build}->{$test_build_key};
> -
> - # For each test run section
> - foreach my $test_run_key (keys(%{$test_build->{test_run}})) {
> - my $test_run = $test_build->{test_run}->{$test_run_key};
> -
> - # For each test name
> - foreach my $test_name_key (keys(%{$test_run->{test_name}})) {
> - my $test_name = $test_run->{test_name}->{$test_name_key};
> -
> - # For each np
> - foreach my $test_np_key (keys(%{$test_name->{test_np}})) {
> - my $test_np = $test_name->{test_np}->{$test_np_key};
> -
> - # For each test command
> - foreach my $test_cmd_key (keys(%{$test_np->{test_cmd}})) {
> - my $test_cmd = $test_np->{test_cmd}->{$test_cmd_key};
> -
> - $MTT::Test::runs->{$mpi_get_key}->{$mpi_version_key}->{$mpi_install_key}->{$test_build_key}->{$test_run_key}->{$test_name_key}->{$test_np_key}->{$test_cmd_key} =
> - $in->{mpi_get}->{$mpi_get_key}->{mpi_version}->{$mpi_version_key}->{mpi_install}->{$mpi_install_key}->{test_build}->{$test_build_key}->{test_run}->{$test_run_key}->{test_name}->{$test_name_key}->{test_np}->{$test_np_key}->{test_cmd}->{$test_cmd_key};
> - }
> - }
> - }
> - }
> - }
> - }
> - }
> - }
> - }
> + # See SaveRuns, below, for an explanation. We traverse
> + # directories looking for dump files and read them into the
> + # appropriate section of the $MTT::Test::runs hash.
> +
> + $load_run_file_start_dir = "$dir/$runs_subdir";
> + find(\&load_run_file, $load_run_file_start_dir)
> + if (-d $load_run_file_start_dir);
> +}
> +
> +sub load_run_file {
> + # We only want files named $runs_data_filename
> + return 0
> + if (! -f $_ || $runs_data_filename ne $_);
> +
> + # Read in the file
> + my $data;
> + MTT::Files::load_dumpfile($File::Find::name, \$data);
> +
> + # Put the loaded data in the hash in the right place. Per
> + # SaveRuns(), below, we look at the key values in $data->{VAR1} to
> + # know where to put $data->{VAR2} in the $MTT::Test::runs hash.
> + my $str = "\$MTT::Test::runs";
> + my $k = 1;
> + do {
> + $str .= "->{\"" . $data->{VAR1}->{$k} . "\"}";
> + ++$k;
> + } while (exists($data->{VAR1}->{$k}));
> + $str .= " = \$data->{VAR2}";
> + eval $str;
> }
>
> #--------------------------------------------------------------------------
> @@ -365,15 +163,25 @@
> sub SaveRuns {
> my ($dir) = @_;
>
> - _setup_runs_xml()
> - if (!$runs_xs);
> + # Because test run data can get very, very large, we break it up
> + # and store it in lots of smaller files so that we can write out
> + # to disk in small portions.
> +
> + # Test runs are stored in the hash with this order of keys:
> +
> + # mpi_get_simple_section_name
> + # mpi_version
> + # mpi_install_simple_section_name
> + # test_build_simple_section_name
> + # test_run_simple_section_name
> + # test_name
> + # np
> + # command
>
> - # Transform $MTT::Test::runs to something XML::Simple can write
> - # into valid XML (see comment in SaveSources). Wow.
> - my $transformed;
> + # We save from test_name and down in a single file.
>
> # For each MPI get section
> - foreach my $mpi_get_key (keys(%{$MTT::Test::runs})) {
> + foreach my $mpi_get_key (keys(%{$MTT::Test::runs_to_be_saved})) {
> my $mpi_get = $MTT::Test::runs->{$mpi_get_key};
>
> # For each source of that MPI
> @@ -395,23 +203,22 @@
> # For each test name
> foreach my $test_name_key (keys(%{$test_run})) {
> my $test_name = $test_run->{$test_name_key};
> -
> - # For each np
> - foreach my $test_np_key (keys(%{$test_name})) {
> - my $test_np = $test_name->{$test_np_key};
> -
> - # For each cmd
> - foreach my $test_cmd_key (keys(%{$test_np})) {
> - my $test_cmd = $test_np->{$test_cmd_key};
> -
> - # Deep copy and scrub
> - my $h;
> - %$h = %$test_cmd;
> - _scrub($h);
> -
> - $transformed->{mpi_get}->{$mpi_get_key}->{mpi_version}->{$mpi_version_key}->{mpi_install}->{$mpi_install_key}->{test_build}->{$test_build_key}->{test_run}->{$test_run_key}->{test_name}->{$test_name_key}->{test_np}->{$test_np_key}->{test_cmd}->{$test_cmd_key} = $h;
> - }
> - }
> +
> + my $file = MTT::Files::safe_mkdir("$dir/$runs_subdir/$mpi_get_key/$mpi_version_key/$mpi_install_key/$test_build_key/$test_run_key/$test_name_key") . "/$runs_data_filename";
> +
> + # We need to save two items in test run
> + # data files -- the actual data and where
> + # it belongs in the $MTT::Test::runs hash.
> + my $hashname = {
> + 1 => $mpi_get_key,
> + 2 => $mpi_version_key,
> + 3 => $mpi_install_key,
> + 4 => $test_build_key,
> + 5 => $test_run_key,
> + 6 => $test_name_key,
> + };
> + MTT::Files::save_dumpfile($file, $hashname,
> + $test_name);
> }
> }
> }
> @@ -419,13 +226,8 @@
> }
> }
>
> - # Write out the file
> - my $xml = $runs_xs->XMLout($transformed);
> - my $file = "$dir/$runs_data_filename";
> - open(FILE, ">$file.new");
> - print FILE $xml;
> - close(FILE);
> - system("mv $file.new $file");
> + # Explicitly reset the test runs to be saved
> + $MTT::Test::runs_to_be_saved = undef;
> }
>
> 1;
>
> Modified: trunk/lib/MTT/Test/RunEngine.pm
> ==============================================================================
> --- trunk/lib/MTT/Test/RunEngine.pm (original)
> +++ trunk/lib/MTT/Test/RunEngine.pm 2006-11-30 07:55:07 EST (Thu, 30 Nov 2006)
> @@ -67,6 +67,9 @@
> }
> }
> ++$count;
> +
> + # Write out the "to be saved" test run results
> + MTT::Test::SaveRuns($top_dir);
>
> # Output a progress bar
> if ($verbose_out > 50) {
> @@ -189,8 +192,21 @@
> # Assume that the Analyze module will output one line
> ++$verbose_out;
>
> - $MTT::Test::runs->{$mpi_details->{mpi_get_simple_section_name}}->{$mpi_details->{version}}->{$mpi_details->{mpi_install_simple_section_name}}->{$run->{test_build_simple_section_name}}->{$run->{simple_section_name}}->{$name}->{$MTT::Test::Run::test_np}->{$cmd} = $report;
> - MTT::Test::SaveRuns($top_dir);
> + # For Test Runs data, we have two datasets: the "to be saved" set
> + # and the "all results" set. The "to be saved" set is a
> + # relatively small set of data that is written out to disk
> + # periodically (i.e., augmenting what has already been written
> + # out). The "all results" set is everything that has occurred so
> + # far. We do this because the "all results" set can get *very*
> + # large, so we don't want to write out the whole thing every time
> + # we save the results to disk.
> +
> + # So save this new result in both the "to be saved" and "all
> + # results" sets. We'll write out the "to be saved" results
> + # shortly.
> +
> + $MTT::Test::runs_to_be_saved->{$mpi_details->{mpi_get_simple_section_name}}->{$mpi_details->{version}}->{$mpi_details->{mpi_install_simple_section_name}}->{$run->{test_build_simple_section_name}}->{$run->{simple_section_name}}->{$name}->{$MTT::Test::Run::test_np}->{$cmd} =
> + $MTT::Test::runs->{$mpi_details->{mpi_get_simple_section_name}}->{$mpi_details->{version}}->{$mpi_details->{mpi_install_simple_section_name}}->{$run->{test_build_simple_section_name}}->{$run->{simple_section_name}}->{$name}->{$MTT::Test::Run::test_np}->{$cmd} = $report;
> MTT::Reporter::QueueAdd("Test Run", $run->{simple_section_name}, $report);
>
> # If there is an after_each step, run it
> _______________________________________________
> mtt-svn mailing list
> mtt-svn_at_[hidden]
> http://www.open-mpi.org/mailman/listinfo.cgi/mtt-svn
-- -Ethan