Open MPI logo

Open MPI User's Mailing List Archives

  |   Home   |   Support   |   FAQ   |   all Open MPI User's mailing list

Subject: [OMPI users] RE : MPI_Comm_connect() fails
From: Audet, Martin (Martin.Audet_at_[hidden])
Date: 2008-03-12 16:58:12


Hi again,

Thanks Pak for the link and suggesting to start an "orted" deamon, by doing so my clients and servers jobs were able to establish an intercommunicator between them.

However I modified my programs to perform an MPI_Allgather() of a single "int" over the new intercommunicator to test communication a litle bit and I did encountered problems. I am now wondering if there is a problem in MPI_Allreduce() itself for intercommunicators. Note that the same program run without problems with mpich2 (ch3:sock).

For example if I start orted as follows:

   orted --persistent --seed --scope public --universe univ1

and then start the server with three process:

  mpiexec --universe univ1 -n 3 ./aserver

it prints:

  Server port = '0.2.0:2000'

Now if I start the client with two process as follow (using the server port):

   mpiexec --universe univ1 -n 2 ./aclient '0.2.0:2000'

The server prints:

  intercomm_flag = 1
  intercomm_remote_size = 2
  rem_rank_tbl[2] = { 0 1}

which is the correct output. The client then prints:

  intercomm_flag = 1
  intercomm_remote_size = 3
  rem_rank_tbl[3] = { 0 1 2}
  [linux15:30895] *** An error occurred in MPI_Allgather
  [linux15:30895] *** on communicator
  [linux15:30895] *** MPI_ERR_TRUNCATE: message truncated
  [linux15:30895] *** MPI_ERRORS_ARE_FATAL (goodbye)
  mpiexec noticed that job rank 0 with PID 30894 on node linux15 exited on signal 15 (Terminated).

As you can see the first messages are correct but the client job terminate with an error (and the server hang).

After re-reading the documentation about MPI_Allgather() over an intercommunicator, I don't see anything wrong in my simple code. Also if I run the client and server process with valgrind, I get a few messages like:

  ==29821== Syscall param writev(vector[...]) points to uninitialised byte(s)
  ==29821== at 0x36235C2130: writev (in /lib64/libc-2.3.5.so)
  ==29821== by 0x7885583: mca_btl_tcp_frag_send (in /home/publique/openmpi-1.2.5/lib/openmpi/mca_btl_tcp.so)
  ==29821== by 0x788501B: mca_btl_tcp_endpoint_send (in /home/publique/openmpi-1.2.5/lib/openmpi/mca_btl_tcp.so)
  ==29821== by 0x7467947: mca_pml_ob1_send_request_start_prepare (in /home/publique/openmpi-1.2.5/lib/openmpi/mca_pml_ob1.so)
  ==29821== by 0x7461494: mca_pml_ob1_isend (in /home/publique/openmpi-1.2.5/lib/openmpi/mca_pml_ob1.so)
  ==29821== by 0x798BF9D: mca_coll_basic_allgather_inter (in /home/publique/openmpi-1.2.5/lib/openmpi/mca_coll_basic.so)
  ==29821== by 0x4A5069C: PMPI_Allgather (in /home/publique/openmpi-1.2.5/lib/libmpi.so.0.0.0)
  ==29821== by 0x400EED: main (aserver.c:53)
  ==29821== Address 0x40d6cac is not stack'd, malloc'd or (recently) free'd

in both MPI_Allgather() and MPI_Comm_disconnect() calls for client and server with valgrind always reporting that the address in question are "not stack'd, malloc'd or (recently) free'd".

So is there a problem with MPI_Allgather() on intercommunicators or am I doing something wrong ?

Thanks,

Martin

/* aserver.c */
#include <stdio.h>
#include <mpi.h>

#include <assert.h>
#include <stdlib.h>

int main(int argc, char **argv)
{
   int comm_rank,comm_size;
   char port_name[MPI_MAX_PORT_NAME];
   MPI_Comm intercomm;
   int ok_flag;

   int intercomm_flag;
   int intercomm_remote_size;
   int *rem_rank_tbl;
   int ii;

   MPI_Init(&argc, &argv);

   MPI_Comm_rank(MPI_COMM_WORLD, &comm_rank);
   MPI_Comm_size(MPI_COMM_WORLD, &comm_size);

   ok_flag = (comm_rank != 0) || (argc == 1);
   MPI_Bcast(&ok_flag, 1, MPI_INT, 0, MPI_COMM_WORLD);

   if (!ok_flag) {
      if (comm_rank == 0) {
         fprintf(stderr,"Usage: %s\n",argv[0]);
      }
      MPI_Abort(MPI_COMM_WORLD, 1);
   }

   MPI_Open_port(MPI_INFO_NULL, port_name);

   if (comm_rank == 0) {
      printf("Server port = '%s'\n", port_name);
   }
   MPI_Comm_accept(port_name, MPI_INFO_NULL, 0, MPI_COMM_WORLD, &intercomm);

   MPI_Close_port(port_name);

   MPI_Comm_test_inter(intercomm, &intercomm_flag);
   if (comm_rank == 0) {
      printf("intercomm_flag = %d\n", intercomm_flag);
   }
   assert(intercomm_flag != 0);
   MPI_Comm_remote_size(intercomm, &intercomm_remote_size);
   if (comm_rank == 0) {
      printf("intercomm_remote_size = %d\n", intercomm_remote_size);
   }
   rem_rank_tbl = malloc(intercomm_remote_size*sizeof(*rem_rank_tbl));
   MPI_Allgather(&comm_rank, 1, MPI_INT,
                 rem_rank_tbl, 1, MPI_INT,
                 intercomm);
   if (comm_rank == 0) {
      printf("rem_rank_tbl[%d] = {", intercomm_remote_size);
      for (ii=0; ii < intercomm_remote_size; ii++) {
          printf(" %d", rem_rank_tbl[ii]);
      }
      printf("}\n");
   }
   free(rem_rank_tbl);

   MPI_Comm_disconnect(&intercomm);

   MPI_Finalize();

   return 0;
}

/* aclient.c */
#include <stdio.h>
#include <unistd.h>

#include <mpi.h>

#include <assert.h>
#include <stdlib.h>

int main(int argc, char **argv)
{
   int comm_rank,comm_size;
   int ok_flag;
   MPI_Comm intercomm;

   int intercomm_flag;
   int intercomm_remote_size;
   int *rem_rank_tbl;
   int ii;

   MPI_Init(&argc, &argv);

   MPI_Comm_rank(MPI_COMM_WORLD, &comm_rank);
   MPI_Comm_size(MPI_COMM_WORLD, &comm_size);

   ok_flag = (comm_rank != 0) || ((argc == 2) && argv[1] && (*argv[1] != '\0'));
   MPI_Bcast(&ok_flag, 1, MPI_INT, 0, MPI_COMM_WORLD);

   if (!ok_flag) {
      if (comm_rank == 0) {
         fprintf(stderr,"Usage: %s mpi_port\n", argv[0]);
      }
      MPI_Abort(MPI_COMM_WORLD, 1);
   }

   while (MPI_Comm_connect((comm_rank == 0) ? argv[1] : 0, MPI_INFO_NULL, 0, MPI_COMM_WORLD, &intercomm) != MPI_SUCCESS) {
      if (comm_rank == 0) {
         printf("MPI_Comm_connect() failled, sleeping and retrying...\n");
      }
      sleep(1);
   }

   MPI_Comm_test_inter(intercomm, &intercomm_flag);
   if (comm_rank == 0) {
      printf("intercomm_flag = %d\n", intercomm_flag);
   }
   assert(intercomm_flag != 0);
   MPI_Comm_remote_size(intercomm, &intercomm_remote_size);
   if (comm_rank == 0) {
      printf("intercomm_remote_size = %d\n", intercomm_remote_size);
   }
   rem_rank_tbl = malloc(intercomm_remote_size*sizeof(*rem_rank_tbl));
   MPI_Allgather(&comm_rank, 1, MPI_INT,
                 rem_rank_tbl, 1, MPI_INT,
                 intercomm);
   if (comm_rank == 0) {
      printf("rem_rank_tbl[%d] = {", intercomm_remote_size);
      for (ii=0; ii < intercomm_remote_size; ii++) {
          printf(" %d", rem_rank_tbl[ii]);
      }
      printf("}\n");
   }
   free(rem_rank_tbl);

   MPI_Comm_disconnect(&intercomm);

   MPI_Finalize();

   return 0;
}