Open MPI logo

Open MPI User's Mailing List Archives

  |   Home   |   Support   |   FAQ   |   all Open MPI User's mailing list

Subject: Re: [OMPI users] Bug in 1.3.2?: sm btl and isend is serializes
From: Mark Bolstad (the.render.dude_at_[hidden])
Date: 2009-06-19 11:02:44


Not that long, 150 lines.

Here it is:

#include <mpi.h>
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

#define BUFLEN 25000
#define LOOPS 10
#define BUFFERS 4

#define GROUP_SIZE 4

int main(int argc, char *argv[])
{
   int myid, numprocs, next, namelen;
   int color, key, newid;
   char buffer[BUFLEN], processor_name[MPI_MAX_PROCESSOR_NAME];
   MPI_Comm world_comm, comp_comm, server_comm;

   MPI_Init(&argc,&argv);
   MPI_Comm_size(MPI_COMM_WORLD,&numprocs);
   MPI_Comm_rank(MPI_COMM_WORLD,&myid);
   MPI_Get_processor_name(processor_name,&namelen);

   MPI_Comm_dup( MPI_COMM_WORLD, &world_comm );

   if ( myid == 0 )
      color = MPI_UNDEFINED;
   else
   {
      color = (myid - 1) / GROUP_SIZE;
      key = (myid - 1) % GROUP_SIZE;
   }

   MPI_Comm_split( MPI_COMM_WORLD, color, key, &comp_comm );

   if ( myid == 0 || (myid - 1) % GROUP_SIZE == 0 )
      color = MPI_UNDEFINED;
   else
   {
      int r = myid - 2;

      color = 1;
      key = r - r / GROUP_SIZE;
   }

   MPI_Comm_split( MPI_COMM_WORLD, color, key, &server_comm );
   if ( myid == 0 )
   {
      fprintf(stderr,"Master [id = %d of %d] is running on %s\n", myid,
numprocs, processor_name);
   }
   else
   {
      int s_id;

      MPI_Comm_rank( comp_comm, &newid );
      if ( (myid - 1) % GROUP_SIZE == 0 )
     fprintf(stderr,"Compositor [id = %d, %d of %d] is running on %s\n",
myid, newid, numprocs, processor_name);
      else
      {
     MPI_Comm_rank( server_comm, &s_id );
     fprintf(stderr,"Server [id = %d, %d, %d of %d] is running on %s\n",
myid, newid, s_id, numprocs, processor_name);
      }
   }

   if ( myid == 0 )
      next = 10;

   MPI_Bcast( &next, 1, MPI_INT, 0, world_comm );
   fprintf(stderr,"[%d] next = %d\n", myid, next );

   if ( myid > 0 )
   {
      int i, j;
      int rank, size;
      MPI_Status status;

      MPI_Comm_size( comp_comm, &size );
      MPI_Comm_rank( comp_comm, &rank );

      if ( rank == 0 )
      {
     char buffer[BUFLEN];

     for (i = 0; i < LOOPS * ( size - 1 ); i++)
     {
        int which_source, which_tag;

        MPI_Probe( MPI_ANY_SOURCE, MPI_ANY_TAG, comp_comm, &status );
        which_source = status.MPI_SOURCE;
        which_tag = status.MPI_TAG;
        printf( "Receiving buffer from %d, buffer = ", which_source );
        MPI_Recv( buffer, BUFLEN, MPI_CHAR, which_source, which_tag,
comp_comm, &status );
        printf( "%s\n", buffer );
     }
      }
      else
      {
     MPI_Request* request[BUFFERS];
     int sent[ BUFFERS ];
     int index = 0;
     char* buffer[BUFFERS];

     for (i = 0; i < BUFFERS; i++)
     {
        MPI_Request* requester = (MPI_Request *) malloc( sizeof( MPI_Request
) );
        char* c = (char *) malloc(BUFLEN * sizeof( MPI_Request ) );
        /* Should really check for failure, but not for this test */
        request[ i ] = requester;
        sent[ i ] = 0;
        buffer[ i ] = c;
     }

     for (i = 0; i < LOOPS; i++)
     {
        printf( "Sending buffer %d from %d\n", i, rank );
        sprintf( buffer[ index ], "hello from %d for the %d time", rank, i
);
        if ( sent[ index ] )
        {
           sent[ index ] = 0;
           MPI_Wait( request[ index ], &status );
        }

        MPI_Isend( buffer[ index ], BUFLEN, MPI_CHAR, 0, 99, comp_comm,
               request[ index ] );

        sent[ index ] = 1;
        index = ( index + 1 ) % BUFFERS;

        /* Randomly sleep to fake a computation loop*/
        usleep( (unsigned long)(500000 * drand48()) );
     }

     /* Clean up */
     for (i = 0; i < BUFFERS; i++)
     {
        if ( sent[ i ] )
        {
           sent[ i ] = 0;
           MPI_Wait( request[ i ], &status );
        }
        free( request[ i ] );
        free( buffer[ i ] );
     }
      }
   }

   MPI_Barrier( world_comm );
   MPI_Finalize();
   return (0);
}

On Fri, Jun 19, 2009 at 10:50 AM, Eugene Loh <Eugene.Loh_at_[hidden]> wrote:

> Mark Bolstad wrote:
>
> I'll post the test code if requested (this email is already long)
>>
>
> Yipes, how long is the test code? Short enough to send, yes? Please send.
> _______________________________________________
> users mailing list
> users_at_[hidden]
> http://www.open-mpi.org/mailman/listinfo.cgi/users
>