Hello list,

I have an application which uses MPI_Allgather with derived types. It works correctly with mpich2 and mvapich2. However it crashes periodically with openmpi2. After investigation I found that the crash takes place when I use derived datatypes with MPI_AllGather and number of ranks greater than 8. I’ve written a simple application which demonstrates the crash. It simply calls for MPI_Allgather with derived datatype that consists of 1 shifted integer . The sample works correctly with number of ranks 2-8. But when number of ranks is greater than 8 it crashes with segmentation fault inside MPI_Type_free, MPI_Allgather or MPI_Type_create_struct functions. This sample also works correctly with mv2, mpich2 with any number of ranks. Is this a limitation of ompi allgather?

Crashed output:

Press any key...

[amd1:24260] *** Process received signal ***

[amd1:24260] Signal: Segmentation fault (11)

[amd1:24260] Signal code: Address not mapped (1)

[amd1:24260] Failing at address: 0x18

[amd1:24262] *** Process received signal ***

[amd1:24262] Signal: Segmentation fault (11)

[amd1:24262] Signal code: Address not mapped (1)

[amd1:24262] Failing at address: 0x18

[amd1:24258] *** Process received signal ***

[amd1:24258] Signal: Segmentation fault (11)

[amd1:24258] Signal code: Address not mapped (1)

[amd1:24258] Failing at address: 0x18

[amd1:24260] [ 0] /lib64/libpthread.so.0 [0x3d6b20eb10]

[amd1:24260] [ 1] /lib64/libc.so.6 [0x3d6a671d80]

[amd1:24260] [ 2] /lib64/libc.so.6(cfree+0x4b) [0x3d6a67276b]

[amd1:24260] [ 3] /hpc/home/USERS/senina/projects/openmpi-1.4.3/install/lib/libopen-pal.so.0(opal_free+0x4e) [0x2ae52f5836bd]

[amd1:24260] [ 4] /hpc/home/USERS/senina/projects/openmpi-1.4.3/install/lib/libmpi.so.1 [0x2ae52efd05aa]

[amd1:24260] [ 5] /hpc/home/USERS/senina/projects/openmpi-1.4.3/install/lib/libmpi.so.1 [0x2ae52efd1e20]

[amd1:24260] [ 6] /hpc/home/USERS/senina/projects/openmpi-1.4.3/install/lib/libmpi.so.1(ompi_ddt_destroy+0xe3) [0x2ae52efd1d7b]

[amd1:24260] [ 7] /hpc/home/USERS/senina/projects/openmpi-1.4.3/install/lib/libmpi.so.1(MPI_Type_free+0xf0) [0x2ae52f0202ec]

[amd1:24260] [ 8] ./gather_openmpi_153(main+0xef) [0x400dc8]

[amd1:24260] [ 9] /lib64/libc.so.6(__libc_start_main+0xf4) [0x3d6a61d994]

[amd1:24260] [10] ./gather_openmpi_153 [0x400ba9]

[amd1:24260] *** End of error message ***

[amd1:24262] [ 0] /lib64/libpthread.so.0 [0x3d6b20eb10]

[amd1:24262] [ 1] /lib64/libc.so.6 [0x3d6a671d80]

[amd1:24262] [ 2] /lib64/libc.so.6(cfree+0x4b) [0x3d6a67276b]

[amd1:24262] [ 3] /hpc/home/USERS/senina/projects/openmpi-1.4.3/install/lib/libopen-pal.so.0(opal_free+0x4e) [0x2aedeea596bd]

[amd1:24262] [ 4] /hpc/home/USERS/senina/projects/openmpi-1.4.3/install/lib/libmpi.so.1 [0x2aedee4a65aa]

[amd1:24262] [ 5] /hpc/home/USERS/senina/projects/openmpi-1.4.3/install/lib/libmpi.so.1 [0x2aedee4a7e20]

[amd1:24262] [ 6] /hpc/home/USERS/senina/projects/openmpi-1.4.3/install/lib/libmpi.so.1(ompi_ddt_destroy+0xe3) [0x2aedee4a7d7b]

[amd1:24262] [ 7] /hpc/home/USERS/senina/projects/openmpi-1.4.3/install/lib/libmpi.so.1(MPI_Type_free+0xf0) [0x2aedee4f62ec]

[amd1:24262] [ 8] ./gather_openmpi_153(main+0xef) [0x400dc8]

[amd1:24262] [ 9] /lib64/libc.so.6(__libc_start_main+0xf4) [0x3d6a61d994]

[amd1:24262] [10] ./gather_openmpi_153 [0x400ba9]

[amd1:24262] *** End of error message ***

[amd1:24258] [ 0] /lib64/libpthread.so.0 [0x3d6b20eb10]

[amd1:24258] [ 1] /lib64/libc.so.6 [0x3d6a671d80]

[amd1:24258] [ 2] /lib64/libc.so.6(cfree+0x4b) [0x3d6a67276b]

[amd1:24258] [ 3] /hpc/home/USERS/senina/projects/openmpi-1.4.3/install/lib/libopen-pal.so.0(opal_free+0x4e) [0x2ab92cc786bd]

[amd1:24258] [ 4] /hpc/home/USERS/senina/projects/openmpi-1.4.3/install/lib/libmpi.so.1 [0x2ab92c6c55aa]

[amd1:24258] [ 5] /hpc/home/USERS/senina/projects/openmpi-1.4.3/install/lib/libmpi.so.1 [0x2ab92c6c6e20]

[amd1:24258] [ 6] /hpc/home/USERS/senina/projects/openmpi-1.4.3/install/lib/libmpi.so.1(ompi_ddt_destroy+0xe3) [0x2ab92c6c6d7b]

[amd1:24258] [ 7] /hpc/home/USERS/senina/projects/openmpi-1.4.3/install/lib/libmpi.so.1(MPI_Type_free+0xf0) [0x2ab92c7152ec]

[amd1:24258] [ 8] ./gather_openmpi_153(main+0xef) [0x400dc8]

[amd1:24258] [ 9] /lib64/libc.so.6(__libc_start_main+0xf4) [0x3d6a61d994]

[amd1:24258] [10] ./gather_openmpi_153 [0x400ba9]

[amd1:24258] *** End of error message ***

[amd1:24256] *** Process received signal ***

[amd1:24256] Signal: Segmentation fault (11)

[amd1:24256] Signal code: Address not mapped (1)

[amd1:24256] Failing at address: 0x18

[amd1:24256] [ 0] /lib64/libpthread.so.0 [0x3d6b20eb10]

[amd1:24256] [ 1] /lib64/libc.so.6 [0x3d6a671d80]

[amd1:24256] [ 2] /lib64/libc.so.6(cfree+0x4b) [0x3d6a67276b]

[amd1:24256] [ 3] /hpc/home/USERS/senina/projects/openmpi-1.4.3/install/lib/libopen-pal.so.0(opal_free+0x4e) [0x2ba5a8c866bd]

[amd1:24256] [ 4] /hpc/home/USERS/senina/projects/openmpi-1.4.3/install/lib/libmpi.so.1 [0x2ba5a86d35aa]

[amd1:24256] [ 5] /hpc/home/USERS/senina/projects/openmpi-1.4.3/install/lib/libmpi.so.1 [0x2ba5a86d4e20]

[amd1:24256] [ 6] /hpc/home/USERS/senina/projects/openmpi-1.4.3/install/lib/libmpi.so.1(ompi_ddt_destroy+0xe3) [0x2ba5a86d4d7b]

[amd1:24256] [ 7] /hpc/home/USERS/senina/projects/openmpi-1.4.3/install/lib/libmpi.so.1(MPI_Type_free+0xf0) [0x2ba5a87232ec]

[amd1:24256] [ 8] ./gather_openmpi_153(main+0xef) [0x400dc8]

[amd1:24256] [ 9] /lib64/libc.so.6(__libc_start_main+0xf4) [0x3d6a61d994]

[amd1:24256] [10] ./gather_openmpi_153 [0x400ba9]

[amd1:24256] *** End of error message ***

--------------------------------------------------------------------------

mpirun noticed that process rank 7 with PID 24262 on node amd1 exited on signal 11 (Segmentation fault).

--------------------------------------------------------------------------

The sample source code:

#include "mpi.h"

#include <string.h>

#include "stdio.h"

#include <stdlib.h>

#define MAX_ERROR_LENGTH 128

#define num_types 3

MPI_Aint disp[num_types] = {0, 10, 20};

MPI_Datatype type[num_types] = {MPI_LB, MPI_INT, MPI_UB};

int blocklen[num_types] = {1, 1, 1};

void generate_derived_type(MPI_Datatype *dtype)

{

int rank;

int status;

MPI_Comm_rank(MPI_COMM_WORLD, &rank);

status = MPI_Type_create_struct( num_types, blocklen, disp, type, dtype );

if (status != MPI_SUCCESS)

{

printf("status of create_struct = %i, rank = %i\n",

status, rank);

}

status = MPI_Type_commit( dtype );

if (status != MPI_SUCCESS)

{

printf("status of type_commit = %i, rank = %i\n",

status, rank);

}

int main(int argc, char *argv[])

{

int i;

int rank, size;

MPI_Datatype stype;

int *send_buf = 0, *recv_buf = 0;

send_buf = (int*)calloc(sizeof(int), 10000000);

recv_buf = (int*)calloc(sizeof(int), 10000000);

MPI_Init(&argc, &argv);

MPI_Comm_rank(MPI_COMM_WORLD, &rank);

MPI_Comm_size(MPI_COMM_WORLD, &size);

printf("Press any key...\n");

getchar();

for (i = 0; i < 20000; i++)

{

int status = 0;

generate_derived_type(&stype);

status = MPI_Allgather(send_buf, 1, stype, recv_buf, 1, stype, MPI_COMM_WORLD);

if (status != MPI_SUCCESS)

{

printf("status of all_gather = %i, rank = %i\n",

status, rank);

}

status = MPI_Type_free( &stype);

if (status != MPI_SUCCESS)

{

printf("status of type_free = %i, rank = %i\n",

status, rank);

}

MPI_Finalize();

free(send_buf);

free(recv_buf);

return 0;

}

Thanks,

Andrew.