On Sun, Aug 19, 2012 at 12:59 AM, Christian Perrier
<
christian01.perrier@gmail.com> wrote:
> Hi,
>
> I have a problem with MPI_Senrecv communication where I send columns on
> edges between processes.
> For debugging, I show you below a basic example where I initialize a 10x10
> matrix ("x0" array) with x_domain=4
> and y_domain=4. For the test, I simply initialize the 2D array values with
> x0[i][j] = i+j . After, in updateBound.c", I'm
> using the MPI_Sendrecv functions for the North-South and Est-West process.
>
> Here's the main program "example.c" :
>
> -------------------------------------------------------------------------------------------
>
> #include <stdio.h>
> #include <stdlib.h>
> #include <math.h>
> #include "mpi.h"
>
> int main(int argc, char *argv[])
> {
> /* size of the discretization */
>
> double** x;
> double** x0;
>
> int i,j,k,l;
> int nproc;
> int ndims;
> int S=0, E=1, N=2, W=3;
> int NeighBor[4];
> int xcell, ycell, size_tot_x, size_tot_y;
> int *xs,*ys,*xe,*ye;
> int size_x = 4;
> int size_y = 4;
> int me;
> int x_domains=2;
> int y_domains=2;
>
> MPI_Comm comm, comm2d;
> int dims[2];
> int periods[2];
> int reorganisation = 0;
> int row;
> MPI_Datatype column_type;
>
>
>
> size_tot_x=size_x+2*x_domains+2;
> size_tot_y=size_y+2*y_domains+2;
>
> xcell=(size_x/x_domains);
> ycell=(size_y/y_domains);
>
> MPI_Init(&argc, &argv);
> comm = MPI_COMM_WORLD;
> MPI_Comm_size(comm,&nproc);
> MPI_Comm_rank(comm,&me);
>
> x = malloc(size_tot_y*sizeof(double*));
> x0 = malloc(size_tot_y*sizeof(double*));
>
>
> for(j=0;j<=size_tot_y-1;j++) {
> x[j] = malloc(size_tot_x*sizeof(double));
> x0[j] = malloc(size_tot_x*sizeof(double));
> }
>
> xs = malloc(nproc*sizeof(int));
> xe = malloc(nproc*sizeof(int));
> ys = malloc(nproc*sizeof(int));
> ye = malloc(nproc*sizeof(int));
>
> /* Create 2D cartesian grid */
> periods[0] = 0;
> periods[1] = 0;
>
> ndims = 2;
> dims[0]=x_domains;
> dims[1]=y_domains;
>
> MPI_Cart_create(comm, ndims, dims, periods, reorganisation, &comm2d);
>
> /* Identify neighbors */
>
> NeighBor[0] = MPI_PROC_NULL;
> NeighBor[1] = MPI_PROC_NULL;
> NeighBor[2] = MPI_PROC_NULL;
> NeighBor[3] = MPI_PROC_NULL;
>
> /* Left/West and right/Est neigbors */
>
> MPI_Cart_shift(comm2d,0,1,&NeighBor[W],&NeighBor[E]);
>
> /* Bottom/South and Upper/North neigbors */
>
> MPI_Cart_shift(comm2d,1,1,&NeighBor[S],&NeighBor[N]);
>
> /* coordinates of current cell with me rank */
>
> xcell=(size_x/x_domains);
> ycell=(size_y/y_domains);
>
> ys[me]=(y_domains-me%(y_domains)-1)*(ycell+2)+2;
> ye[me]=ys[me]+ycell-1;
>
> for(i=0;i<=y_domains-1;i++)
> {xs[i]=2;}
>
> for(i=0;i<=y_domains-1;i++)
> {xe[i]=xs[i]+xcell-1;}
>
> for(i=1;i<=(x_domains-1);i++)
> { for(j=0;j<=(y_domains-1);j++)
> {
> xs[i*y_domains+j]=xs[(i-1)*y_domains+j]+xcell+2;
> xe[i*y_domains+j]=xs[i*y_domains+j]+xcell-1;
> }
> }
>
> for(i=0;i<=size_tot_y-1;i++)
> { for(j=0;j<=size_tot_x-1;j++)
> { x0[i][j]= i+j;
> // printf("%f\n",x0[i][j]);
> }
> }
>
> /* Create column data type to communicate with South and North
> neighbors */
>
> MPI_Type_vector( ycell, 1, size_tot_x, MPI_DOUBLE, &column_type);
> MPI_Type_commit(&column_type);
>
> updateBound(x0, NeighBor, comm2d, column_type, me, xs, ys, xe, ye,
> xcell);
>
>
> for(i=0;i<=size_tot_y-1;i++)
> {
> free(x[i]);
> free(x0[i]);
> }
>
> free(x);
> free(x0);
>
> free(xs);
> free(xe);
> free(ys);
> free(ye);
>
> MPI_Finalize();
>
> return 0;
> }
> -------------------------------------------------------------------------------------------
>
> and the second file "updateBound.c" which sends the columns and rows
>
>
> -------------------------------------------------------------------------------------------
>
>
> #include "mpi.h"
> #include <stdio.h>
>
> /*******************************************************************/
> /* Update Bounds of subdomain with me process */
> /*******************************************************************/
>
> void updateBound(double** x,int NeighBor[], MPI_Comm comm2d, MPI_Datatype
> column_type , int me, int* xs, int* ys, int* xe, int* ye, int xcell)
> {
>
> int S=0, E=1, N=2, W=3;
> int flag;
> MPI_Status status;
>
> int i,j;
>
> if(me==0) {printf("verif_update_before\n");
> for(i=ys[me]-1;i<=ye[me]+1;i++)
> { for(j=xs[me]-1;j<=xe[me]+1;j++)
> { printf("%f ",x[i][j]);
> }
> printf("\n");
> }
> printf("\n");
> }
>
> /********* North/South communication **********************************/
> flag = 1;
> /*Send my boundary to North and receive from South*/
> MPI_Sendrecv(&x[ys[me]][xs[me]], xcell, MPI_DOUBLE, NeighBor[N], flag,
> &x[ye[me]+1][xs[me]], xcell, MPI_DOUBLE, NeighBor[S], flag, comm2d,
> &status);
>
> /*Send my boundary to South and receive from North*/
> MPI_Sendrecv(&x[ye[me]][xs[me]], xcell, MPI_DOUBLE, NeighBor[S], flag,
> &x[ys[me]-1][xs[me]], xcell, MPI_DOUBLE, NeighBor[N], flag, comm2d,
> &status);
>
> /********* Est/West communication ************************************/
> flag = 2;
> /*Send my boundary to Est and receive from West*/
> MPI_Sendrecv(&x[ys[me]][xe[me]], 1, column_type, NeighBor[E], flag,
> &x[ys[me]][xs[me]-1], 1, column_type, NeighBor[W], flag, comm2d, &status);
>
> /*Send my boundary to West and receive from Est*/
> MPI_Sendrecv(&x[ys[me]][xs[me]], 1, column_type, NeighBor[W], flag,
> &x[ys[me]][xe[me]+1], 1, column_type, NeighBor[E], flag, comm2d, &status);
>
> if(me==0) {printf("verif_update_after\n");
> for(i=ys[me]-1;i<=ye[me]+1;i++)
> { for(j=xs[me]-1;j<=xe[me]+1;j++)
> { printf("%f ",x[i][j]);
> }
> printf("\n");
> }
> printf("\n");
> }
> }
>
> ------------------------------------------------------------------------------
>
> Running with nproc=4, I print the values of the subarray with rank=0 (so at
> left bottom of the grid) and I get before and after the
> bounds udpate :
>
> verif_update_before
> 6.000000 7.000000 8.000000 9.000000
> 7.000000 8.000000 9.000000 10.000000
> 8.000000 9.000000 10.000000 11.000000
> 9.000000 10.000000 11.000000 12.000000
>
> verif_update_after
> 6.000000 5.000000 6.000000 9.000000
> 7.000000 8.000000 9.000000 12.000000
> 8.000000 9.000000 10.000000 11.000000
> 9.000000 10.000000 11.000000 12.000000
>
> As you can see, after the udpate, I don't have the correct value ( in
> underligned bold : 11.0 ) at the second element
> of the column coming from the Est. I expected 13.0 instead of 11.0.
>
> So there's a problem with the column datatype which only send the first
> element of this column.
>
> In "example.c", I define the column as following :
>
> MPI_Type_vector( ycell, 1, size_tot_x, MPI_DOUBLE, &column_type);
> MPI_Type_commit(&column_type);
>
> However, It seems ok and the computation of begin and end coordinates as a
> function of rank "me" is also good.
>
> I make you notice there's no problem between the exchange of rows between
> the North and the South, only
> between columns.
>
> If you could help me, I don't know what to do.
>
> Regards
>
>