I am combining mpi and cuda. Trying to find out sum of array elements using cuda and using mpi to distribute the array.

my cuda code

#include <stdio.h>
  
__global__ void add(int *devarray, int *devsum)
{
        int index = blockIdx.x * blockDim.x + threadIdx.x;
        *devsum = *devsum + devarray[index];
}
 
extern "C"

int * run_kernel(int array[],int nelements)
{
        int  *devarray, *sum, *devsum;
        sum =(int *) malloc(1 * sizeof(int));
        
        printf("\nrun_kernel called..............");       

        
        cudaMalloc((void**) &devarray, sizeof(int)*nelements);
        cudaMalloc((void**) &devsum, sizeof(int));       
        cudaMemcpy(devarray, array, sizeof(int)*nelements, cudaMemcpyHostToDevice);

        //cudaMemcpy(devsum, sum, sizeof(int), cudaMemcpyHostToDevice);
        add<<<2, 3>>>(devarray, devsum);
      //  printf("\ndevsum is %d", devsum);		

	cudaMemcpy(sum, devsum, sizeof(int), cudaMemcpyDeviceToHost);
 
        printf(" \nthe sum is %d\n", *sum);
        cudaFree(devarray);

        cudaFree(devsum);
	return sum;
        
}

 

#include "mpi.h"

#include <stdio.h>
#include <stdlib.h>

#include <string.h>
 
#define  ARRAYSIZE	2000 

#define  MASTER		0 
int  data[ARRAYSIZE]; 


int main(int argc, char* argv[])
{

 
int   numtasks, taskid, rc, dest, offset, i, j, tag1, tag2, source, chunksize, namelen;  

int mysum; 
long sum; 
int update(int myoffset, int chunk, int myid);

char myname[MPI_MAX_PROCESSOR_NAME]; 
MPI_Status status;
double start = 0.0, stop = 0.0, time = 0.0;

double totaltime;
FILE *fp;
char line[128];

char element;
int n;
int k=0;

 
/***** Initializations *****/
 
MPI_Init(&argc, &argv); 

MPI_Comm_size(MPI_COMM_WORLD, &numtasks); 
MPI_Comm_rank(MPI_COMM_WORLD,&taskid); 

MPI_Get_processor_name(myname, &namelen); 
printf ("MPI task %d has started on host %s...\n", taskid, myname); 

chunksize = (ARRAYSIZE / numtasks); 
tag2 = 1; 

tag1 = 2; 

/***** Master task only ******/

 
if (taskid == MASTER){
 
  fp=fopen("integers.txt", "r");

  if(fp != NULL){
   sum = 0;

   while(fgets(line, sizeof line, fp)!= NULL){

    fscanf(fp,"%d",&data[k]);
    sum = sum + data[k]; // calculate sum to verify later on

    k++;
   }
  }
 

printf("Initialized array sum %d\n", sum);

 
  /* Send each task its portion of the array - master keeps 1st part */ 

  offset = chunksize; 
  for (dest=1; dest<numtasks; dest++) { 

    MPI_Send(&offset, 1, MPI_INT, dest, tag1, MPI_COMM_WORLD); 

    MPI_Send(&data[offset], chunksize, MPI_INT, dest, tag2, MPI_COMM_WORLD); 

    printf("Sent %d elements to task %d offset= %d\n",chunksize,dest,offset); 

    offset = offset + chunksize; 
   }

 

 
  /* Master does its part of the work */

 
  offset = 0;
  mysum = run_kernel(&data[offset], chunksize);

  printf("Kernel returns sum %d", mysum); 

  //mysum = update(offset, chunksize, taskid);

 
  /* Wait to receive results from each task */

 
  for (i=1; i<numtasks; i++) { 

    source = i; 
    MPI_Recv(&offset, 1, MPI_INT, source, tag1, MPI_COMM_WORLD, &status); 

    MPI_Recv(&data[offset], chunksize, MPI_INT, source, tag2,MPI_COMM_WORLD, &status); 

   }
  
  /* Get final sum and print sample results */  

 
  MPI_Reduce(&mysum, &sum, 1, MPI_INT, MPI_SUM, MASTER, MPI_COMM_WORLD); 

  printf("\n*** Final sum= %d ***\n",sum); 

 }  /* end of master section */

 
/***** Non-master tasks only *****/


 
if (taskid > MASTER) {
 

  /* Receive my portion of array from the master task */

  start= MPI_Wtime(); 
  source = MASTER; 

  MPI_Recv(&offset, 1, MPI_INT, source, tag1, MPI_COMM_WORLD, &status); 

  MPI_Recv(&data[offset], chunksize, MPI_INT, source, tag2,MPI_COMM_WORLD, &status); 

  mysum = run_kernel(&data[offset], chunksize);
  printf("\nKernel returns sum %d ", mysum);

 
// mysum = update(offset, chunksize, taskid);

  stop = MPI_Wtime();
  time = stop -start;

  printf("time taken by process %d to recieve elements and caluclate own sum is = %lf seconds \n", taskid, time);  

 // totaltime = totaltime + time;

 

 
  /* Send my results back to the master task */ 

  dest = MASTER; 
  MPI_Send(&offset, 1, MPI_INT, dest, tag1, MPI_COMM_WORLD); 

  MPI_Send(&data[offset], chunksize, MPI_INT, MASTER, tag2, MPI_COMM_WORLD); 

  MPI_Reduce(&mysum, &sum, 1, MPI_INT, MPI_SUM, MASTER, MPI_COMM_WORLD);

 
  } /* end of non-master */

 
 MPI_Finalize(); 
}

here is the output of above code -

MPI task 2 has started on host 4
MPI task 3 has started on host 4
MPI task 0 has started on host 4
MPI task 1 has started on host 4

Initialized array sum 9061
Sent 500 elements to task 1 offset= 500
Sent 500 elements to task 2 offset= 1000
Sent 500 elements to task 3 offset= 1500

run_kernel called..............
the sum is 10

Kernel returns sum 159300360 time taken by process 2 to recieve elements and caluclate own sum is = 0.290016 seconds
run_kernel called..............
the sum is 268452367
run_kernel called..............
the sum is 10

Kernel returns sum 145185544 time taken by process 3 to recieve elements and caluclate own sum is = 0.293579 seconds
run_kernel called..............
the sum is 1048

Kernel returns sum 156969736 time taken by process 1 to recieve elements and caluclate own sum is = 0.297599 seconds
Kernel returns sum 152148496
*** Final sum= 613604136 ***

The final sum and initialized sum is not matching. I am guessing its a pointer issue. mysum should be pointer? but then MPI_REDUCE does not execute properly and segmentation fault occurs.

Any idea what is going wrong?
Thanks