On Tue, Mar 3, 2009 at 12:51 PM, Nathan Baca <nathan.baca@gmail.com> wrote:

Hello,

I am seeing inconsistent mpi-io behavior when writing to a Lustre file system using open mpi 1.3 with romio. What follows is a simple reproducer and output. Essentially one or more of the running processes does not read or write the correct amount of data to its part of a file residing on a Lustre (parallel) file system.

Any help figuring out what is happening is greatly appreciated. Thanks, Nate

program gcrm_test_io
  implicit none
  include "mpif.h"
 
  integer X_SIZE
 
      integer w_me, w_nprocs
      integer  my_info
 
      integer i
      integer (kind=4) :: ierr
      integer (kind=4) :: fileID
      
      integer (kind=MPI_OFFSET_KIND)        :: mylen
      integer (kind=MPI_OFFSET_KIND)        :: offset
      integer status(MPI_STATUS_SIZE)
      integer count
      integer ncells
      real (kind=4), allocatable, dimension (:)     :: array2
      logical sync
 
      call mpi_init(ierr)
      call MPI_COMM_SIZE(MPI_COMM_WORLD,w_nprocs,ierr)
      call MPI_COMM_RANK(MPI_COMM_WORLD,w_me,ierr)
 
      call mpi_info_create(my_info, ierr)
!     optional ways to set things in mpi-io
!     call mpi_info_set   (my_info, "romio_ds_read" , "enable"   , ierr)
!     call mpi_info_set   (my_info, "romio_ds_write", "enable"   , ierr)
!     call mpi_info_set   (my_info, "romio_cb_write", "enable"    , ierr)
 
      x_size = 410011  ! A 'big' number, with bigger numbers it is more likely to fail
      sync = .true.  ! Extra file synchronization
 
      ncells = (X_SIZE * w_nprocs)
 
!  Use node zero to fill it with nines
      if (w_me .eq. 0) then
          call MPI_FILE_OPEN  (MPI_COMM_SELF, "output.dat", MPI_MODE_CREATE+MPI_MODE_WRONLY, my_info, fileID, ierr)
          allocate (array2(ncells)) 
          array2(:) = 9.0
          mylen = ncells
          offset = 0 * 4
          call MPI_FILE_SET_VIEW(fileID,offset, MPI_REAL,MPI_REAL, "native",MPI_INFO_NULL,ierr)
          call MPI_File_write(fileID, array2, mylen , MPI_REAL, status,ierr)
          call MPI_Get_count(status,MPI_INTEGER, count, ierr)
          if (count .ne. mylen) print*, "Wrong initial write count:", count,mylen
          deallocate(array2)
          if (sync) call MPI_FILE_SYNC (fileID,ierr)
          call MPI_FILE_CLOSE (fileID,ierr)
      endif
 
!  All nodes now fill their area with ones
      call MPI_BARRIER(MPI_COMM_WORLD,ierr)
      allocate (array2( X_SIZE))
      array2(:) = 1.0
      offset = (w_me * X_SIZE) * 4 ! multiply by four, since it is real*4
      mylen = X_SIZE
      call MPI_FILE_OPEN  (MPI_COMM_WORLD,"output.dat",MPI_MODE_WRONLY, my_info, fileID, ierr)
      print*,"node",w_me,"starting",(offset/4) + 1,"ending",(offset/4)+mylen
      call MPI_FILE_SET_VIEW(fileID,offset, MPI_REAL,MPI_REAL, "native",MPI_INFO_NULL,ierr)
      call MPI_File_write(fileID, array2, mylen , MPI_REAL, status,ierr)
      call MPI_Get_count(status,MPI_INTEGER, count, ierr)
      if (count .ne. mylen) print*, "Wrong write count:", count,mylen,w_me
      deallocate(array2)
      if (sync) call MPI_FILE_SYNC (fileID,ierr)
      call MPI_FILE_CLOSE (fileID,ierr)
 
!  Read it back on node zero to see if it is ok data
      if (w_me .eq. 0) then
          call MPI_FILE_OPEN  (MPI_COMM_SELF, "output.dat", MPI_MODE_RDONLY, my_info, fileID, ierr)
          mylen = ncells
          allocate (array2(ncells))
          call MPI_File_read(fileID, array2, mylen , MPI_REAL, status,ierr)
          call MPI_Get_count(status,MPI_INTEGER, count, ierr)
          if (count .ne. mylen) print*, "Wrong read count:", count,mylen
          do i=1,ncells
               if (array2(i) .ne. 1) then
                  print*, "ERROR", i,array2(i), ((i-1)*4), ((i-1)*4)/(1024d0*1024d0) ! Index, value, # of good bytes,MB
                  goto 999
               end if
          end do
          print*, "All done with nothing wrong"
 999      deallocate(array2)
          call MPI_FILE_CLOSE (fileID,ierr)
          call MPI_file_delete ("output.dat",MPI_INFO_NULL,ierr)
      endif
 
      call mpi_finalize(ierr)
 
end program gcrm_test_io  
 
1.3 Open MPI
 node           0 starting                     1 ending                410011
 node           1 starting                410012 ending                820022
 node           2 starting                820023 ending               1230033
 node           3 starting               1230034 ending               1640044
 node           4 starting               1640045 ending               2050055
 node           5 starting               2050056 ending               2460066
 All done with nothing wrong


 node           0 starting                     1 ending                410011
 node           1 starting                410012 ending                820022
 node           2 starting                820023 ending               1230033
 node           5 starting               2050056 ending               2460066
 node           4 starting               1640045 ending               2050055
 node           3 starting               1230034 ending               1640044
 Wrong write count:      228554                410011           2
 Wrong read count:     1048576               2460066
 ERROR     1048577  0.0000000E+00     4194304   4.00000000000000   


 node           1 starting                410012 ending                820022
 node           0 starting                     1 ending                410011
 node           2 starting                820023 ending               1230033
 node           3 starting               1230034 ending               1640044
 node           4 starting               1640045 ending               2050055
 node           5 starting               2050056 ending               2460066
 Wrong read count:     1229824               2460066
 ERROR     1229825  0.0000000E+00     4919296   4.69140625000000

--
Nathan Baca
nathan.baca@gmail.com

_______________________________________________
users mailing list
users@open-mpi.org
http://www.open-mpi.org/mailman/listinfo.cgi/users