Merge pull request #2016 from NCAR/fix/netcdf4_var_par_access

Collective access when extending a file

Author: jedwards4b

src/clib/pio_getput_int.c

@@ -1081,7 +1081,6 @@ PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Offset
             else
                 fake_stride = (PIO_Offset *)stride;
         }
-
 #ifdef _PNETCDF
         if (file->iotype == PIO_IOTYPE_PNETCDF)
         {
@@ -1137,6 +1136,15 @@ PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Offset
         {
             PLOG((2, "PIOc_put_vars_tc calling netcdf function file->iotype = %d",
                   file->iotype));
+#ifdef _NETCDF4
+	    if (file->iotype == PIO_IOTYPE_NETCDF4P)
+	    {
+		PLOG((2, "Setting NC_COLLECTIVE mode"));
+		if ((ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE)))
+		    return pio_err(ios, file, ierr, __FILE__, __LINE__);
+	    }
+            
+#endif
             switch(xtype)
             {
             case NC_BYTE:

src/clib/pioc_support.c

@@ -2720,7 +2720,7 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename,
 #ifdef _MPISERIAL
             ierr = nc_open(filename, mode, &file->fh);
 #else
-            imode = mode |  NC_MPIIO;
+            imode = mode |  NC_MPIIO | NC_NETCDF4;
             if ((ierr = nc_open_par(filename, imode, ios->io_comm, ios->info,
                                     &file->fh))){
 	      PLOG((2, "%d: PIOc_openfile_retry nc_open_par ierr=%d",__LINE__,ierr));
@@ -2736,6 +2736,7 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename,
                                           &pio_type_size, &mpi_type,
                                           &mpi_type_size, &ndims)))
                 break;
+	    
             PLOG((2, "PIOc_openfile_retry:nc_open_par filename = %s mode = %d "
                   "imode = %d ierr = %d", filename, mode, imode, ierr));
 #endif

tests/cunit/test_pioc_putget.c

@@ -1767,7 +1767,7 @@ int putget_read_vars_nt(int ncid, int *varid, PIO_Offset *start, PIO_Offset *cou
  * @returns 0 for success, error code otherwise.
  */
 int create_putget_file(int iosysid, int access, int unlim, int flavor, int *dim_len,
-                       int *varid, const char *filename, int *ncidp)
+                       int *varid, const char *filename, int *ncidp, int *tsvarid)
 {
     int dimids[NDIM];        /* The dimension IDs. */
     int num_vars = NUM_CLASSIC_TYPES + 1;
@@ -1786,7 +1786,9 @@ int create_putget_file(int iosysid, int access, int unlim, int flavor, int *dim_
         return ret;
 
     /* Are we using unlimited dimension? */
-    if (!unlim)
+    if (unlim)
+	dim_len[0] = NC_UNLIMITED;
+    else
         dim_len[0] = NUM_TIMESTEPS;
 
     /* Define netCDF dimensions and variable. */
@@ -1812,7 +1814,14 @@ int create_putget_file(int iosysid, int access, int unlim, int flavor, int *dim_
         if ((ret = PIOc_def_var(ncid, var_name, my_type, NDIM, dimids, &varid[v])))
             return ret;
     }
-
+    if(unlim){
+	/* create an additional variable with only the unlimdim dimension */
+	if ((ret = PIOc_def_var(ncid, "timestep", PIO_INT, 1, dimids, tsvarid)))
+	{
+	    printf("Defined a timestep variable %d\n", ret);
+	    return ret;
+	}
+    }
     /* For the first access, also test attributes. */
     if (access == 0)
         if ((ret = test_write_atts(ncid, varid, flavor)))
@@ -1908,7 +1917,8 @@ int test_putget(int iosysid, int num_flavors, int *flavor, int my_rank,
                 MPI_Comm test_comm)
 {
     int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN};
-
+    int tsvarid;
+    
 #define NUM_ACCESS 8
     for (int unlim = 0; unlim < 2; unlim++)
         for (int access = 0; access < NUM_ACCESS; access++)
@@ -1931,15 +1941,15 @@ int test_putget(int iosysid, int num_flavors, int *flavor, int my_rank,
 
                 /* Create test file with dims and vars defined. */
                 if ((ret = create_putget_file(iosysid, access, unlim, flavor[fmt], dim_len, varid,
-                                              filename, &ncid)))
+                                              filename, &ncid, &tsvarid)))
                     return ret;
 
                 /* Write some data. */
                 PIO_Offset index[NDIM] = {0, 0, 0};
                 PIO_Offset start[NDIM] = {0, 0, 0};
                 PIO_Offset count[NDIM] = {1, X_DIM_LEN, Y_DIM_LEN};
                 PIO_Offset stride[NDIM] = {1, 1, 1};
-
+		PIO_Offset index1d = 0;
                 switch (access)
                 {
                 case 0:
@@ -2003,7 +2013,21 @@ int test_putget(int iosysid, int num_flavors, int *flavor, int my_rank,
                 /* Check contents of the file. */
                 if ((ret = check_file(access, ncid, varid, flavor[fmt], index, start, count, stride, unlim)))
                     return ret;
-
+		if(unlim > 0) {
+		    /* try advancing the file */
+		    index1d = 0;
+		    if ((ret = PIOc_put_var1_int(ncid, tsvarid, &index1d, dim_len)))
+		    {
+			printf("extend file time test %d\n",ret);
+			return ret;
+		    }
+		    index1d = 1;
+		    if ((ret = PIOc_put_var1_int(ncid, tsvarid, &index1d, dim_len+1)))
+		    {
+			printf("extend file time test %d\n",ret);
+			return ret;
+		    }
+		}
                 /* Close the netCDF file. */
                 if ((ret = PIOc_closefile(ncid)))
                     ERR(ret);

tests/fncint/ftst_pio.f90

@@ -24,7 +24,7 @@ program ftst_pio
   integer, dimension(3) :: var_dim
   integer :: maplen
   integer :: decompid, iosysid
-  integer :: varid, i
+  integer :: varid, i, tsvarid
   integer :: ierr
 
   ! Set up MPI.
@@ -75,13 +75,19 @@ program ftst_pio
   ! Define a data variable.
   ierr = nf_def_var(ncid, VAR_NAME, NF_INT, NDIM3, var_dim, varid)
   if (ierr .ne. nf_noerr) call handle_err(ierr)
+
+  ierr = nf_def_var(ncid, 'timestep', NF_INT, 1, var_dim(3), tsvarid)
+
   ierr = nf_enddef(ncid)
   if (ierr .ne. nf_noerr) call handle_err(ierr)
 
   ! Write 1st record with distributed arrays.
   ierr = nf_put_vard_int(ncid, varid, decompid, 1, data_buffer)
   if (ierr .ne. nf_noerr) call handle_err(ierr)
 
+  ierr = nf_put_var(ncid, tsvarid, 2, 2) 
+  if (ierr .ne. nf_noerr) call handle_err(ierr)
+  
   ! Close the file.
   ierr = nf_close(ncid)
   if (ierr .ne. nf_noerr) call handle_err(ierr)

tests/general/ncdf_simple_tests.F90.in

@@ -100,6 +100,69 @@ PIO_TF_AUTO_TEST_SUB_BEGIN test_def_var
 
 PIO_TF_AUTO_TEST_SUB_END test_def_var
 
+PIO_TF_AUTO_TEST_SUB_BEGIN test_extend_unlimited_var
+  use ncdf_simple_tests_tgv
+  Implicit none
+  type(file_desc_t) :: pio_file
+  type(var_desc_t)  :: pio_var
+  integer :: pio_dim
+  integer :: ret, i
+  integer, parameter :: initial_len = 5, extend_len = 3
+  integer, dimension(initial_len + extend_len) :: data_out, data_in
+  character(len=PIO_TF_MAX_STR_LEN), parameter :: fname = "pio_test_extend_unlim.nc"
+
+  ! Create file with unlimited dimension and write initial data
+  ret = PIO_createfile(pio_tf_iosystem_, pio_file, tgv_iotype, fname, PIO_CLOBBER)
+  PIO_TF_CHECK_ERR(ret, "Failed to create:" // trim(fname))
+
+  ret = PIO_def_dim(pio_file, 'unlim_dim', PIO_UNLIMITED, pio_dim) ! -1 for unlimited
+  PIO_TF_CHECK_ERR(ret, "Failed to define unlimited dim:" // trim(fname))
+
+  ret = PIO_def_var(pio_file, 'unlim_var', PIO_int, (/pio_dim/), pio_var)
+  PIO_TF_CHECK_ERR(ret, "Failed to define var:" // trim(fname))
+
+  ret = PIO_enddef(pio_file)
+  PIO_TF_CHECK_ERR(ret, "Failed to enddef:" // trim(fname))
+
+  do i = 1, initial_len
+    data_out(i) = i
+  end do
+  ret = PIO_put_var(pio_file, pio_var, data_out(1:initial_len))
+  PIO_TF_CHECK_ERR(ret, "Failed to write initial data:" // trim(fname))
+
+  call PIO_closefile(pio_file)
+
+  ! Reopen file and extend variable
+  ret = PIO_openfile(pio_tf_iosystem_, pio_file, tgv_iotype, fname, PIO_write)
+  PIO_TF_CHECK_ERR(ret, "Failed to reopen:" // trim(fname))
+
+  do i = 1, extend_len
+    data_out(initial_len + i) = 100 + i
+  end do
+  ret = PIO_put_var(pio_file, pio_var, start=(/initial_len/), count=(/extend_len/), &
+                    ival=data_out(initial_len+1:initial_len+extend_len))
+  PIO_TF_CHECK_ERR(ret, "Failed to extend data:" // trim(fname))
+
+  call PIO_syncfile(pio_file)
+
+  ! Optionally, read back and verify
+  ret = PIO_get_var(pio_file, pio_var, data_in)
+  PIO_TF_CHECK_ERR(ret, "Failed to read back data:" // trim(fname))
+  do i = 1, initial_len
+    if (data_in(i) /= i) then
+      print *, "ERROR: Initial data mismatch at ", i, ":", data_in(i), "!=", i
+    end if
+  end do
+  do i = 1, extend_len
+    if (data_in(initial_len + i) /= 100 + i) then
+      print *, "ERROR: Extended data mismatch at ", initial_len + i, ":", data_in(initial_len + i), "!=", 100 + i
+    end if
+  end do
+
+  call PIO_closefile(pio_file)
+  call PIO_deletefile(pio_tf_iosystem_, fname)
+
+PIO_TF_AUTO_TEST_SUB_END test_extend_unlimited_var
 PIO_TF_TEMPLATE<PIO_TF_PREDEF_TYPENAME PIO_TF_DATA_TYPE, PIO_TF_PREDEF_TYPENAME PIO_TF_FC_DATA_TYPE>
 PIO_TF_AUTO_TEST_SUB_BEGIN test_data_conversion
   use ncdf_simple_tests_tgv
@@ -190,6 +253,8 @@ PIO_TF_TEST_DRIVER_BEGIN
 
     ! Make sure that global variables are set correctly before running the tests
     PIO_TF_AUTO_TESTS_RUN(trim(iotype_descs(i)))
+    PRINT *, "PIO_TF: Starting test_extend_unlimited_var for ", trim(iotype_descs(i))
+    CALL test_extend_unlimited_var()
 
     call PIO_deletefile(pio_tf_iosystem_, tgv_fname)
   end do