diff --git a/input/restart.sh b/input/restart.sh new file mode 100755 index 000000000..f5f1559ac --- /dev/null +++ b/input/restart.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +NPROCS=$1 +AMRFILE=$2 +EXE=/Users/jwise/codes/EnzoWOC/week-of-code/src/enzo/enzo.exe + +# Extract data dump number +NCHARS=`echo $AMRFILE | wc -c | gawk '{printf "%d", $1}'` +C1=`expr $NCHARS - 4` +C2=`expr $NCHARS - 5` +NUM=`echo $AMRFILE | cut -c${C1}- | gawk '{printf "%d", $1}'` +PREFIX=`echo $AMRFILE | cut -c1-${C2}` + +pwd +date + +echo "mpirun -np $NPROCS $EXE -d -r $AMRFILE" +mpirun -np $NPROCS $EXE -d -r $AMRFILE & diff --git a/src/enzo/CheckForOutput.C b/src/enzo/CheckForOutput.C index fe404160e..238c1a2b1 100644 --- a/src/enzo/CheckForOutput.C +++ b/src/enzo/CheckForOutput.C @@ -64,11 +64,8 @@ int CheckForOutput(HierarchyEntry *TopGrid, TopGridData &MetaData, MetaData.TimeLastDataDump += MetaData.dtDataDump; //#ifdef USE_HDF5_GROUPS - if (Group_WriteAllData(MetaData.DataDumpName, MetaData.DataDumpNumber++, - TopGrid, MetaData, Exterior) == FAIL) { - fprintf(stderr, "Error in Group_WriteAllData.\n"); - ENZO_FAIL(""); - } + Group_WriteAllData(MetaData.DataDumpName, MetaData.DataDumpNumber++, + TopGrid, MetaData, Exterior); // #else // if (WriteAllData(MetaData.DataDumpName, MetaData.DataDumpNumber++, // TopGrid, MetaData, Exterior) == FAIL) { @@ -88,11 +85,8 @@ int CheckForOutput(HierarchyEntry *TopGrid, TopGridData &MetaData, MetaData.CycleLastDataDump += MetaData.CycleSkipDataDump; //#ifdef USE_HDF5_GROUPS - if (Group_WriteAllData(MetaData.DataDumpName, MetaData.DataDumpNumber++, - TopGrid, MetaData, Exterior) == FAIL) { - fprintf(stderr, "Error in Group_WriteAllData.\n"); - ENZO_FAIL(""); - } + Group_WriteAllData(MetaData.DataDumpName, MetaData.DataDumpNumber++, + TopGrid, MetaData, Exterior); // #else // if (WriteAllData(MetaData.DataDumpName, MetaData.DataDumpNumber++, // TopGrid, MetaData, Exterior) == FAIL) { @@ -123,11 +117,8 @@ int CheckForOutput(HierarchyEntry *TopGrid, TopGridData &MetaData, MetaData.CycleLastDataDump = MetaData.CycleNumber; if (debug) printf("CPUtime-based output!\n"); //#ifdef USE_HDF5_GROUPS - if (Group_WriteAllData(MetaData.DataDumpName, MetaData.DataDumpNumber++, - TopGrid, MetaData, Exterior) == FAIL) { - fprintf(stderr, "Error in Group_WriteAllData.\n"); - ENZO_FAIL(""); - } + Group_WriteAllData(MetaData.DataDumpName, MetaData.DataDumpNumber++, + TopGrid, MetaData, Exterior); // #else // if (WriteAllData(MetaData.DataDumpName, MetaData.DataDumpNumber++, // TopGrid, MetaData, Exterior) == FAIL) { @@ -138,6 +129,48 @@ int CheckForOutput(HierarchyEntry *TopGrid, TopGridData &MetaData, WroteData = TRUE; } // ENDIF + /* Check for output: restart-based. */ + + char *param; + FILE *pfptr; + + if ((CurrentCPUTime >= MetaData.dtRestartDump && + MetaData.dtRestartDump > 0 ) || + (MetaData.CycleNumber - MetaData.CycleLastRestartDump >= + MetaData.CycleSkipRestartDump && + MetaData.CycleSkipRestartDump > 0)) { + + MetaData.CycleLastRestartDump = MetaData.CycleNumber; + + if (debug) printf("Writing restart dump.\n"); + Group_WriteAllData(MetaData.RestartDumpName, MetaData.RestartDumpNumber++, + TopGrid, MetaData, Exterior); + + /* On the root processor, write the restart parameter filename to + a file that will be read by a (batch) script to restart enzo. + We cannot call another MPI application from here. */ + + if (MyProcessorNumber == ROOT_PROCESSOR) { + param = new char[512]; + if (MetaData.RestartDumpDir != NULL) + sprintf(param, "%s%"CYCLE_TAG_FORMAT""ISYM"/%s%"CYCLE_TAG_FORMAT""ISYM, + MetaData.RestartDumpDir, MetaData.RestartDumpNumber-1, + MetaData.RestartDumpName, MetaData.RestartDumpNumber-1); + else + sprintf(param, "%s%"CYCLE_TAG_FORMAT""ISYM, + MetaData.RestartDumpName, MetaData.RestartDumpNumber-1); + + if ((pfptr = fopen("RestartParamFile", "w")) == NULL) + ENZO_FAIL("Error opening RestartParamFile"); + fprintf(pfptr, "%s", param); + fclose(pfptr); + + delete [] param; + } // ENDIF ROOT_PROCESSOR + + WroteData = TRUE; + } + /* Check for output: redshift-based. */ if (ComovingCoordinates) @@ -155,10 +188,7 @@ int CheckForOutput(HierarchyEntry *TopGrid, TopGridData &MetaData, } //#ifdef USE_HDF5_GROUPS - if (Group_WriteAllData(Name, Number, TopGrid, MetaData, Exterior) == FAIL) { - fprintf(stderr, "Error in Group_WriteAllData.\n"); - ENZO_FAIL(""); - } + Group_WriteAllData(Name, Number, TopGrid, MetaData, Exterior); // #else // if (WriteAllData(Name, Number, TopGrid, MetaData, Exterior) == FAIL) { // fprintf(stderr, "Error in WriteAllData.\n"); diff --git a/src/enzo/CheckForResubmit.C b/src/enzo/CheckForResubmit.C index b50d604c2..44575696f 100644 --- a/src/enzo/CheckForResubmit.C +++ b/src/enzo/CheckForResubmit.C @@ -31,7 +31,7 @@ int CheckForResubmit(TopGridData &MetaData, int &Stop) if (MetaData.ResubmitOn == FALSE) return SUCCESS; - char *cmd = new char[256]; + char *cmd = new char[512]; double CurrentCPUTime = ReturnWallTime() - MetaData.StartCPUTime; if (CurrentCPUTime + MetaData.LastCycleCPUTime > MetaData.StopCPUTime) { @@ -40,13 +40,13 @@ int CheckForResubmit(TopGridData &MetaData, int &Stop) "Executing resubmission script, %s\n", MetaData.ResubmitCommand); if (MyProcessorNumber == ROOT_PROCESSOR) { if (MetaData.DataDumpDir != NULL) - sprintf(cmd, "%s/%s %d %s%4.4d/%s%4.4d", MetaData.GlobalDir, - MetaData.ResubmitCommand, NumberOfProcessors, + sprintf(cmd, "%s/%s %"ISYM" %s%"CYCLE_TAG_FORMAT""ISYM"/%s%"CYCLE_TAG_FORMAT""ISYM, + MetaData.GlobalDir, MetaData.ResubmitCommand, NumberOfProcessors, MetaData.DataDumpDir, MetaData.DataDumpNumber-1, MetaData.DataDumpName, MetaData.DataDumpNumber-1); else - sprintf(cmd, "%s/%s %d %s%4.4d", MetaData.GlobalDir, - MetaData.ResubmitCommand, NumberOfProcessors, + sprintf(cmd, "%s/%s %"ISYM" %s%"CYCLE_TAG_FORMAT""ISYM, + MetaData.GlobalDir, MetaData.ResubmitCommand, NumberOfProcessors, MetaData.DataDumpName, MetaData.DataDumpNumber-1); printf("command: %s\n", cmd); system(cmd); @@ -54,5 +54,7 @@ int CheckForResubmit(TopGridData &MetaData, int &Stop) Stop = TRUE; } // ENDIF + delete [] cmd; + return SUCCESS; } diff --git a/src/enzo/EvolveHierarchy.C b/src/enzo/EvolveHierarchy.C index 92e6faa86..8186366b2 100644 --- a/src/enzo/EvolveHierarchy.C +++ b/src/enzo/EvolveHierarchy.C @@ -116,6 +116,7 @@ int EvolveHierarchy(HierarchyEntry &TopGrid, TopGridData &MetaData, float dt; int i, dim, Stop = FALSE, WroteData; + int Restart = FALSE; double tlev0, tlev1, treb0, treb1, tloop0, tloop1, tentry, texit; LevelHierarchyEntry *Temp; double LastCPUTime; @@ -146,10 +147,7 @@ int EvolveHierarchy(HierarchyEntry &TopGrid, TopGridData &MetaData, NumberOfGrids++; Temp = Temp->NextGridThisLevel; } - if (Enzo_Dims_create(NumberOfGrids, MetaData.TopGridRank, Layout) == FAIL) { - fprintf(stderr, "Error in Enzo_Dims_create.\n"); - ENZO_FAIL(""); - } + Enzo_Dims_create(NumberOfGrids, MetaData.TopGridRank, Layout); for (dim = 0; dim < MetaData.TopGridRank; dim++) if (MetaData.TopGridDims[dim] % Layout[MAX_DIMENSION-1-dim] != 0) { if (debug) @@ -199,10 +197,8 @@ int EvolveHierarchy(HierarchyEntry &TopGrid, TopGridData &MetaData, } if (CopyOverlappingZones(Temp->GridData, &MetaData, LevelArray, 0) - == FAIL) { - fprintf(stderr, "Error in CopyOverlappingZones.\n"); - ENZO_FAIL(""); - } + == FAIL) + ENZO_FAIL("Error in CopyOverlappingZones."); Temp = Temp->NextGridThisLevel; } @@ -210,15 +206,9 @@ int EvolveHierarchy(HierarchyEntry &TopGrid, TopGridData &MetaData, Temp = LevelArray[0]; while (Temp != NULL) { -// if (Temp->GridData->SetExternalBoundaryValues(Exterior) == FAIL) { -// fprintf(stderr, "Error in grid->SetExternalBoundaryValues.\n"); -// ENZO_FAIL(""); -// } if (CopyOverlappingZones(Temp->GridData, &MetaData, LevelArray, 0) - == FAIL) { - fprintf(stderr, "Error in CopyOverlappingZones.\n"); - ENZO_FAIL(""); - } + == FAIL) + ENZO_FAIL("Error in CopyOverlappingZones."); Temp = Temp->NextGridThisLevel; } @@ -226,10 +216,7 @@ int EvolveHierarchy(HierarchyEntry &TopGrid, TopGridData &MetaData, CommunicationBarrier(); #endif - if (CommunicationReceiveHandler() == FAIL) { - fprintf(stderr, "Error in CommunicationReceiveHandle.\n"); - ENZO_FAIL(""); - } + CommunicationReceiveHandler(); #ifdef FORCE_MSG_PROGRESS CommunicationBarrier(); @@ -253,14 +240,12 @@ int EvolveHierarchy(HierarchyEntry &TopGrid, TopGridData &MetaData, /* Check for output. */ - if (CheckForOutput(&TopGrid, MetaData, Exterior, WroteData) == FAIL) { - fprintf(stderr, "Error in CheckForOutput.\n"); - ENZO_FAIL(""); - } + CheckForOutput(&TopGrid, MetaData, Exterior, WroteData); #ifdef MEM_TRACE - MemInUse = mused(); - fprintf(memtracePtr, "Output %8"ISYM" %16"ISYM" \n", MetaData.CycleNumber, MemInUse); + MemInUse = mused(); + fprintf(memtracePtr, "Output %8"ISYM" %16"ISYM" \n", + MetaData.CycleNumber, MemInUse); #endif /* Compute the acceleration field so ComputeTimeStep can find dtAccel. @@ -277,14 +262,12 @@ int EvolveHierarchy(HierarchyEntry &TopGrid, TopGridData &MetaData, /* Do the first grid regeneration. */ - if (RebuildHierarchy(&MetaData, LevelArray, 0) == FAIL) { - fprintf(stderr, "Error in RebuildHierarchy.\n"); - ENZO_FAIL(""); - } + RebuildHierarchy(&MetaData, LevelArray, 0); #ifdef MEM_TRACE - MemInUse = mused(); - fprintf(memtracePtr, "1st rebuild %8"ISYM" %16"ISYM" \n", MetaData.CycleNumber, MemInUse); + MemInUse = mused(); + fprintf(memtracePtr, "1st rebuild %8"ISYM" %16"ISYM" \n", + MetaData.CycleNumber, MemInUse); #endif /* Open the OutputLevelInformation file. */ @@ -327,18 +310,16 @@ int EvolveHierarchy(HierarchyEntry &TopGrid, TopGridData &MetaData, /* Output level information to log file. */ - if (MyProcessorNumber == ROOT_PROCESSOR) { + if (MyProcessorNumber == ROOT_PROCESSOR) LevelInfofptr = fopen("OutputLevelInformation.out", "a"); - } // OutputLevelInformation() only needs to be called by all processors // when jbPerf is enabled. OutputLevelInformation(LevelInfofptr, MetaData, LevelArray); - if (MyProcessorNumber == ROOT_PROCESSOR) { + if (MyProcessorNumber == ROOT_PROCESSOR) fclose(LevelInfofptr); - } /* Compute minimum timestep on the top level. */ @@ -501,32 +482,33 @@ int EvolveHierarchy(HierarchyEntry &TopGrid, TopGridData &MetaData, printf("Stopping on CPU time limit.\n"); Stop = TRUE; } - + if ((ReturnWallTime() - MetaData.StartCPUTime >= MetaData.dtRestartDump && + MetaData.dtRestartDump > 0) || + (MetaData.CycleNumber - MetaData.CycleLastRestartDump >= + MetaData.CycleSkipRestartDump && + MetaData.CycleSkipRestartDump > 0)) { + if (MyProcessorNumber == ROOT_PROCESSOR) + printf("Stopping to restart.\n"); + Stop = TRUE; + Restart = TRUE; + } /* Check for time-actions. */ - if (CheckForTimeAction(LevelArray, MetaData) == FAIL) { - fprintf(stderr, "Error in CheckForTimeActions.\n"); - ENZO_FAIL(""); - } + CheckForTimeAction(LevelArray, MetaData); /* Check for output. */ - if (CheckForOutput(&TopGrid, MetaData, Exterior, WroteData) == FAIL) { - fprintf(stderr, "Error in CheckForOutput.\n"); - ENZO_FAIL(""); - } + CheckForOutput(&TopGrid, MetaData, Exterior, WroteData); /* Check for resubmission */ - if (CheckForResubmit(MetaData, Stop) == FAIL) { - fprintf(stderr, "Error in CheckForResubmit.\n"); - ENZO_FAIL(""); - } + if (!Restart) + CheckForResubmit(MetaData, Stop); /* If stopping, inline halo finder one more time */ - if (Stop) + if (Stop && !Restart) FOF(&MetaData, LevelArray); @@ -535,11 +517,7 @@ int EvolveHierarchy(HierarchyEntry &TopGrid, TopGridData &MetaData, #ifdef REDUCE_FRAGMENTATION if (WroteData && !Stop) - if (ReduceFragmentation(TopGrid, MetaData, Exterior, LevelArray) - == FAIL) { - fprintf(stderr, "Error in ReduceFragmentation.\n"); - ENZO_FAIL(""); - } + ReduceFragmentation(TopGrid, MetaData, Exterior, LevelArray); #endif /* REDUCE_FRAGMENTATION */ @@ -634,10 +612,8 @@ int EvolveHierarchy(HierarchyEntry &TopGrid, TopGridData &MetaData, !WroteData) //#ifdef USE_HDF5_GROUPS if (Group_WriteAllData(MetaData.DataDumpName, MetaData.DataDumpNumber, - &TopGrid, MetaData, Exterior, -666) == FAIL) { - fprintf(stderr, "Error in Group_WriteAllData.\n"); - ENZO_FAIL(""); - } + &TopGrid, MetaData, Exterior, -666) == FAIL) + ENZO_FAIL("Error in Group_WriteAllData."); // #else // if (WriteAllData(MetaData.DataDumpName, MetaData.DataDumpNumber, // &TopGrid, MetaData, Exterior, -666) == FAIL) { @@ -646,6 +622,16 @@ int EvolveHierarchy(HierarchyEntry &TopGrid, TopGridData &MetaData, // } // #endif + /* Write a file to indicate that we're finished. */ + + FILE *Exit_fptr; + if (!Restart && MyProcessorNumber == ROOT_PROCESSOR) { + if ((Exit_fptr = fopen("RunFinished", "w")) == NULL) + ENZO_FAIL("Error opening RunFinished."); + fprintf(Exit_fptr, "Finished on cycle %"ISYM"\n", MetaData.CycleNumber); + fclose(Exit_fptr); + } + if (NumberOfProcessors > 1) printf("Communication: processor %"ISYM" CommunicationTime = %"FSYM"\n", MyProcessorNumber, CommunicationTime); diff --git a/src/enzo/Group_WriteAllData.C b/src/enzo/Group_WriteAllData.C index 138135881..d79b5d721 100644 --- a/src/enzo/Group_WriteAllData.C +++ b/src/enzo/Group_WriteAllData.C @@ -193,6 +193,8 @@ int Group_WriteAllData(char *basename, int filenumber, sprintf(id, "%"CYCLE_TAG_FORMAT""ISYM, filenumber); sprintf(pid, "%"TASK_TAG_FORMAT""ISYM, MyProcessorNumber); + + /******************** CYCLE / DT BASED OUTPUTS ********************/ if ( (cptr = strstr(basename, MetaData.DataDumpName)) ) { @@ -251,6 +253,8 @@ int Group_WriteAllData(char *basename, int filenumber, if (debug) fprintf(stderr, "DATA dump: %s\n", name); } // if DataDumpName + + /******************** RESTART BASED OUTPUTS ********************/ if ( (cptr = strstr(basename, MetaData.RedshiftDumpName)) ) { @@ -310,6 +314,67 @@ int Group_WriteAllData(char *basename, int filenumber, fprintf(stderr, "REDSHIFT dump: %s\n", name); } // if RedshiftDumpName + + /******************** RESTART BASED OUTPUTS ********************/ + + if ( (cptr = strstr(basename, MetaData.RestartDumpName)) ) { + + if (MetaData.RestartDumpDir != NULL) + { + if (MetaData.LocalDir != NULL) { + // Local fs + strcpy(dumpdirname, MetaData.LocalDir); + strcat(dumpdirname, "/"); + strcat(dumpdirname, MetaData.RestartDumpDir); + strcat(dumpdirname, id); + + // Create once per node... +#ifdef USE_NODE_LOCAL + strcat(dumpdirname, "/mpi"); + strcat(dumpdirname, pid); +#endif /* USE_NODE_LOCAL */ + + strcpy(name, dumpdirname); + strcat(name, "/"); + strcat(name, basename); + } // if LocalDir + + else + + { + if (MetaData.GlobalDir != NULL) { + // Global fs + strcpy(dumpdirname, MetaData.GlobalDir); + strcat(dumpdirname, "/"); + strcat(dumpdirname, MetaData.RestartDumpDir); + strcat(dumpdirname, id); + // Do mkdir on cpu 0 only + strcpy(name, dumpdirname); + strcat(name, "/"); + strcat(name, basename); + } // if GlobalDir + + else + + { + // No local or global specified + strcpy(name, basename); + } // else GlobalDir + + } // else LocalDir + + } // if RedshiftDumpDir + + else + + { + strcpy(name, basename); + } // else RedshiftDumpDir + + if (debug) + fprintf(stderr, "RESTART dump: %s\n", name); + + } // if RestartDumpName if (filenumber >= 0) strcat(name, id); @@ -373,7 +438,7 @@ int Group_WriteAllData(char *basename, int filenumber, if (debug) fprintf(stderr, "DATA dump: %s == %"ISYM"\n", unixcommand, unixresult); #endif } - } + } // ENDIF datadump if ( (cptr = strstr(basename, MetaData.RedshiftDumpName)) ) { if (MetaData.RedshiftDumpDir != NULL) { @@ -386,10 +451,24 @@ int Group_WriteAllData(char *basename, int filenumber, fprintf(stderr, "REDSHIFT dump: %s == %"ISYM"\n", unixcommand, unixresult); #endif } - } - } - } - } + } // ENDIF redshift + + if ( (cptr = strstr(basename, MetaData.RestartDumpName)) ) { + if (MetaData.RestartDumpDir != NULL) { +#ifdef SYSCALL + unixresult = SysMkdir("", dumpdirname); + fprintf(stderr, "RESTART dump: dumpdirname=(%s) == unixresult=%"ISYM"\n", dumpdirname, unixresult); +#else + strcat(strcpy(unixcommand, "mkdir -p "), dumpdirname); + unixresult = system(unixcommand); + fprintf(stderr, "RESTART dump: %s == %"ISYM"\n", unixcommand, unixresult); +#endif + } + } // ENDIF restart + + } // ENDIF pe == ii + } // ENDFOR ii + } // ENDIF local if ( global ) { @@ -407,7 +486,7 @@ int Group_WriteAllData(char *basename, int filenumber, if (debug) fprintf(stderr, "DATA dump: %s == %"ISYM"\n", unixcommand, unixresult); #endif } - } + } // ENDIF datadump if ( (cptr = strstr(basename, MetaData.RedshiftDumpName)) ) { if (MetaData.RedshiftDumpDir != NULL) { @@ -420,10 +499,23 @@ int Group_WriteAllData(char *basename, int filenumber, fprintf(stderr, "REDSHIFT dump: %s == %"ISYM"\n", unixcommand, unixresult); #endif } - } + } // ENDIF redshift + + if ( (cptr = strstr(basename, MetaData.RestartDumpName)) ) { + if (MetaData.RestartDumpDir != NULL) { +#ifdef SYSCALL + unixresult = SysMkdir("", dumpdirname); + fprintf(stderr, "RESTART dump: dumpdirname=(%s) == unixresult=%"ISYM"\n", dumpdirname, unixresult); +#else + strcat(strcpy(unixcommand, "mkdir -p "), dumpdirname); + unixresult = system(unixcommand); + fprintf(stderr, "RESTART dump: %s == %"ISYM"\n", unixcommand, unixresult); +#endif + } + } // ENDIF restart - } - } + } // ENDIF ROOT_PROCESSOR + } // ENDIF global // fprintf(stderr, "Sync point ok\n"); @@ -480,10 +572,8 @@ int Group_WriteAllData(char *basename, int filenumber, if (WriteTime >= 0) fprintf(fptr, "# WARNING! Interpolated output: level = %"ISYM"\n", MetaData.OutputFirstTimeAtLevel-1); - if (WriteParameterFile(fptr, MetaData) == FAIL) { - fprintf(stderr, "Error in WriteParameterFile\n"); - ENZO_FAIL(""); - } + if (WriteParameterFile(fptr, MetaData) == FAIL) + ENZO_FAIL("Error in WriteParameterFile"); fclose(fptr); } @@ -498,10 +588,8 @@ int Group_WriteAllData(char *basename, int filenumber, } strcat(MetaData.BoundaryConditionName, hdfsuffix); if (Exterior->WriteExternalBoundary(fptr, MetaData.BoundaryConditionName) - == FAIL) { - fprintf(stderr, "Error in WriteExternalBoundary\n"); - ENZO_FAIL(""); - } + == FAIL) + ENZO_FAIL("Error in WriteExternalBoundary"); fclose(fptr); } @@ -539,10 +627,8 @@ int Group_WriteAllData(char *basename, int filenumber, ENZO_FAIL(""); } - if (Group_WriteDataHierarchy(fptr, MetaData, TempTopGrid, gridbasename, GridID, WriteTime, file_id) == FAIL) { - fprintf(stderr, "Error in Group_WriteDataHierarchy\n"); - ENZO_FAIL(""); - } + if (Group_WriteDataHierarchy(fptr, MetaData, TempTopGrid, gridbasename, GridID, WriteTime, file_id) == FAIL) + ENZO_FAIL("Error in Group_WriteDataHierarchy"); // At this point all the grid data has been written @@ -562,10 +648,8 @@ int Group_WriteAllData(char *basename, int filenumber, ENZO_FAIL(""); } - if (WriteMemoryMap(mptr, TempTopGrid, gridbasename, GridKD, WriteTime) == FAIL) { - fprintf(stderr, "Error in WriteMemoryMap\n"); - ENZO_FAIL(""); - } + if (WriteMemoryMap(mptr, TempTopGrid, gridbasename, GridKD, WriteTime) == FAIL) + ENZO_FAIL("Error in WriteMemoryMap"); // Output configure @@ -587,10 +671,8 @@ int Group_WriteAllData(char *basename, int filenumber, ENZO_FAIL(""); } - if (WriteTaskMap(tptr, TempTopGrid, gridbasename, GridLD, WriteTime) == FAIL) { - fprintf(stderr, "Error in WriteTaskMap\n"); - ENZO_FAIL(""); - } + if (WriteTaskMap(tptr, TempTopGrid, gridbasename, GridLD, WriteTime) == FAIL) + ENZO_FAIL("Error in WriteTaskMap"); int TGdims[3]; @@ -600,12 +682,9 @@ int Group_WriteAllData(char *basename, int filenumber, // fprintf(stderr, "TGdims %"ISYM" %"ISYM" %"ISYM"\n", TGdims[0], TGdims[1], TGdims[2]); - if (CubeDumpEnabled == 1) { - if (WriteDataCubes(TempTopGrid, TGdims, name, GridJD, WriteTime) == FAIL) { - fprintf(stderr, "Error in WriteDataCubes\n"); - ENZO_FAIL(""); - } - } + if (CubeDumpEnabled == 1) + if (WriteDataCubes(TempTopGrid, TGdims, name, GridJD, WriteTime) == FAIL) + ENZO_FAIL("Error in WriteDataCubes"); // Clean up combined top level grid, and first two levels of hierarchy @@ -618,10 +697,8 @@ int Group_WriteAllData(char *basename, int filenumber, // Output StarParticle data (actually just number of stars) - if (WriteStarParticleData(fptr) == FAIL) { - fprintf(stderr, "Error in WriteStarParticleData\n"); - ENZO_FAIL(""); - } + if (WriteStarParticleData(fptr) == FAIL) + ENZO_FAIL("Error in WriteStarParticleData"); // Create radiation name and write radiation data @@ -637,10 +714,8 @@ int Group_WriteAllData(char *basename, int filenumber, fprintf(stderr, "Error opening radiation file %s\n", radiationname); ENZO_FAIL(""); } - if (WriteRadiationData(Radfptr) == FAIL) { - fprintf(stderr, "Error in WriteRadiationData\n"); - ENZO_FAIL(""); - } + if (WriteRadiationData(Radfptr) == FAIL) + ENZO_FAIL("Error in WriteRadiationData"); fclose(Radfptr); diff --git a/src/enzo/InitializeNew.C b/src/enzo/InitializeNew.C index 425a0778a..f89246cbe 100644 --- a/src/enzo/InitializeNew.C +++ b/src/enzo/InitializeNew.C @@ -624,8 +624,6 @@ int InitializeNew(char *filename, HierarchyEntry &TopGrid, MetaData.TimeLastDataDump = MetaData.Time - MetaData.dtDataDump*1.00001; if (MetaData.TimeLastHistoryDump == FLOAT_UNDEFINED) MetaData.TimeLastHistoryDump = MetaData.Time - MetaData.dtHistoryDump; - if (MetaData.TimeLastMovieDump == FLOAT_UNDEFINED) - MetaData.TimeLastMovieDump = MetaData.Time - MetaData.dtMovieDump; if (MetaData.TimeLastTracerParticleDump == FLOAT_UNDEFINED) MetaData.TimeLastTracerParticleDump = diff --git a/src/enzo/SetDefaultGlobalValues.C b/src/enzo/SetDefaultGlobalValues.C index 2f581cd5b..1d4a6f326 100644 --- a/src/enzo/SetDefaultGlobalValues.C +++ b/src/enzo/SetDefaultGlobalValues.C @@ -69,7 +69,7 @@ int SetDefaultGlobalValues(TopGridData &MetaData) MetaData.ResubmitCommand = NULL; MetaData.TimeLastRestartDump = 0.0; - MetaData.dtRestartDump = 5.0*3600.0; // every 5 hours + MetaData.dtRestartDump = FLOAT_UNDEFINED; MetaData.TimeLastDataDump = FLOAT_UNDEFINED; MetaData.dtDataDump = 0.0; MetaData.TimeLastHistoryDump = FLOAT_UNDEFINED; diff --git a/src/enzo/TopGridData.h b/src/enzo/TopGridData.h index af669aa35..48d92ff55 100644 --- a/src/enzo/TopGridData.h +++ b/src/enzo/TopGridData.h @@ -24,9 +24,13 @@ struct TopGridData double CPUTime; // Current CPU time used double StartCPUTime; double LastCycleCPUTime; // CPU time used in the last cycle - char *ResubmitCommand; // Script name for job resubmission int ResubmitOn; // Resubmit job after StopCPUTime + /* Script names for resubmission to queues and restarting to reduce + memory fragmentation. */ + + char *ResubmitCommand; // Script name for job resubmission + /* Stopping criteria for TopGrid. */ FLOAT StopTime; // time to stop at @@ -36,8 +40,8 @@ struct TopGridData /* Parameters governing when output is done. */ - FLOAT TimeLastRestartDump; // CPU time of the last restart dump (seconds) - FLOAT dtRestartDump; // CPU time between restart dumps (0 = never) + float TimeLastRestartDump; // CPU time of the last restart dump (seconds) + float dtRestartDump; // CPU time between restart dumps (0 = never) FLOAT TimeLastDataDump; // Problem time of the last data dump FLOAT dtDataDump; // Problem time between data dumps (0 = never) diff --git a/src/enzo/Unigrid_EvolveLevel.C b/src/enzo/Unigrid_EvolveLevel.C index 7afddcb10..a1fd87131 100644 --- a/src/enzo/Unigrid_EvolveLevel.C +++ b/src/enzo/Unigrid_EvolveLevel.C @@ -514,29 +514,6 @@ int EvolveLevel(TopGridData *MetaData, LevelHieraR7V&w& JBPERF_STOP("evolve-level-22"); // CommunicationUpdateStarParticleCount() #endif - /* Check for movie output (only check if this is bottom of hierarchy). */ - -#ifdef USE_JBPERF - JBPERF_START("evolve-level-23"); // WriteMovieData() -#endif - - if (LevelArray[level+1] == NULL) - if (LevelArray[level]->GridData->ReturnTime() >= - MetaData->TimeLastMovieDump + MetaData->dtMovieDump && - MetaData->dtMovieDump > 0.0) { - MetaData->TimeLastMovieDump += MetaData->dtMovieDump; - if (WriteMovieData(MetaData->MovieDumpName, - MetaData->MovieDumpNumber++, LevelArray, MetaData, - LevelArray[level]->GridData->ReturnTime()) == FAIL) { - fprintf(stderr, "Error in WriteMovieData.\n"); - ENZO_FAIL(""); - } - } - -#ifdef USE_JBPERF - JBPERF_STOP("evolve-level-23"); // WriteMovieData() -#endif - /* Check for tracer particle output (only if this bottom of hierarchy). */ #ifdef USE_JBPERF diff --git a/src/enzo/WriteParameterFile.C b/src/enzo/WriteParameterFile.C index ee31acb97..7911ab8e0 100644 --- a/src/enzo/WriteParameterFile.C +++ b/src/enzo/WriteParameterFile.C @@ -83,7 +83,7 @@ int WriteParameterFile(FILE *fptr, TopGridData &MetaData) fprintf(fptr, "StopSteps = %"ISYM"\n", MetaData.StopSteps); fprintf(fptr, "StopCPUTime = %lg\n", MetaData.StopCPUTime); fprintf(fptr, "ResubmitOn = %"ISYM"\n", MetaData.ResubmitOn); - fprintf(fptr, "ResubmitCommand = %s\n\n", MetaData.ResubmitCommand); + fprintf(fptr, "ResubmitCommand = %s\n", MetaData.ResubmitCommand); fprintf(fptr, "TimeLastRestartDump = %"GOUTSYM"\n", MetaData.TimeLastRestartDump); fprintf(fptr, "dtRestartDump = %"GOUTSYM"\n", MetaData.dtRestartDump);