Skip to content

Commit

Permalink
Automated merge with http://hg.spacepope.org/week-of-code/
Browse files Browse the repository at this point in the history 
--HG--
branch : week-of-code
  • Loading branch information
John Wise committed Jul 16, 2009
2 parents 689307e + 69760c0 commit 6f86e52
Show file tree
Hide file tree
Showing 10 changed files with 246 additions and 156 deletions.
18 changes: 18 additions & 0 deletions input/restart.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
#!/bin/sh

NPROCS=$1
AMRFILE=$2
EXE=/Users/jwise/codes/EnzoWOC/week-of-code/src/enzo/enzo.exe

# Extract data dump number
NCHARS=`echo $AMRFILE | wc -c | gawk '{printf "%d", $1}'`
C1=`expr $NCHARS - 4`
C2=`expr $NCHARS - 5`
NUM=`echo $AMRFILE | cut -c${C1}- | gawk '{printf "%d", $1}'`
PREFIX=`echo $AMRFILE | cut -c1-${C2}`

pwd
date

echo "mpirun -np $NPROCS $EXE -d -r $AMRFILE"
mpirun -np $NPROCS $EXE -d -r $AMRFILE &
68 changes: 49 additions & 19 deletions src/enzo/CheckForOutput.C
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,11 +64,8 @@ int CheckForOutput(HierarchyEntry *TopGrid, TopGridData &MetaData,
MetaData.TimeLastDataDump += MetaData.dtDataDump;

//#ifdef USE_HDF5_GROUPS
if (Group_WriteAllData(MetaData.DataDumpName, MetaData.DataDumpNumber++,
TopGrid, MetaData, Exterior) == FAIL) {
fprintf(stderr, "Error in Group_WriteAllData.\n");
ENZO_FAIL("");
}
Group_WriteAllData(MetaData.DataDumpName, MetaData.DataDumpNumber++,
TopGrid, MetaData, Exterior);
// #else
// if (WriteAllData(MetaData.DataDumpName, MetaData.DataDumpNumber++,
// TopGrid, MetaData, Exterior) == FAIL) {
Expand All @@ -88,11 +85,8 @@ int CheckForOutput(HierarchyEntry *TopGrid, TopGridData &MetaData,
MetaData.CycleLastDataDump += MetaData.CycleSkipDataDump;

//#ifdef USE_HDF5_GROUPS
if (Group_WriteAllData(MetaData.DataDumpName, MetaData.DataDumpNumber++,
TopGrid, MetaData, Exterior) == FAIL) {
fprintf(stderr, "Error in Group_WriteAllData.\n");
ENZO_FAIL("");
}
Group_WriteAllData(MetaData.DataDumpName, MetaData.DataDumpNumber++,
TopGrid, MetaData, Exterior);
// #else
// if (WriteAllData(MetaData.DataDumpName, MetaData.DataDumpNumber++,
// TopGrid, MetaData, Exterior) == FAIL) {
Expand Down Expand Up @@ -123,11 +117,8 @@ int CheckForOutput(HierarchyEntry *TopGrid, TopGridData &MetaData,
MetaData.CycleLastDataDump = MetaData.CycleNumber;
if (debug) printf("CPUtime-based output!\n");
//#ifdef USE_HDF5_GROUPS
if (Group_WriteAllData(MetaData.DataDumpName, MetaData.DataDumpNumber++,
TopGrid, MetaData, Exterior) == FAIL) {
fprintf(stderr, "Error in Group_WriteAllData.\n");
ENZO_FAIL("");
}
Group_WriteAllData(MetaData.DataDumpName, MetaData.DataDumpNumber++,
TopGrid, MetaData, Exterior);
// #else
// if (WriteAllData(MetaData.DataDumpName, MetaData.DataDumpNumber++,
// TopGrid, MetaData, Exterior) == FAIL) {
Expand All @@ -138,6 +129,48 @@ int CheckForOutput(HierarchyEntry *TopGrid, TopGridData &MetaData,
WroteData = TRUE;
} // ENDIF

/* Check for output: restart-based. */

char *param;
FILE *pfptr;

if ((CurrentCPUTime >= MetaData.dtRestartDump &&
MetaData.dtRestartDump > 0 ) ||
(MetaData.CycleNumber - MetaData.CycleLastRestartDump >=
MetaData.CycleSkipRestartDump &&
MetaData.CycleSkipRestartDump > 0)) {

MetaData.CycleLastRestartDump = MetaData.CycleNumber;

if (debug) printf("Writing restart dump.\n");
Group_WriteAllData(MetaData.RestartDumpName, MetaData.RestartDumpNumber++,
TopGrid, MetaData, Exterior);

/* On the root processor, write the restart parameter filename to
a file that will be read by a (batch) script to restart enzo.
We cannot call another MPI application from here. */

if (MyProcessorNumber == ROOT_PROCESSOR) {
param = new char[512];
if (MetaData.RestartDumpDir != NULL)
sprintf(param, "%s%"CYCLE_TAG_FORMAT""ISYM"/%s%"CYCLE_TAG_FORMAT""ISYM,
MetaData.RestartDumpDir, MetaData.RestartDumpNumber-1,
MetaData.RestartDumpName, MetaData.RestartDumpNumber-1);
else
sprintf(param, "%s%"CYCLE_TAG_FORMAT""ISYM,
MetaData.RestartDumpName, MetaData.RestartDumpNumber-1);

if ((pfptr = fopen("RestartParamFile", "w")) == NULL)
ENZO_FAIL("Error opening RestartParamFile");
fprintf(pfptr, "%s", param);
fclose(pfptr);

delete [] param;
} // ENDIF ROOT_PROCESSOR

WroteData = TRUE;
}

/* Check for output: redshift-based. */

if (ComovingCoordinates)
Expand All @@ -155,10 +188,7 @@ int CheckForOutput(HierarchyEntry *TopGrid, TopGridData &MetaData,
}

//#ifdef USE_HDF5_GROUPS
if (Group_WriteAllData(Name, Number, TopGrid, MetaData, Exterior) == FAIL) {
fprintf(stderr, "Error in Group_WriteAllData.\n");
ENZO_FAIL("");
}
Group_WriteAllData(Name, Number, TopGrid, MetaData, Exterior);
// #else
// if (WriteAllData(Name, Number, TopGrid, MetaData, Exterior) == FAIL) {
// fprintf(stderr, "Error in WriteAllData.\n");
Expand Down
12 changes: 7 additions & 5 deletions src/enzo/CheckForResubmit.C
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ int CheckForResubmit(TopGridData &MetaData, int &Stop)
if (MetaData.ResubmitOn == FALSE)
return SUCCESS;

char *cmd = new char[256];
char *cmd = new char[512];
double CurrentCPUTime = ReturnWallTime() - MetaData.StartCPUTime;

if (CurrentCPUTime + MetaData.LastCycleCPUTime > MetaData.StopCPUTime) {
Expand All @@ -40,19 +40,21 @@ int CheckForResubmit(TopGridData &MetaData, int &Stop)
"Executing resubmission script, %s\n", MetaData.ResubmitCommand);
if (MyProcessorNumber == ROOT_PROCESSOR) {
if (MetaData.DataDumpDir != NULL)
sprintf(cmd, "%s/%s %d %s%4.4d/%s%4.4d", MetaData.GlobalDir,
MetaData.ResubmitCommand, NumberOfProcessors,
sprintf(cmd, "%s/%s %"ISYM" %s%"CYCLE_TAG_FORMAT""ISYM"/%s%"CYCLE_TAG_FORMAT""ISYM,
MetaData.GlobalDir, MetaData.ResubmitCommand, NumberOfProcessors,
MetaData.DataDumpDir, MetaData.DataDumpNumber-1,
MetaData.DataDumpName, MetaData.DataDumpNumber-1);
else
sprintf(cmd, "%s/%s %d %s%4.4d", MetaData.GlobalDir,
MetaData.ResubmitCommand, NumberOfProcessors,
sprintf(cmd, "%s/%s %"ISYM" %s%"CYCLE_TAG_FORMAT""ISYM,
MetaData.GlobalDir, MetaData.ResubmitCommand, NumberOfProcessors,
MetaData.DataDumpName, MetaData.DataDumpNumber-1);
printf("command: %s\n", cmd);
system(cmd);
}
Stop = TRUE;
} // ENDIF

delete [] cmd;

return SUCCESS;
}
104 changes: 45 additions & 59 deletions src/enzo/EvolveHierarchy.C
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,7 @@ int EvolveHierarchy(HierarchyEntry &TopGrid, TopGridData &MetaData,
float dt;

int i, dim, Stop = FALSE, WroteData;
int Restart = FALSE;
double tlev0, tlev1, treb0, treb1, tloop0, tloop1, tentry, texit;
LevelHierarchyEntry *Temp;
double LastCPUTime;
Expand Down Expand Up @@ -146,10 +147,7 @@ int EvolveHierarchy(HierarchyEntry &TopGrid, TopGridData &MetaData,
NumberOfGrids++;
Temp = Temp->NextGridThisLevel;
}
if (Enzo_Dims_create(NumberOfGrids, MetaData.TopGridRank, Layout) == FAIL) {
fprintf(stderr, "Error in Enzo_Dims_create.\n");
ENZO_FAIL("");
}
Enzo_Dims_create(NumberOfGrids, MetaData.TopGridRank, Layout);
for (dim = 0; dim < MetaData.TopGridRank; dim++)
if (MetaData.TopGridDims[dim] % Layout[MAX_DIMENSION-1-dim] != 0) {
if (debug)
Expand Down Expand Up @@ -199,37 +197,26 @@ int EvolveHierarchy(HierarchyEntry &TopGrid, TopGridData &MetaData,

}
if (CopyOverlappingZones(Temp->GridData, &MetaData, LevelArray, 0)
== FAIL) {
fprintf(stderr, "Error in CopyOverlappingZones.\n");
ENZO_FAIL("");
}
== FAIL)
ENZO_FAIL("Error in CopyOverlappingZones.");
Temp = Temp->NextGridThisLevel;
}

CommunicationDirection = COMMUNICATION_SEND;

Temp = LevelArray[0];
while (Temp != NULL) {
// if (Temp->GridData->SetExternalBoundaryValues(Exterior) == FAIL) {
// fprintf(stderr, "Error in grid->SetExternalBoundaryValues.\n");
// ENZO_FAIL("");
// }
if (CopyOverlappingZones(Temp->GridData, &MetaData, LevelArray, 0)
== FAIL) {
fprintf(stderr, "Error in CopyOverlappingZones.\n");
ENZO_FAIL("");
}
== FAIL)
ENZO_FAIL("Error in CopyOverlappingZones.");
Temp = Temp->NextGridThisLevel;
}

#ifdef FORCE_MSG_PROGRESS
CommunicationBarrier();
#endif

if (CommunicationReceiveHandler() == FAIL) {
fprintf(stderr, "Error in CommunicationReceiveHandle.\n");
ENZO_FAIL("");
}
CommunicationReceiveHandler();

#ifdef FORCE_MSG_PROGRESS
CommunicationBarrier();
Expand All @@ -253,14 +240,12 @@ int EvolveHierarchy(HierarchyEntry &TopGrid, TopGridData &MetaData,

/* Check for output. */

if (CheckForOutput(&TopGrid, MetaData, Exterior, WroteData) == FAIL) {
fprintf(stderr, "Error in CheckForOutput.\n");
ENZO_FAIL("");
}
CheckForOutput(&TopGrid, MetaData, Exterior, WroteData);

#ifdef MEM_TRACE
MemInUse = mused();
fprintf(memtracePtr, "Output %8"ISYM" %16"ISYM" \n", MetaData.CycleNumber, MemInUse);
MemInUse = mused();
fprintf(memtracePtr, "Output %8"ISYM" %16"ISYM" \n",
MetaData.CycleNumber, MemInUse);
#endif

/* Compute the acceleration field so ComputeTimeStep can find dtAccel.
Expand All @@ -277,14 +262,12 @@ int EvolveHierarchy(HierarchyEntry &TopGrid, TopGridData &MetaData,

/* Do the first grid regeneration. */

if (RebuildHierarchy(&MetaData, LevelArray, 0) == FAIL) {
fprintf(stderr, "Error in RebuildHierarchy.\n");
ENZO_FAIL("");
}
RebuildHierarchy(&MetaData, LevelArray, 0);

#ifdef MEM_TRACE
MemInUse = mused();
fprintf(memtracePtr, "1st rebuild %8"ISYM" %16"ISYM" \n", MetaData.CycleNumber, MemInUse);
MemInUse = mused();
fprintf(memtracePtr, "1st rebuild %8"ISYM" %16"ISYM" \n",
MetaData.CycleNumber, MemInUse);
#endif

/* Open the OutputLevelInformation file. */
Expand Down Expand Up @@ -327,18 +310,16 @@ int EvolveHierarchy(HierarchyEntry &TopGrid, TopGridData &MetaData,

/* Output level information to log file. */

if (MyProcessorNumber == ROOT_PROCESSOR) {
if (MyProcessorNumber == ROOT_PROCESSOR)
LevelInfofptr = fopen("OutputLevelInformation.out", "a");
}

// OutputLevelInformation() only needs to be called by all processors
// when jbPerf is enabled.

OutputLevelInformation(LevelInfofptr, MetaData, LevelArray);

if (MyProcessorNumber == ROOT_PROCESSOR) {
if (MyProcessorNumber == ROOT_PROCESSOR)
fclose(LevelInfofptr);
}

/* Compute minimum timestep on the top level. */

Expand Down Expand Up @@ -501,32 +482,33 @@ int EvolveHierarchy(HierarchyEntry &TopGrid, TopGridData &MetaData,
printf("Stopping on CPU time limit.\n");
Stop = TRUE;
}

if ((ReturnWallTime() - MetaData.StartCPUTime >= MetaData.dtRestartDump &&
MetaData.dtRestartDump > 0) ||
(MetaData.CycleNumber - MetaData.CycleLastRestartDump >=
MetaData.CycleSkipRestartDump &&
MetaData.CycleSkipRestartDump > 0)) {
if (MyProcessorNumber == ROOT_PROCESSOR)
printf("Stopping to restart.\n");
Stop = TRUE;
Restart = TRUE;
}

/* Check for time-actions. */

if (CheckForTimeAction(LevelArray, MetaData) == FAIL) {
fprintf(stderr, "Error in CheckForTimeActions.\n");
ENZO_FAIL("");
}
CheckForTimeAction(LevelArray, MetaData);

/* Check for output. */

if (CheckForOutput(&TopGrid, MetaData, Exterior, WroteData) == FAIL) {
fprintf(stderr, "Error in CheckForOutput.\n");
ENZO_FAIL("");
}
CheckForOutput(&TopGrid, MetaData, Exterior, WroteData);

/* Check for resubmission */

if (CheckForResubmit(MetaData, Stop) == FAIL) {
fprintf(stderr, "Error in CheckForResubmit.\n");
ENZO_FAIL("");
}
if (!Restart)
CheckForResubmit(MetaData, Stop);

/* If stopping, inline halo finder one more time */

if (Stop)
if (Stop && !Restart)
FOF(&MetaData, LevelArray);


Expand All @@ -535,11 +517,7 @@ int EvolveHierarchy(HierarchyEntry &TopGrid, TopGridData &MetaData,
#ifdef REDUCE_FRAGMENTATION

if (WroteData && !Stop)
if (ReduceFragmentation(TopGrid, MetaData, Exterior, LevelArray)
== FAIL) {
fprintf(stderr, "Error in ReduceFragmentation.\n");
ENZO_FAIL("");
}
ReduceFragmentation(TopGrid, MetaData, Exterior, LevelArray);

#endif /* REDUCE_FRAGMENTATION */

Expand Down Expand Up @@ -634,10 +612,8 @@ int EvolveHierarchy(HierarchyEntry &TopGrid, TopGridData &MetaData,
!WroteData)
//#ifdef USE_HDF5_GROUPS
if (Group_WriteAllData(MetaData.DataDumpName, MetaData.DataDumpNumber,
&TopGrid, MetaData, Exterior, -666) == FAIL) {
fprintf(stderr, "Error in Group_WriteAllData.\n");
ENZO_FAIL("");
}
&TopGrid, MetaData, Exterior, -666) == FAIL)
ENZO_FAIL("Error in Group_WriteAllData.");
// #else
// if (WriteAllData(MetaData.DataDumpName, MetaData.DataDumpNumber,
// &TopGrid, MetaData, Exterior, -666) == FAIL) {
Expand All @@ -646,6 +622,16 @@ int EvolveHierarchy(HierarchyEntry &TopGrid, TopGridData &MetaData,
// }
// #endif

/* Write a file to indicate that we're finished. */

FILE *Exit_fptr;
if (!Restart && MyProcessorNumber == ROOT_PROCESSOR) {
if ((Exit_fptr = fopen("RunFinished", "w")) == NULL)
ENZO_FAIL("Error opening RunFinished.");
fprintf(Exit_fptr, "Finished on cycle %"ISYM"\n", MetaData.CycleNumber);
fclose(Exit_fptr);
}

if (NumberOfProcessors > 1)
printf("Communication: processor %"ISYM" CommunicationTime = %"FSYM"\n",
MyProcessorNumber, CommunicationTime);
Expand Down
Loading

0 comments on commit 6f86e52

Please sign in to comment.