Skip to content

Commit c31e780

Browse files
robywwrobyww
committed
DM-10370: Optimize server memory usage for fits images
 - Server memory size cut in half - Optimized drawing so that user sees working message - Optimized scrolling so the it does not block when loading images - beta value (used for asinh strech) now only computed when required - added a getBetaValue service call - Uploading fits files now show masking while uploading - removed addColorBand call from server (not used) - Added way to push out memory in cache before large data is read in - Fixed bug in computing in CsysConvert.pointInPlot with rectangular images
1 parent 6b3ff9d commit c31e780

29 files changed

+350
-234
lines changed

‎src/firefly/java/edu/caltech/ipac/firefly/data/ServerParams.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ public class ServerParams {
103103
public static final String CREATE_PLOT_GROUP = "CmdCreatePlotGroup";
104104
public static final String ZOOM = "CmdZoom";
105105
public static final String STRETCH = "CmdStretch";
106-
public static final String ADD_BAND = "CmdAddBand";
106+
public static final String GET_BETA = "CmdGetBeta";
107107
public static final String REMOVE_BAND = "CmdRemoveBand";
108108
public static final String CHANGE_COLOR = "CmdChangeColor";
109109
public static final String ROTATE_NORTH = "CmdRotateNorth";

‎src/firefly/java/edu/caltech/ipac/firefly/rpc/PlotServiceJson.java‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -117,9 +117,6 @@ public void deleteColorBand(PlotState state, Band band, AsyncCallback<WebPlotRes
117117
}
118118

119119
public void addColorBand(PlotState state, WebPlotRequest bandRequest, Band band, AsyncCallback<WebPlotResult> async) {
120-
doPlotService(ServerParams.ADD_BAND, async, state,
121-
new Param(ServerParams.BAND, band.toString()),
122-
new Param(ServerParams.REQUEST, bandRequest.toString()));
123120
}
124121

125122
public void getFileFlux(FileAndHeaderInfo[] fileAndHeader, ImagePt inIpt, final AsyncCallback<String[]> async) {

‎src/firefly/java/edu/caltech/ipac/firefly/server/ServerCommandAccess.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ private static void initCommand() {
4646
_cmdMap.put(ServerParams.CREATE_PLOT_GROUP, new VisServerCommands.GetWebPlotGroupCmd());
4747
_cmdMap.put(ServerParams.ZOOM, new VisServerCommands.ZoomCmd());
4848
_cmdMap.put(ServerParams.STRETCH, new VisServerCommands.StretchCmd());
49-
_cmdMap.put(ServerParams.ADD_BAND, new VisServerCommands.AddBandCmd());
49+
_cmdMap.put(ServerParams.GET_BETA, new VisServerCommands.GetBetaCmd());
5050
_cmdMap.put(ServerParams.REMOVE_BAND, new VisServerCommands.RemoveBandCmd());
5151
_cmdMap.put(ServerParams.CHANGE_COLOR, new VisServerCommands.ChangeColor());
5252
_cmdMap.put(ServerParams.DELETE, new VisServerCommands.DeletePlot());

‎src/firefly/java/edu/caltech/ipac/firefly/server/cache/EhcacheProvider.java‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ public class EhcacheProvider implements Cache.Provider {
4040
private static long curConfModTime = 0;
4141

4242
static {
43+
4344
URL url = null;
4445
File f = getConfFile("ehcache.xml");
4546
if (f != null && f.canRead()) {
@@ -65,7 +66,14 @@ public class EhcacheProvider implements Cache.Provider {
6566
File ignoreSizeOf = getConfFile("ignore_sizeof.txt");
6667
System.setProperty("net.sf.ehcache.sizeof.filter", ignoreSizeOf.getAbsolutePath());
6768

68-
sharedManager = CacheManager.create(sharedConfig.getAbsolutePath());
69+
try {
70+
System.setProperty("net.sf.ehcache.sizeofengine.shared.VIS_SHARED_MEM",
71+
"edu.caltech.ipac.firefly.server.cache.ObjectSizeEngineWrapper");
72+
sharedManager = CacheManager.create(sharedConfig.getAbsolutePath());
73+
} catch (RuntimeException e) {
74+
System.clearProperty("net.sf.ehcache.sizeofengine.shared.VIS_SHARED_MEM");
75+
sharedManager = CacheManager.create(sharedConfig.getAbsolutePath());
76+
}
6977
float pctVisSharedMemSize = AppProperties.getFloatProperty("pct.vis.shared.mem.size", 0F);
7078

7179
// check to see if vis.shared.mem.size is setup in the environment or setup for auto-config.

‎src/firefly/java/edu/caltech/ipac/firefly/server/rpc/PlotServiceImpl.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,7 @@ public WebPlotResult deleteColorBand(PlotState state, Band band) {
160160
public WebPlotResult addColorBand(PlotState state,
161161
WebPlotRequest bandRequest,
162162
Band band) {
163-
return VisServerOps.addColorBand(state, bandRequest, band);
163+
return null; // no longer used
164164
}
165165

166166

‎src/firefly/java/edu/caltech/ipac/firefly/server/rpc/VisServerCommands.java‎

Lines changed: 26 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,32 @@ public String doCommand(Map<String, String[]> paramMap) throws IllegalArgumentEx
200200
}
201201
}
202202

203+
204+
public static class GetBetaCmd extends ServCommand {
205+
206+
public String doCommand(Map<String, String[]> paramMap) throws IllegalArgumentException {
207+
208+
SrvParam sp= new SrvParam(paramMap);
209+
PlotState state= sp.getState();
210+
double resultAry[]= VisServerOps.getBeta(state);
211+
212+
JSONObject obj= new JSONObject();
213+
obj.put("success", true);
214+
215+
JSONArray data= new JSONArray();
216+
for(double r : resultAry) data.add(r);
217+
218+
JSONArray wrapperAry= new JSONArray();
219+
obj.put("data", data);
220+
wrapperAry.add(obj);
221+
222+
return wrapperAry.toJSONString();
223+
}
224+
}
225+
226+
227+
228+
203229
public static class StretchCmd extends ServCommand {
204230

205231
public String doCommand(Map<String, String[]> paramMap) throws IllegalArgumentException {
@@ -232,19 +258,6 @@ public String doCommand(Map<String, String[]> paramMap) throws IllegalArgumentEx
232258
}
233259
}
234260

235-
public static class AddBandCmd extends ServCommand {
236-
237-
public String doCommand(Map<String, String[]> paramMap) throws IllegalArgumentException {
238-
239-
SrvParam sp= new SrvParam(paramMap);
240-
PlotState state= sp.getState();
241-
Band band = Band.parse(sp.getRequired(ServerParams.BAND));
242-
WebPlotRequest req= WebPlotRequest.parse(sp.getRequired(ServerParams.REQUEST));
243-
WebPlotResult result = VisServerOps.addColorBand(state, req, band);
244-
return WebPlotResultSerializer.createJson(result, sp.isJsonDeep());
245-
}
246-
}
247-
248261
public static class RemoveBandCmd extends ServCommand {
249262

250263
public String doCommand(Map<String, String[]> paramMap) throws IllegalArgumentException {

‎src/firefly/java/edu/caltech/ipac/firefly/server/visualize/CtxControl.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,6 @@ private static ActiveCallCtx revalidatePlot(PlotClientCtx ctx) {
112112
state.getColorTableId(), rv);
113113
plot.getPlotGroup().setZoomTo(state.getZoomLevel());
114114
if (state.isThreeColor()) plot.setThreeColorBand(fr[imageIdx],band,frGroup);
115-
ctx.setPlot(plot);
116115
first= false;
117116
}
118117
else {
@@ -217,7 +216,10 @@ public static PlotClientCtx getPlotCtx(String ctxStr) {
217216
return (PlotClientCtx) getCache().get(new StringKey(ctxStr));
218217
}
219218

220-
public static boolean isCtxAvailable(String ctxStr) { return getPlotCtx(ctxStr)!=null; }
219+
public static boolean isCtxAvailable(String ctxStr) {
220+
PlotClientCtx ctx= getPlotCtx(ctxStr);
221+
return (ctx!=null && ctx.getCachedPlot()!=null);
222+
}
221223

222224
public static void putPlotCtx(PlotClientCtx ctx) {
223225
if (ctx!=null && ctx.getKey()!=null) {

‎src/firefly/java/edu/caltech/ipac/firefly/server/visualize/FitsCacher.java‎

Lines changed: 31 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
package edu.caltech.ipac.firefly.server.visualize;
66

77
import edu.caltech.ipac.firefly.server.ServerContext;
8+
import edu.caltech.ipac.firefly.server.cache.ObjectSizeEngineWrapper;
89
import edu.caltech.ipac.firefly.server.util.Logger;
910
import edu.caltech.ipac.util.FileUtil;
1011
import edu.caltech.ipac.util.UTCTimeUtil;
@@ -32,10 +33,14 @@ public class FitsCacher {
3233
private static final Logger.LoggerImpl _log = Logger.getLogger();
3334

3435
static FitsRead[] readFits(File fitsFile) throws FitsException, FailedRequestException, IOException {
36+
return readFits(fitsFile,true, false);
37+
}
38+
39+
static FitsRead[] readFits(File fitsFile, boolean useCache, boolean clearHdu) throws FitsException, FailedRequestException, IOException {
3540
CacheKey key= new StringKey(fitsFile.getPath());
36-
FitsRead frAry[];
41+
FitsRead frAry[]= null;
3742

38-
frAry= getFromCache(key);
43+
if (useCache) frAry= getFromCache(key);
3944
if (frAry!=null) { // check first with out any locking
4045
return frAry;
4146
}
@@ -59,9 +64,13 @@ static FitsRead[] readFits(File fitsFile) throws FitsException, FailedRequestExc
5964
else {
6065
Fits fits= null;
6166
try {
67+
if (memCache != null) {
68+
memCache.put(key, new ObjectSizeEngineWrapper.BluffSize(fitsFile.length()));
69+
memCache.put(key, null);
70+
}
6271
fits= new Fits(fitsFile.getPath());
6372
long start = System.currentTimeMillis();
64-
frAry = FitsRead.createFitsReadArray(fits);
73+
frAry = FitsRead.createFitsReadArray(fits, clearHdu);
6574
if (memCache != null) memCache.put(key, frAry);
6675
long elapse = System.currentTimeMillis() - start;
6776
String timeStr = UTCTimeUtil.getHMSFromMills(elapse);
@@ -116,6 +125,25 @@ static FitsRead[] loadFits(Fits fits, File cachePath) throws FitsException, Fail
116125
}
117126
}
118127

128+
129+
130+
static void clearCachedHDU(File fitsFile) {
131+
CacheKey key= new StringKey(fitsFile.getPath());
132+
FitsRead frAry[]= getFromCache(key);
133+
if (frAry!=null) {
134+
boolean needsReinsert= false;
135+
for (FitsRead fr : frAry) {
136+
if (fr!=null && fr.hasHdu()) {
137+
fr.clearHDU();
138+
needsReinsert= true;
139+
}
140+
}
141+
if (memCache != null && needsReinsert) memCache.put(key, frAry);
142+
}
143+
}
144+
145+
146+
119147
public static void addFitsReadToCache(File fitsFile, FitsRead frAry[]) {
120148
addFitsReadToCache(fitsFile.getPath(), frAry);
121149
}

‎src/firefly/java/edu/caltech/ipac/firefly/server/visualize/ImagePlotCreator.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,7 @@ public static ModFileWriter createBand(PlotState state,
197197
Band band= readInfo.getBand();
198198

199199

200+
FitsCacher.clearCachedHDU(readInfo.getOriginalFile());
200201
plot.setThreeColorBand(state.isBandVisible(readInfo.getBand()) ? readInfo.getFitsRead() :null,
201202
readInfo.getBand(),frGroup);
202203
HistogramOps histOps= plot.getHistogramOps(band,frGroup);
@@ -320,7 +321,8 @@ public static WebFitsData makeWebFitsData(ImagePlot plot, ActiveFitsReadGroup fr
320321
long fileLength= (f!=null && f.canRead()) ? f.length() : 0;
321322
HistogramOps ops= plot.getHistogramOps(band,frGroup);
322323
Histogram hist= ops.getDataHistogram();
323-
return new WebFitsData( ops.getDataMin(hist), ops.getDataMax(hist),ops.getBeta(),
324+
return new WebFitsData( ops.getDataMin(hist), ops.getDataMax(hist),
325+
Double.NaN /*no longer pass beta by default*/,
324326
fileLength, plot.getFluxUnits(band,frGroup));
325327
}
326328
}

‎src/firefly/java/edu/caltech/ipac/firefly/server/visualize/ModFileWriter.java‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -38,18 +38,15 @@ private ModFileWriter(Band band, File targetFile, boolean markAsOriginal) {
3838
protected boolean doTask() { return true; }
3939

4040
/**
41-
* Start the file writing in separate thread. This method also update the working fits file to the name that is about
42-
* to be created.
41+
* Write the fits file and update data structures
4342
* @param state the PlotState to update
4443
*/
45-
public void go(PlotState state) {
44+
public void writeFile(PlotState state) {
4645
PlotStateUtil.setWorkingFitsFile(state, _targetFile, _band);
4746
if (_markAsOriginal) {
4847
PlotStateUtil.setOriginalFitsFile(state, _targetFile, _band);
4948
}
50-
if (doTask()) {
51-
write();
52-
}
49+
if (doTask()) write();
5350
}
5451

5552
public boolean getCreatesOnlyOneImage() { return true; }
@@ -117,6 +114,7 @@ protected void write() {
117114
OutputStream os= new BufferedOutputStream(new FileOutputStream(f), 1024*16);
118115
_fr.writeSimpleFitsFile(os);
119116
FileUtil.silentClose(os);
117+
_fr.clearHDU();
120118
} catch (Exception e) {
121119
_log.warn(e,"geom write failed", "geom file: "+f.getPath());
122120
}

0 commit comments

Comments
 (0)