Skip to content

Commit 3f1de4a

Browse files
robywwrobyww
authored
DM-10370: Merge PR #371 from dm-10370-mem-half
DM-10370: Optimize server memory usage for fits images
2 parents 634ace1 + dcc3d5c commit 3f1de4a

30 files changed

+405
-234
lines changed

‎src/firefly/java/edu/caltech/ipac/firefly/data/ServerParams.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ public class ServerParams {
103103
public static final String CREATE_PLOT_GROUP = "CmdCreatePlotGroup";
104104
public static final String ZOOM = "CmdZoom";
105105
public static final String STRETCH = "CmdStretch";
106-
public static final String ADD_BAND = "CmdAddBand";
106+
public static final String GET_BETA = "CmdGetBeta";
107107
public static final String REMOVE_BAND = "CmdRemoveBand";
108108
public static final String CHANGE_COLOR = "CmdChangeColor";
109109
public static final String ROTATE_NORTH = "CmdRotateNorth";

‎src/firefly/java/edu/caltech/ipac/firefly/rpc/PlotServiceJson.java‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -117,9 +117,6 @@ public void deleteColorBand(PlotState state, Band band, AsyncCallback<WebPlotRes
117117
}
118118

119119
public void addColorBand(PlotState state, WebPlotRequest bandRequest, Band band, AsyncCallback<WebPlotResult> async) {
120-
doPlotService(ServerParams.ADD_BAND, async, state,
121-
new Param(ServerParams.BAND, band.toString()),
122-
new Param(ServerParams.REQUEST, bandRequest.toString()));
123120
}
124121

125122
public void getFileFlux(FileAndHeaderInfo[] fileAndHeader, ImagePt inIpt, final AsyncCallback<String[]> async) {

‎src/firefly/java/edu/caltech/ipac/firefly/server/ServerCommandAccess.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ private static void initCommand() {
4646
_cmdMap.put(ServerParams.CREATE_PLOT_GROUP, new VisServerCommands.GetWebPlotGroupCmd());
4747
_cmdMap.put(ServerParams.ZOOM, new VisServerCommands.ZoomCmd());
4848
_cmdMap.put(ServerParams.STRETCH, new VisServerCommands.StretchCmd());
49-
_cmdMap.put(ServerParams.ADD_BAND, new VisServerCommands.AddBandCmd());
49+
_cmdMap.put(ServerParams.GET_BETA, new VisServerCommands.GetBetaCmd());
5050
_cmdMap.put(ServerParams.REMOVE_BAND, new VisServerCommands.RemoveBandCmd());
5151
_cmdMap.put(ServerParams.CHANGE_COLOR, new VisServerCommands.ChangeColor());
5252
_cmdMap.put(ServerParams.DELETE, new VisServerCommands.DeletePlot());

‎src/firefly/java/edu/caltech/ipac/firefly/server/cache/EhcacheProvider.java‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ public class EhcacheProvider implements Cache.Provider {
4040
private static long curConfModTime = 0;
4141

4242
static {
43+
4344
URL url = null;
4445
File f = getConfFile("ehcache.xml");
4546
if (f != null && f.canRead()) {
@@ -65,7 +66,22 @@ public class EhcacheProvider implements Cache.Provider {
6566
File ignoreSizeOf = getConfFile("ignore_sizeof.txt");
6667
System.setProperty("net.sf.ehcache.sizeof.filter", ignoreSizeOf.getAbsolutePath());
6768

68-
sharedManager = CacheManager.create(sharedConfig.getAbsolutePath());
69+
// Two 2 tries to start cache manager:
70+
// 1. The first time will only work in the single app deployment such the firefly standalone version.
71+
// Ehcahce is not deployed in the tomcat lib directory.
72+
//
73+
// 2. In the typical multi app production case there will be an exception, because ehcache is in the tomcat lib
74+
// directory and has a different class loader. Then the cache manager will start without
75+
// the sizeofEngine override. To use the sizeofEngine wrapper in multi app production case it would need to
76+
// be a jar that is placed in the tomcat lib directory alone with EHcache.
77+
try {
78+
String sizeEngName= ObjectSizeEngineWrapper.class.getName();
79+
System.setProperty("net.sf.ehcache.sizeofengine.shared.VIS_SHARED_MEM", sizeEngName);
80+
sharedManager = CacheManager.create(sharedConfig.getAbsolutePath());
81+
} catch (RuntimeException e) {
82+
System.clearProperty("net.sf.ehcache.sizeofengine.shared.VIS_SHARED_MEM");
83+
sharedManager = CacheManager.create(sharedConfig.getAbsolutePath());
84+
}
6985
float pctVisSharedMemSize = AppProperties.getFloatProperty("pct.vis.shared.mem.size", 0F);
7086

7187
// check to see if vis.shared.mem.size is setup in the environment or setup for auto-config.

‎src/firefly/java/edu/caltech/ipac/firefly/server/cache/ObjectSizeEngineWrapper.java‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
/*
2+
* License information at https://github.com/Caltech-IPAC/firefly/blob/master/License.txt
3+
*/
4+
5+
package edu.caltech.ipac.firefly.server.cache;
6+
7+
8+
import net.sf.ehcache.pool.Size;
9+
import net.sf.ehcache.pool.SizeOfEngine;
10+
import net.sf.ehcache.pool.impl.DefaultSizeOfEngine;
11+
12+
/**
13+
* @author Trey Roby
14+
*/
15+
public class ObjectSizeEngineWrapper extends DefaultSizeOfEngine {
16+
17+
public ObjectSizeEngineWrapper() {
18+
super(1000, false);
19+
}
20+
public ObjectSizeEngineWrapper(int maxDepth, boolean abortWhenMaxDepthExceeded) {
21+
super(maxDepth, abortWhenMaxDepthExceeded);
22+
}
23+
24+
public ObjectSizeEngineWrapper(int maxDepth, boolean abortWhenMaxDepthExceeded, boolean silent) {
25+
super(maxDepth, abortWhenMaxDepthExceeded, silent);
26+
}
27+
28+
@Override
29+
public SizeOfEngine copyWith(int maxDepth, boolean abortWhenMaxDepthExceeded) {
30+
// return super.copyWith(maxDepth, abortWhenMaxDepthExceeded);
31+
return new ObjectSizeEngineWrapper(maxDepth, abortWhenMaxDepthExceeded);
32+
}
33+
34+
@Override
35+
public Size sizeOf(Object key, Object value, Object container) {
36+
return (value instanceof BluffSize) ?
37+
new Size(((BluffSize)value).getSize(), true) :
38+
super.sizeOf(key,value,container);
39+
}
40+
41+
static public class BluffSize {
42+
private long size;
43+
public BluffSize(long size) {this.size= size;}
44+
public long getSize() { return size; }
45+
}
46+
}
47+

‎src/firefly/java/edu/caltech/ipac/firefly/server/rpc/PlotServiceImpl.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,7 @@ public WebPlotResult deleteColorBand(PlotState state, Band band) {
160160
public WebPlotResult addColorBand(PlotState state,
161161
WebPlotRequest bandRequest,
162162
Band band) {
163-
return VisServerOps.addColorBand(state, bandRequest, band);
163+
return null; // no longer used
164164
}
165165

166166

‎src/firefly/java/edu/caltech/ipac/firefly/server/rpc/VisServerCommands.java‎

Lines changed: 26 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,32 @@ public String doCommand(Map<String, String[]> paramMap) throws IllegalArgumentEx
200200
}
201201
}
202202

203+
204+
public static class GetBetaCmd extends ServCommand {
205+
206+
public String doCommand(Map<String, String[]> paramMap) throws IllegalArgumentException {
207+
208+
SrvParam sp= new SrvParam(paramMap);
209+
PlotState state= sp.getState();
210+
double resultAry[]= VisServerOps.getBeta(state);
211+
212+
JSONObject obj= new JSONObject();
213+
obj.put("success", true);
214+
215+
JSONArray data= new JSONArray();
216+
for(double r : resultAry) data.add(r);
217+
218+
JSONArray wrapperAry= new JSONArray();
219+
obj.put("data", data);
220+
wrapperAry.add(obj);
221+
222+
return wrapperAry.toJSONString();
223+
}
224+
}
225+
226+
227+
228+
203229
public static class StretchCmd extends ServCommand {
204230

205231
public String doCommand(Map<String, String[]> paramMap) throws IllegalArgumentException {
@@ -232,19 +258,6 @@ public String doCommand(Map<String, String[]> paramMap) throws IllegalArgumentEx
232258
}
233259
}
234260

235-
public static class AddBandCmd extends ServCommand {
236-
237-
public String doCommand(Map<String, String[]> paramMap) throws IllegalArgumentException {
238-
239-
SrvParam sp= new SrvParam(paramMap);
240-
PlotState state= sp.getState();
241-
Band band = Band.parse(sp.getRequired(ServerParams.BAND));
242-
WebPlotRequest req= WebPlotRequest.parse(sp.getRequired(ServerParams.REQUEST));
243-
WebPlotResult result = VisServerOps.addColorBand(state, req, band);
244-
return WebPlotResultSerializer.createJson(result, sp.isJsonDeep());
245-
}
246-
}
247-
248261
public static class RemoveBandCmd extends ServCommand {
249262

250263
public String doCommand(Map<String, String[]> paramMap) throws IllegalArgumentException {

‎src/firefly/java/edu/caltech/ipac/firefly/server/visualize/CtxControl.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,6 @@ private static ActiveCallCtx revalidatePlot(PlotClientCtx ctx) {
112112
state.getColorTableId(), rv);
113113
plot.getPlotGroup().setZoomTo(state.getZoomLevel());
114114
if (state.isThreeColor()) plot.setThreeColorBand(fr[imageIdx],band,frGroup);
115-
ctx.setPlot(plot);
116115
first= false;
117116
}
118117
else {
@@ -217,7 +216,10 @@ public static PlotClientCtx getPlotCtx(String ctxStr) {
217216
return (PlotClientCtx) getCache().get(new StringKey(ctxStr));
218217
}
219218

220-
public static boolean isCtxAvailable(String ctxStr) { return getPlotCtx(ctxStr)!=null; }
219+
public static boolean isCtxAvailable(String ctxStr) {
220+
PlotClientCtx ctx= getPlotCtx(ctxStr);
221+
return (ctx!=null && ctx.getCachedPlot()!=null);
222+
}
221223

222224
public static void putPlotCtx(PlotClientCtx ctx) {
223225
if (ctx!=null && ctx.getKey()!=null) {

‎src/firefly/java/edu/caltech/ipac/firefly/server/visualize/FitsCacher.java‎

Lines changed: 31 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
package edu.caltech.ipac.firefly.server.visualize;
66

77
import edu.caltech.ipac.firefly.server.ServerContext;
8+
import edu.caltech.ipac.firefly.server.cache.ObjectSizeEngineWrapper;
89
import edu.caltech.ipac.firefly.server.util.Logger;
910
import edu.caltech.ipac.util.FileUtil;
1011
import edu.caltech.ipac.util.UTCTimeUtil;
@@ -32,10 +33,14 @@ public class FitsCacher {
3233
private static final Logger.LoggerImpl _log = Logger.getLogger();
3334

3435
static FitsRead[] readFits(File fitsFile) throws FitsException, FailedRequestException, IOException {
36+
return readFits(fitsFile,true, false);
37+
}
38+
39+
static FitsRead[] readFits(File fitsFile, boolean useCache, boolean clearHdu) throws FitsException, FailedRequestException, IOException {
3540
CacheKey key= new StringKey(fitsFile.getPath());
36-
FitsRead frAry[];
41+
FitsRead frAry[]= null;
3742

38-
frAry= getFromCache(key);
43+
if (useCache) frAry= getFromCache(key);
3944
if (frAry!=null) { // check first with out any locking
4045
return frAry;
4146
}
@@ -59,9 +64,13 @@ static FitsRead[] readFits(File fitsFile) throws FitsException, FailedRequestExc
5964
else {
6065
Fits fits= null;
6166
try {
67+
if (memCache != null) {
68+
memCache.put(key, new ObjectSizeEngineWrapper.BluffSize(fitsFile.length()));
69+
memCache.put(key, null);
70+
}
6271
fits= new Fits(fitsFile.getPath());
6372
long start = System.currentTimeMillis();
64-
frAry = FitsRead.createFitsReadArray(fits);
73+
frAry = FitsRead.createFitsReadArray(fits, clearHdu);
6574
if (memCache != null) memCache.put(key, frAry);
6675
long elapse = System.currentTimeMillis() - start;
6776
String timeStr = UTCTimeUtil.getHMSFromMills(elapse);
@@ -116,6 +125,25 @@ static FitsRead[] loadFits(Fits fits, File cachePath) throws FitsException, Fail
116125
}
117126
}
118127

128+
129+
130+
static void clearCachedHDU(File fitsFile) {
131+
CacheKey key= new StringKey(fitsFile.getPath());
132+
FitsRead frAry[]= getFromCache(key);
133+
if (frAry!=null) {
134+
boolean needsReinsert= false;
135+
for (FitsRead fr : frAry) {
136+
if (fr!=null && fr.hasHdu()) {
137+
fr.clearHDU();
138+
needsReinsert= true;
139+
}
140+
}
141+
if (memCache != null && needsReinsert) memCache.put(key, frAry);
142+
}
143+
}
144+
145+
146+
119147
public static void addFitsReadToCache(File fitsFile, FitsRead frAry[]) {
120148
addFitsReadToCache(fitsFile.getPath(), frAry);
121149
}

‎src/firefly/java/edu/caltech/ipac/firefly/server/visualize/ImagePlotCreator.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,7 @@ public static ModFileWriter createBand(PlotState state,
197197
Band band= readInfo.getBand();
198198

199199

200+
FitsCacher.clearCachedHDU(readInfo.getOriginalFile());
200201
plot.setThreeColorBand(state.isBandVisible(readInfo.getBand()) ? readInfo.getFitsRead() :null,
201202
readInfo.getBand(),frGroup);
202203
HistogramOps histOps= plot.getHistogramOps(band,frGroup);
@@ -320,7 +321,8 @@ public static WebFitsData makeWebFitsData(ImagePlot plot, ActiveFitsReadGroup fr
320321
long fileLength= (f!=null && f.canRead()) ? f.length() : 0;
321322
HistogramOps ops= plot.getHistogramOps(band,frGroup);
322323
Histogram hist= ops.getDataHistogram();
323-
return new WebFitsData( ops.getDataMin(hist), ops.getDataMax(hist),ops.getBeta(),
324+
return new WebFitsData( ops.getDataMin(hist), ops.getDataMax(hist),
325+
Double.NaN /*no longer pass beta by default*/,
324326
fileLength, plot.getFluxUnits(band,frGroup));
325327
}
326328
}

0 commit comments

Comments
 (0)