@Test
public void testMerlinDataLoaderU32() throws Exception {
final String path = testFileFolder + "I13-20150210-101459-1.mib";
IDataHolder dataHolder = LoaderFactory.getData(path, null);
IDataset data = dataHolder.getDataset(0);
int[] shape = data.getShape();
assertEquals(515, shape[0], 0.0);
assertEquals(515, shape[1], 0.0);
}
@Test
public void testMerlinDataLoader() throws Exception {
final String path = testFileFolder + "default1.mib";
IDataHolder dataHolder = LoaderFactory.getData(path, null);
IDataset data = dataHolder.getDataset(0);
int[] shape = data.getShape();
assertEquals(512, shape[0], 0.0);
assertEquals(512, shape[1], 0.0);
assertEquals(4095, data.max().intValue(), 0.0);
}
@Test
public void loadLoaderFactory() throws Exception {
IDataHolder dataHolder = LoaderFactory.getData(testFileFolder + file, null);
IDataset data = dataHolder.getDataset("Portable Grey Map");
// Check the first data point
assertEquals(data.getDouble(0, 0), 0.0, 0.0);
// Check the middle data point
assertEquals(data.getDouble(512, 511), 15104.0, 0.0);
// Check the last data point
assertEquals(data.getDouble(1023, 1023), 0.0, 0.0);
}
public double absorptionCorrectionCommon(boolean isSampleScatterer, boolean isSampleAttenuator) {
// Make the two components as ComponentGeometries
XPDFComponentGeometry sample = new XPDFComponentCylinder();
XPDFComponentGeometry capillary = new XPDFComponentCylinder();
sample.setDistances(0.0, 0.15);
sample.setStreamality(true, true);
capillary.setDistances(0.15, 0.16);
capillary.setStreamality(true, true);
XPDFComponentGeometry scatterer = (isSampleScatterer) ? sample : capillary;
XPDFComponentGeometry attenuator = (isSampleAttenuator) ? sample : capillary;
// These will be calculated by the Composition class
double quartzAttenuationCoefficient = 0.0529783195963;
double ceriaAttenuationCoefficient = 1.86349597825;
double attenuationCoefficient =
(isSampleAttenuator) ? ceriaAttenuationCoefficient : quartzAttenuationCoefficient;
XPDFBeamData beamData = new XPDFBeamData();
beamData.setBeamEnergy(76.6);
beamData.setBeamHeight(0.07);
beamData.setBeamWidth(0.07);
// Read the target data, and also the angles (in radians) to run
String dataPath = "/home/rkl37156/ceria_dean_data/";
IDataHolder dh = null;
String scattererName, attenuatorName;
scattererName = (isSampleScatterer) ? "sample" : "container";
attenuatorName = (isSampleAttenuator) ? "sample" : "container";
try {
dh = LoaderFactory.getData(dataPath + scattererName + "_" + attenuatorName + "_abs.xy");
} catch (Exception e) {
}
Dataset delta1D = DatasetUtils.convertToDataset(dh.getLazyDataset("Column_1").getSlice());
Dataset delta = new DoubleDataset(delta1D.getSize(), 1);
for (int i = 0; i < delta1D.getSize(); i++) delta.set(delta1D.getDouble(i), i, 0);
Dataset gamma = DoubleDataset.zeros(delta);
Dataset expected = DatasetUtils.convertToDataset(dh.getLazyDataset("Column_2").getSlice());
expected.setShape(expected.getSize(), 1);
Dataset calculated =
scatterer.calculateAbsorptionCorrections(
gamma, delta, attenuator, attenuationCoefficient, beamData, true, true);
// calculated.squeeze();
Dataset difference = Maths.subtract(calculated, expected);
return Math.sqrt((Double) Maths.square(difference).mean());
}
@Test
public void lazyLoadLoaderFactory() throws Exception {
IDataHolder dataHolder = LoaderFactory.getData(testFileFolder + file, true, true, true, null);
ILazyDataset lazy = dataHolder.getLazyDataset("Portable Grey Map");
assertArrayEquals(new int[] {1024, 1024}, lazy.getShape());
IDataset data = lazy.getSlice();
// Check the first data point
assertEquals(data.getDouble(0, 0), 0.0, 0.0);
// Check the middle data point
assertEquals(data.getDouble(512, 511), 15104.0, 0.0);
// Check the last data point
assertEquals(data.getDouble(1023, 1023), 0.0, 0.0);
}
private void updateSliceWidget(String path) {
try {
IDataHolder dh = ServiceHolder.getLoaderService().getData(path, new IMonitor.Stub());
ILazyDataset lazy = dh.getLazyDataset(fileManager.getContext().getDatasetNames().get(0));
int[] shape = lazy.getShape();
int[] dd = Slicer.getDataDimensions(shape, fileManager.getContext().getSliceDimensions());
Slice[] slices =
Slicer.getSliceArrayFromSliceDimensions(
fileManager.getContext().getSliceDimensions(), shape);
csw.setDatasetShapeInformation(shape, dd.clone(), slices);
String ss = Slice.createString(csw.getCurrentSlice());
currentSliceLabel.setText("Current slice of data: [" + ss + "]");
currentSliceLabel.getParent().layout(true);
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
private void processClientLocalUpdate() {
DataBean bean = new DataBean();
int gridDim = (int) Math.ceil(Math.sqrt(filesToLoad.size()));
if (imageGrid != null) imageGrid.dispose();
imageGrid = new PlotServerSWTImageGrid(gridDim, gridDim, canvas, plotViewName);
imageGrid.setThumbnailSize(getPreferenceImageSize());
Iterator<String> iter = filesToLoad.iterator();
while (iter.hasNext()) {
String filename = iter.next();
try {
IDataHolder holder = LoaderFactory.getData(filename);
IDataset data = holder.getDataset(0);
List<DatasetWithAxisInformation> datalist = new ArrayList<DatasetWithAxisInformation>();
DatasetWithAxisInformation d = new DatasetWithAxisInformation();
if (data != null) {
d.setData(data);
datalist.add(d);
bean.setData(datalist);
}
} catch (Exception e) {
logger.error("Error loading data with filename " + filename, e);
}
bean.putGuiParameter(GuiParameters.FILENAME, filename);
processNewFile(bean);
}
if (liveActive) {
sldProgress
.getDisplay()
.asyncExec(
new Runnable() {
@Override
public void run() {
playback.moveToLast();
}
});
}
}
@BeforeClass
public static void buildData() throws Exception {
file = folder.newFile("file1.nxs");
MapNexusFileBuilderUtils.makeGridScanWithSum(file.getAbsolutePath());
IDataHolder data = LoaderFactory.getData(file.getAbsolutePath());
ILazyDataset lazyDataset = data.getLazyDataset(MapNexusFileBuilderUtils.DETECTOR_PATH);
MapScanDimensions msd = new MapScanDimensions(1, 0, 2);
gridScanBlock =
new MappedDataBlock(
MapNexusFileBuilderUtils.DETECTOR_PATH, lazyDataset, file.getAbsolutePath(), msd);
ILazyDataset sum = data.getLazyDataset(MapNexusFileBuilderUtils.SUM_PATH);
gridScanMap =
new MappedData(
MapNexusFileBuilderUtils.SUM_PATH,
sum.getSlice(),
gridScanBlock,
file.getAbsolutePath());
fileRemap = folder.newFile("file2.nxs");
MapNexusFileBuilderUtils.makeDiagLineScanWithSum(fileRemap.getAbsolutePath());
}
@Override
protected IStatus run(IProgressMonitor monitor) {
output.setEnabled(false);
EscapableSliceVisitor sliceVisitor = null;
try {
if (path == null) path = context.getFilePaths().get(0);
final IDataHolder dh = ServiceHolder.getLoaderService().getData(path, new IMonitor.Stub());
ILazyDataset lazyDataset = dh.getLazyDataset(context.getDatasetNames().get(0));
if (lazyDataset == null) {
logger.error("Selected dataset not in file!!!!");
return Status.CANCEL_STATUS;
}
// take a local view
lazyDataset = lazyDataset.getSliceView();
Map<Integer, String> axesNames = context.getAxesNames();
if (axesNames != null) {
AxesMetadata am =
ServiceHolder.getLoaderService().getAxesMetadata(lazyDataset, path, axesNames);
lazyDataset.setMetadata(am);
// AxesMetadata axMeta = SlicedDataUtils.createAxisMetadata(path, lazyDataset,
// axesNames);
// if (axMeta != null) lazyDataset.setMetadata(axMeta);
// else lazyDataset.clearMetadata(AxesMetadata.class);
}
int[] dataDims =
Slicer.getDataDimensions(lazyDataset.getShape(), context.getSliceDimensions());
Slice[] s = csw.getCurrentSlice();
// Plot input, probably a bit wasteful to do each time
IDataset firstSlice = null;
if (lazyDataset instanceof IDataset) {
firstSlice = ((IDataset) lazyDataset).getSliceView(s);
} else {
firstSlice = lazyDataset.getSlice(s);
}
eventManager.sendInitialDataUpdate(firstSlice.getSliceView().squeeze());
MetadataPlotUtils.plotDataWithMetadata(firstSlice, input);
IOperation<? extends IOperationModel, ? extends OperationData>[] ops = getOperations();
if (ops == null) {
output.clear();
return Status.OK_STATUS;
}
// Only run what is necessary
if (inputData != null) {
if (inputData.getCurrentOperation() != end) {
int pos = 0;
for (int i = 0; i < ops.length; i++) {
if (ops[i] == end) break;
if (ops[i] == inputData.getCurrentOperation()) {
pos = i;
pos++;
break;
}
}
if (pos != 0) {
firstSlice = inputData.getInputData();
ops = Arrays.copyOfRange(ops, pos - 1, ops.length);
}
} else {
firstSlice = inputData.getInputData();
ops = new IOperation[] {inputData.getCurrentOperation()};
}
}
SourceInformation si =
new SourceInformation(path, context.getDatasetNames().get(0), lazyDataset);
SliceInformation sli = csw.getCurrentSliceInformation();
// TODO replace with check shape
firstSlice.setMetadata(new SliceFromSeriesMetadata(si, sli));
// OriginMetadataImpl om = new OriginMetadataImpl(lazyDataset, viewSlice, dataDims, path,
// context.getDatasetNames().get(0));
// om.setCurrentSlice(csw.getCurrentSlice());
// lazyDataset.setMetadata(om);
sliceVisitor =
getSliceVisitor(
ops,
lazyDataset,
Slicer.getDataDimensions(lazyDataset.getShape(), context.getSliceDimensions()));
sliceVisitor.setEndOperation(end);
long start = System.currentTimeMillis();
sliceVisitor.visit(firstSlice);
inputData = sliceVisitor.getOperationInputData();
eventManager.sendInputDataUpdate(inputData);
logger.debug("Ran in: " + (System.currentTimeMillis() - start) / 1000. + " s");
eventManager.sendErrorUpdate(null);
} catch (OperationException e) {
logger.error(e.getMessage(), e);
eventManager.sendErrorUpdate(e);
return Status.CANCEL_STATUS;
} catch (Exception e) {
logger.error(e.getMessage(), e);
String message = e.getMessage();
if (message == null) message = "Unexpected error!";
eventManager.sendErrorUpdate(new OperationException(null, message));
return Status.CANCEL_STATUS;
} finally {
if (sliceVisitor != null) {
inputData = sliceVisitor.getOperationInputData();
eventManager.sendInputDataUpdate(inputData);
}
output.setEnabled(true);
}
return Status.OK_STATUS;
}
@SuppressWarnings("unchecked")
@Override
public void update(Object source, Object changeCode) {
if (stopLoading) return;
if (source == ImageExplorerView.FOLDER_UPDATE_MARKER) { // Folder Update
if (changeCode instanceof List<?>) {
playback.clearPlayback();
btnPlay
.getDisplay()
.asyncExec(
new Runnable() {
@Override
public void run() {
resetPlaying(false);
if (isLive) {
playback.setDelay(50);
playback.setStepping(1);
playback.start();
btnPlay.setSelection(true);
btnPlay.setImage(imgStill);
execSvc.execute(playback);
if (!monActive)
CommandExecutor.executeCommand(
getViewSite(),
"uk.ac.diamond.scisoft.analysis.rcp.MontorDirectoryAction");
cmbDirectoryLocation.setText(currentDir);
isLive = false;
}
}
});
filesToLoad = (List<String>) changeCode;
processClientLocalUpdate();
}
} else {
if (source instanceof RMIPlotServer && changeCode instanceof String) {
String viewName = (String) changeCode;
if (!viewName.startsWith("Image Explorer")) return;
RMIPlotServer server = (RMIPlotServer) source;
try {
String[] names = server.getGuiNames();
final DataBean bean = server.getData(names[0]);
// when imagegrid size is set it means we need to reset the grid
if (bean != null && bean.getGuiParameters().get(GuiParameters.IMAGEGRIDSIZE) != null) {
Display.getDefault()
.syncExec(
new Runnable() {
@Override
public void run() {
Integer[] gridDim =
(Integer[]) bean.getGuiParameters().get(GuiParameters.IMAGEGRIDSIZE);
if (imageGrid != null) imageGrid.dispose();
imageGrid =
new PlotServerSWTImageGrid(
gridDim[0], gridDim[1], canvas, plotViewName);
imageGrid.setThumbnailSize(getPreferenceImageSize());
}
});
}
if (bean != null) {
if (bean.getData().isEmpty()) {
// We try to load the data from the filename if data not in the databean
String filename = (String) bean.getGuiParameters().get(GuiParameters.FILENAME);
if (filename == null) return;
List<DatasetWithAxisInformation> data = new ArrayList<>();
DatasetWithAxisInformation dwai = new DatasetWithAxisInformation();
IDataHolder holder = LoaderFactory.getData(filename);
IDataset loadedData = holder.getDataset(0);
dwai.setData(loadedData);
data.add(dwai);
bean.setData(data);
}
processUpdate(bean);
} else {
throw new Exception("Databean is null. This image grid use is not supported.");
}
} catch (Exception e) {
logger.error("Error updating data from server: ", e);
}
}
}
}
@Test
public void testLoaderFactory() throws Exception {
IDataHolder dh = LoaderFactory.getData(testfile1, null);
if (dh == null || dh.getNames().length < 1) throw new Exception();
}
/**
* In order to implement a file watcher, we loop forever ensuring requesting to take the next
* item from the file watchers queue.
*/
@Override
public void run() {
final Path path = Paths.get(spath);
try {
// We wait until the file we are told to monitor exists.
while (!Files.exists(path)) {
Thread.sleep(200);
}
// get the first event before looping
WatchKey key = null;
while (session.isOpen() && connected && (key = watcher.take()) != null) {
try {
if (!Files.exists(path)) continue;
for (WatchEvent<?> event : key.pollEvents()) {
if (!Files.exists(path)) continue;
Path epath = (Path) event.context();
if (!Files.isDirectory(path) && !path.endsWith(epath)) continue;
try {
// Data has changed, read its shape and publish the event using a web socket.
final IDataHolder holder =
ServiceHolder.getLoaderService().getData(spath, new IMonitor.Stub());
if (holder == null) continue; // We do not stop if the loader got nothing.
final ILazyDataset lz =
sdataset != null && !"".equals(sdataset)
? holder.getLazyDataset(sdataset)
: holder.getLazyDataset(0);
if (lz == null) continue; // We do not stop if the loader got nothing.
if (lz instanceof IDynamicDataset) {
((IDynamicDataset) lz).refreshShape();
}
if (writing) {
ServiceHolder.getLoaderService().clearSoftReferenceCache(spath);
}
final DataEvent evt = new DataEvent(lz.getName(), lz.getShape());
evt.setFilePath(spath);
// We manually JSON the object because we
// do not want a dependency and object simple
String json = evt.encode();
session.getRemote().sendString(json);
if (diagInfo != null) diagInfo.record("JSON Send", json);
} catch (HDF5FunctionArgumentException h5error) {
// This happens sometimes when the file is not ready to read.
logger.trace("Path might not be ready to read " + path);
continue;
} catch (Exception ne) {
logger.error("Exception getting data from " + path);
continue;
}
break;
}
} finally {
key.reset();
}
}
} catch (Exception e) {
logger.error("Exception monitoring " + path, e);
if (session.isOpen()) session.close(403, e.getMessage());
} finally {
if (diagInfo != null) diagInfo.record("Close Thread", Thread.currentThread().getName());
try {
watcher.close();
} catch (IOException e) {
logger.error("Error closing watcher", e);
}
}
}
public static void handleException(
final ScanFileHolderException e,
final IDataHolder dh,
final String fileName,
final double maxVal,
final boolean unsigned,
final int numBits)
throws ScanFileHolderException, UnsupportedOperationException, IllegalArgumentException {
/*
* Need massive error handling here, since we get same type of
* exception with different messages.
* The exception throwing might change as time passes, the
* current handling is determined on 2014-03-04.
* "Unable" means we can not recover, the image could not be
* created. Currently this can not happen, instead we get an
* exception with "Error" in its message. See below.
* "ScaledSaver" means not allowed parameters, that is use either 32 bits or float,
* or <=16 bits with big enough max and not unsigned if there is any negative value.
* "No writer" means the writer could not write the image (for
* example bmp writer can save only RGBDataset), in this case
* converting the dataset might help.
* "Error" means other kind of problem, probably not recoverable.
*/
do {
if (StringUtils.matchStringWithPattern(e.getMessage(), ".*Unable.*", true)) break;
final StringBuffer sb = new StringBuffer();
if (StringUtils.matchStringWithPattern(e.getMessage(), ".*No writer.*", true)) {
sb.append(e.getLocalizedMessage());
sb.append("\n\nSuggestion: ");
sb.append(
"To save the image as "
+ FileUtils.getFileExtension(fileName)
+ " type, you can try to ");
if (dh.getDataset(0) instanceof RGBDataset)
sb.append("convert the RGB dataset to integer dataset.");
else sb.append("convert the dataset to RGB dataset.");
throw new IllegalArgumentException(sb.toString(), e.getCause());
}
if (StringUtils.matchStringWithPattern(e.getMessage(), ".*ScaledSaver.*", true)) {
if (numBits > 16) // Currently impossible case
break;
sb.append(e.getLocalizedMessage());
sb.append("\n\nSuggestion: ");
sb.append(
"To save the image as "
+ FileUtils.getFileExtension(fileName)
+ " type, you can try to any of these possibilites:\n");
sb.append("- Choose 32 bits or float type.\n");
double maxTotalValue = maxVal;
boolean addedSignedHint = !unsigned;
for (int i = 0; i < dh.size() && !addedSignedHint && maxVal > 0; i++) {
final IDataset data = dh.getDataset(i);
if (maxVal > 0) maxTotalValue = Math.max(data.max().doubleValue(), maxTotalValue);
if (!addedSignedHint && (unsigned && data.min().doubleValue() < 0)) {
sb.append("- Choose signed values.\n");
addedSignedHint = true;
}
}
if (maxTotalValue > maxVal)
sb.append("- Choose max value >= max value of image (" + maxTotalValue + ").\n");
sb.append("- Choose autoscale option.");
throw new IllegalArgumentException(sb.toString(), e.getCause());
}
if (StringUtils.matchStringWithPattern(e.getMessage(), ".*Error.*", true)) break;
throw new UnsupportedOperationException(
"Unexpected error: " + e.getLocalizedMessage(), e.getCause());
} while (false);
throw e;
}
private void loadDataset() throws Exception {
IDataHolder dh = LoaderFactory.getData("files/brain.h5");
dataset = dh.getLazyDataset("/entry/edf/data").getSlice();
}