@Test
public void loadLoaderFactory() throws Exception {
IDataHolder dataHolder = LoaderFactory.getData(testFileFolder + file, null);
IDataset data = dataHolder.getDataset("Portable Grey Map");
// Check the first data point
assertEquals(data.getDouble(0, 0), 0.0, 0.0);
// Check the middle data point
assertEquals(data.getDouble(512, 511), 15104.0, 0.0);
// Check the last data point
assertEquals(data.getDouble(1023, 1023), 0.0, 0.0);
}
public static int[] getCommonRangeIndicies(IDataset x1, IDataset x2) {
// TODO checks for no overlap etc
double max1 = x1.max().doubleValue();
double min1 = x1.min().doubleValue();
double max2 = x2.max().doubleValue();
double min2 = x2.min().doubleValue();
double max = max1 < max2 ? max1 : max2;
double min = min1 > min2 ? min1 : min2;
// TODO max needs to be 1 lower, min needs to be 1 higher
int maxpos = ROISliceUtils.findPositionOfClosestValueInAxis(x1, max);
int minpos = ROISliceUtils.findPositionOfClosestValueInAxis(x1, min);
return new int[] {minpos + 1, maxpos - 1};
}
@Test
public void lazyLoadLoaderFactory() throws Exception {
IDataHolder dataHolder = LoaderFactory.getData(testFileFolder + file, true, true, true, null);
ILazyDataset lazy = dataHolder.getLazyDataset("Portable Grey Map");
assertArrayEquals(new int[] {1024, 1024}, lazy.getShape());
IDataset data = lazy.getSlice();
// Check the first data point
assertEquals(data.getDouble(0, 0), 0.0, 0.0);
// Check the middle data point
assertEquals(data.getDouble(512, 511), 15104.0, 0.0);
// Check the last data point
assertEquals(data.getDouble(1023, 1023), 0.0, 0.0);
}
@Override
public double calculateSignificance(int position, int windowSize, IDataset yData) {
double posVal = yData.getDouble(position);
// Calculate the averages of the the left & right windows.
// N.B. left & right diffs are in opposite directions.
Double leftAvg = 0.0;
Double rightAvg = 0.0;
for (int i = 0; i < windowSize; i++) {
leftAvg = leftAvg + yData.getDouble(position - i - 1);
rightAvg = rightAvg + yData.getDouble(position + i + 1);
}
leftAvg = leftAvg / windowSize;
rightAvg = rightAvg / windowSize;
// Calculate the average of difference of the point and the averages (i.e. significance)
double sig = ((posVal - leftAvg) + (posVal - rightAvg)) / 2;
return sig;
}
public static IntegerDataset getIndiciesOfSorted(final IDataset d) {
Integer[] dBox = new Integer[d.getSize()];
for (int i = 0; i < dBox.length; i++) dBox[i] = i;
Arrays.sort(
dBox,
new Comparator<Integer>() {
@Override
public int compare(Integer o1, Integer o2) {
double dif = d.getDouble(o1) - d.getDouble(o2);
return dif > 0 ? (int) Math.ceil(dif) : (int) Math.floor(dif);
}
});
int[] dint = new int[d.getSize()];
for (int i = 0; i < dBox.length; i++) dint[i] = dBox[i];
return new IntegerDataset(dint, new int[] {dint.length});
}
@Override
public StringDatasetBase setSlice(final Object obj, final IndexIterator siter) {
if (obj instanceof IDataset) {
final IDataset ds = (IDataset) obj;
final int[] oshape = ds.getShape();
if (!areShapesCompatible(siter.getShape(), oshape)) {
throw new IllegalArgumentException(
String.format(
"Input dataset is not compatible with slice: %s cf %s",
Arrays.toString(oshape), Arrays.toString(siter.getShape())));
}
if (ds instanceof Dataset) {
final Dataset ads = (Dataset) ds;
final IndexIterator oiter = ads.getIterator();
while (siter.hasNext() && oiter.hasNext())
data[siter.index] = ads.getStringAbs(oiter.index); // GET_ELEMENT_WITH_CAST
} else {
final IndexIterator oiter = new PositionIterator(oshape);
final int[] pos = oiter.getPos();
while (siter.hasNext() && oiter.hasNext())
data[siter.index] = ds.getString(pos); // PRIM_TYPE
}
} else {
try {
String v = obj.toString(); // PRIM_TYPE // FROM_OBJECT
while (siter.hasNext()) data[siter.index] = v;
} catch (IllegalArgumentException e) {
throw new IllegalArgumentException("Object for setting slice is not a dataset or number");
}
}
setDirty();
return this;
}
public static IDataset interpolate(IDataset oldx, IDataset oldy, IDataset newx) {
// TODO more sanity checks on inputs
DoubleDataset dx = (DoubleDataset) DatasetUtils.cast(oldx, Dataset.FLOAT64);
DoubleDataset dy = (DoubleDataset) DatasetUtils.cast(oldy, Dataset.FLOAT64);
boolean sorted = true;
double maxtest = oldx.getDouble(0);
for (int i = 1; i < ((Dataset) oldx).count(); i++) {
if (maxtest > oldx.getDouble(i)) {
sorted = false;
break;
}
maxtest = dx.getDouble(i);
}
double[] sortedx = null;
double[] sortedy = null;
if (!sorted) {
IntegerDataset sIdx = getIndiciesOfSorted(dx);
sortedx = new double[dx.getData().length];
sortedy = new double[dy.getData().length];
for (int i = 0; i < sIdx.getSize(); i++) {
sortedx[i] = dx.getDouble(sIdx.get(i));
sortedy[i] = dy.getDouble(sIdx.get(i));
}
} else {
sortedx = dx.getData();
sortedy = dy.getData();
}
SplineInterpolator si = new SplineInterpolator();
PolynomialSplineFunction poly = si.interpolate(sortedx, sortedy);
IDataset newy = newx.clone();
newy.setName(oldy.getName() + "_interpolated");
for (int i = 0; i < ((Dataset) newx).count(); i++) {
newy.set(poly.value(newx.getDouble(i)), i);
}
return newy;
}
@Test
public void testStackedFile() throws ScanFileHolderException {
ILazyDataset image =
new TIFFImageLoader(TestFileFolder + "untitled1020.TIF").loadFile().getLazyDataset(0);
Assert.assertArrayEquals("Shape not equal", new int[] {3, 2048, 2048}, image.getShape());
IDataset d = image.getSlice(new Slice(1));
Assert.assertArrayEquals("Shape not equal", new int[] {1, 2048, 2048}, d.getShape());
Assert.assertEquals("Type is int32", Integer.class, d.elementClass());
d = image.getSlice(new Slice(1, 3), new Slice(1));
Assert.assertArrayEquals("Shape not equal", new int[] {2, 1, 2048}, d.getShape());
Assert.assertEquals("Type is int32", Integer.class, d.elementClass());
d = image.getSlice(new Slice(1, 3), new Slice(null, null, 4), new Slice(2, 25));
Assert.assertArrayEquals("Shape not equal", new int[] {2, 512, 23}, d.getShape());
Assert.assertEquals("Type is int32", Integer.class, d.elementClass());
}
@Override
protected IStatus run(IProgressMonitor monitor) {
output.setEnabled(false);
EscapableSliceVisitor sliceVisitor = null;
try {
if (path == null) path = context.getFilePaths().get(0);
final IDataHolder dh = ServiceHolder.getLoaderService().getData(path, new IMonitor.Stub());
ILazyDataset lazyDataset = dh.getLazyDataset(context.getDatasetNames().get(0));
if (lazyDataset == null) {
logger.error("Selected dataset not in file!!!!");
return Status.CANCEL_STATUS;
}
// take a local view
lazyDataset = lazyDataset.getSliceView();
Map<Integer, String> axesNames = context.getAxesNames();
if (axesNames != null) {
AxesMetadata am =
ServiceHolder.getLoaderService().getAxesMetadata(lazyDataset, path, axesNames);
lazyDataset.setMetadata(am);
// AxesMetadata axMeta = SlicedDataUtils.createAxisMetadata(path, lazyDataset,
// axesNames);
// if (axMeta != null) lazyDataset.setMetadata(axMeta);
// else lazyDataset.clearMetadata(AxesMetadata.class);
}
int[] dataDims =
Slicer.getDataDimensions(lazyDataset.getShape(), context.getSliceDimensions());
Slice[] s = csw.getCurrentSlice();
// Plot input, probably a bit wasteful to do each time
IDataset firstSlice = null;
if (lazyDataset instanceof IDataset) {
firstSlice = ((IDataset) lazyDataset).getSliceView(s);
} else {
firstSlice = lazyDataset.getSlice(s);
}
eventManager.sendInitialDataUpdate(firstSlice.getSliceView().squeeze());
MetadataPlotUtils.plotDataWithMetadata(firstSlice, input);
IOperation<? extends IOperationModel, ? extends OperationData>[] ops = getOperations();
if (ops == null) {
output.clear();
return Status.OK_STATUS;
}
// Only run what is necessary
if (inputData != null) {
if (inputData.getCurrentOperation() != end) {
int pos = 0;
for (int i = 0; i < ops.length; i++) {
if (ops[i] == end) break;
if (ops[i] == inputData.getCurrentOperation()) {
pos = i;
pos++;
break;
}
}
if (pos != 0) {
firstSlice = inputData.getInputData();
ops = Arrays.copyOfRange(ops, pos - 1, ops.length);
}
} else {
firstSlice = inputData.getInputData();
ops = new IOperation[] {inputData.getCurrentOperation()};
}
}
SourceInformation si =
new SourceInformation(path, context.getDatasetNames().get(0), lazyDataset);
SliceInformation sli = csw.getCurrentSliceInformation();
// TODO replace with check shape
firstSlice.setMetadata(new SliceFromSeriesMetadata(si, sli));
// OriginMetadataImpl om = new OriginMetadataImpl(lazyDataset, viewSlice, dataDims, path,
// context.getDatasetNames().get(0));
// om.setCurrentSlice(csw.getCurrentSlice());
// lazyDataset.setMetadata(om);
sliceVisitor =
getSliceVisitor(
ops,
lazyDataset,
Slicer.getDataDimensions(lazyDataset.getShape(), context.getSliceDimensions()));
sliceVisitor.setEndOperation(end);
long start = System.currentTimeMillis();
sliceVisitor.visit(firstSlice);
inputData = sliceVisitor.getOperationInputData();
eventManager.sendInputDataUpdate(inputData);
logger.debug("Ran in: " + (System.currentTimeMillis() - start) / 1000. + " s");
eventManager.sendErrorUpdate(null);
} catch (OperationException e) {
logger.error(e.getMessage(), e);
eventManager.sendErrorUpdate(e);
return Status.CANCEL_STATUS;
} catch (Exception e) {
logger.error(e.getMessage(), e);
String message = e.getMessage();
if (message == null) message = "Unexpected error!";
eventManager.sendErrorUpdate(new OperationException(null, message));
return Status.CANCEL_STATUS;
} finally {
if (sliceVisitor != null) {
inputData = sliceVisitor.getOperationInputData();
eventManager.sendInputDataUpdate(inputData);
}
output.setEnabled(true);
}
return Status.OK_STATUS;
}
@Override
public List<Dataset> integrate(IDataset dataset) {
// Generate radial and azimuthal look-up arrays as required
// TODO test shape of axis array
if (radialArray == null) {
generateRadialArray(dataset.getShape(), false);
}
if (azimuthalArray == null) {
generateMinMaxAzimuthalArray(
qSpace.getDetectorProperties().getBeamCentreCoords(), dataset.getShape());
}
Dataset mt = mask;
if (mask != null && !Arrays.equals(mask.getShape(), dataset.getShape()))
throw new IllegalArgumentException("Mask shape does not match dataset shape");
List<Dataset> result = new ArrayList<Dataset>();
if (binEdges == null) {
binEdges = calculateBins(radialArray, mt, radialRange, nbins);
binsChi = calculateBins(azimuthalArray, mt, azimuthalRange, nBinsChi);
}
final double[] edgesQ = binEdges.getData();
final double loQ = edgesQ[0];
final double hiQ = edgesQ[nbins];
final double spanQ = (hiQ - loQ) / nbins;
final double[] edgesChi = binsChi.getData();
final double loChi = edgesChi[0];
final double hiChi = edgesChi[nBinsChi];
final double spanChi = (hiChi - loChi) / nBinsChi;
FloatDataset histo = new FloatDataset(nBinsChi, nbins);
FloatDataset intensity = new FloatDataset(nBinsChi, nbins);
// final double[] h = histo.getData();
// final double[] in = intensity.getData();
// if (spanQ <= 0) {
// h[0] = ds.getSize();
// result.add(histo);
// result.add(bins);
// continue;
// }
Dataset a = DatasetUtils.convertToDataset(radialArray[0]);
Dataset b = DatasetUtils.convertToDataset(dataset);
// PositionIterator iter = b.getPositionIterator();
//
// int[] pos = iter.getPos();
// int[] posStop = pos.clone();
IndexIterator iter = a.getIterator();
while (iter.hasNext()) {
// posStop[0] = pos[0]+2;
// posStop[1] = pos[1]+2;
// Dataset qrange = a.getSlice(pos, posStop, null);
// Dataset chirange = azimuthalArray.getSlice(pos, posStop, null);
if (mt != null && !mt.getElementBooleanAbs(iter.index)) continue;
final double qMax = radialArray[1].getElementDoubleAbs(iter.index);
final double qMin = radialArray[0].getElementDoubleAbs(iter.index);
final double chiMax = azimuthalArray[1].getElementDoubleAbs(iter.index);
final double chiMin = azimuthalArray[0].getElementDoubleAbs(iter.index);
final double sig = b.getElementDoubleAbs(iter.index);
if (qMax < loQ || qMin > hiQ) {
continue;
}
if (chiMax < loChi || chiMin > hiChi) {
continue;
}
// losing something here? is flooring (int cast best?)
double minBinExactQ = (qMin - loQ) / spanQ;
double maxBinExactQ = (qMax - loQ) / spanQ;
int minBinQ = (int) minBinExactQ;
int maxBinQ = (int) maxBinExactQ;
double minBinExactChi = (chiMin - loChi) / spanChi;
double maxBinExactChi = (chiMax - loChi) / spanChi;
int minBinChi = (int) minBinExactChi;
int maxBinChi = (int) maxBinExactChi;
// FIXME potentially may need to deal with azimuthal arrays with discontinuities (+180/-180
// degress etc)
double iPerPixel = 1 / ((maxBinExactQ - minBinExactQ) * (maxBinExactChi - minBinExactChi));
double minFracQ = 1 - (minBinExactQ - minBinQ);
double maxFracQ = maxBinExactQ - maxBinQ;
double minFracChi = 1 - (minBinExactChi - minBinChi);
double maxFracChi = maxBinExactChi - maxBinChi;
for (int i = minBinQ; i <= maxBinQ; i++) {
if (i < 0 || i >= binEdges.getSize() - 1) continue;
for (int j = minBinChi; j <= maxBinChi; j++) {
if (j < 0 || j >= binsChi.getSize() - 1) continue;
int[] setPos = new int[] {j, i};
double val = histo.get(setPos);
double modify = 1;
if (i == minBinQ && minBinQ != maxBinQ) modify *= minFracQ;
if (i == maxBinQ && minBinQ != maxBinQ) modify *= maxFracQ;
if (i == minBinChi && minBinChi != maxBinChi) modify *= minFracChi;
if (i == maxBinChi && minBinChi != maxBinChi) modify *= maxFracChi;
histo.set(val + iPerPixel * modify, setPos);
double inVal = intensity.get(setPos);
intensity.set(inVal + sig * iPerPixel * modify, setPos);
}
}
}
processAndAddToResult(intensity, histo, result, radialRange, dataset.getName());
return result;
}
/**
* Fetch diffraction metadata, doesn't add it to the IDataset
*
* @param image
* @param altPath alternative for file path if metadata is null or does not hold it
* @param service
* @param statusText returned message (can be null)
* @return diffraction metadata
*/
public static IDiffractionMetadata getDiffractionMetadata(
IDataset image, String altPath, ILoaderService service, String[] statusText) {
// Now always returns IDiffractionMetadata to prevent creation of a new
// metadata object after listeners have been added to the old metadata
// TODO improve this section- it's pretty horrible
IDiffractionMetadata lockedMeta = service.getLockedDiffractionMetaData();
if (image == null) return lockedMeta;
int[] shape = image.getShape();
IMetadata mdImage = null;
try {
mdImage = image.getMetadata();
} catch (Exception e1) {
// do nothing
}
if (lockedMeta != null) {
if (mdImage instanceof IDiffractionMetadata) {
IDiffractionMetadata dmd = (IDiffractionMetadata) mdImage;
if (!dmd.getDiffractionCrystalEnvironment()
.equals(lockedMeta.getDiffractionCrystalEnvironment())
|| !dmd.getDetector2DProperties().equals(lockedMeta.getDetector2DProperties())) {
try {
DiffractionMetadataUtils.copyNewOverOld(lockedMeta, (IDiffractionMetadata) mdImage);
} catch (IllegalArgumentException e) {
if (statusText != null)
statusText[0] = "Locked metadata does not match image dimensions!";
}
}
} else {
// TODO what if the image is rotated?
if (shape[0] == lockedMeta.getDetector2DProperties().getPx()
&& shape[1] == lockedMeta.getDetector2DProperties().getPy()) {
} else {
IDiffractionMetadata clone = lockedMeta.clone();
clone.getDetector2DProperties().setPx(shape[0]);
clone.getDetector2DProperties().setPy(shape[1]);
if (statusText != null)
statusText[0] = "Locked metadata does not match image dimensions!";
}
}
if (statusText != null && statusText[0] == null) {
statusText[0] = "Metadata loaded from locked version";
}
return lockedMeta;
}
// If not see if the trace has diffraction meta data
if (mdImage instanceof IDiffractionMetadata) {
if (statusText != null && statusText[0] == null) {
statusText[0] = "Metadata loaded from image";
}
return (IDiffractionMetadata) mdImage;
}
// Try and get the filename here, it will help later on
String filePath = mdImage == null ? null : mdImage.getFilePath();
if (filePath == null) {
filePath = altPath;
}
if (filePath != null) {
// see if we can read diffraction info from nexus files
NexusDiffractionMetaCreator ndmc = new NexusDiffractionMetaCreator(filePath);
IDiffractionMetadata difMet = ndmc.getDiffractionMetadataFromNexus(shape);
if (difMet != null) {
// TODO comment out
// image.setMetadata(difMet);
if (statusText != null && statusText[0] == null) {
if (ndmc.isCompleteRead()) {
statusText[0] = "Metadata completely loaded from nexus tree";
return difMet;
} else if (ndmc.isPartialRead()) {
statusText[0] = "Required metadata loaded from nexus tree";
return difMet;
} else if (ndmc.anyValuesRead())
statusText[0] = "Incomplete metadata in nexus tree, loading from preferences";
else statusText[0] = "No metadata in nexus tree, metadata loaded from preferences";
}
}
}
// if it is null try and get it from the loader service
if (mdImage == null && filePath != null) {
IMetadata md = null;
try {
md = service.getMetadata(filePath, null);
} catch (Exception e) {
logger.error("Cannot read meta data from part", e);
}
// If it is there and diffraction data return it
if (md instanceof IDiffractionMetadata) {
if (statusText != null && statusText[0] == null) {
statusText[0] = "Metadata loaded from file";
}
return (IDiffractionMetadata) md;
}
}
// if there is no meta or is not nexus or IDiff create default IDiff and put it in the dataset
mdImage = DiffractionDefaultMetadata.getDiffractionMetadata(filePath, shape);
// image.setMetadata(mdImage);
if (statusText != null && statusText[0] == null) {
statusText[0] = "No metadata found. Values loaded from preferences:";
}
return (IDiffractionMetadata) mdImage;
}
