@Parameters({"target"})
@BeforeClass
public void setUp(String target) throws Exception {
log.info("METADATA VALIDATOR TARGET: " + target);
sf = new TestServiceFactory().proxy();
config = new ImportConfig();
// Let the user know at what level we're logging
ch.qos.logback.classic.Logger lociLogger =
(ch.qos.logback.classic.Logger) LoggerFactory.getLogger("loci");
ch.qos.logback.classic.Logger omeLogger =
(ch.qos.logback.classic.Logger) LoggerFactory.getLogger("ome.formats");
log.info(
String.format(
"Log levels -- Bio-Formats: %s OMERO.importer: %s",
lociLogger.getLevel(), omeLogger.getLevel()));
store = new OMEROMetadataStoreClient();
store.initialize(sf);
store.setEnumerationProvider(new TestEnumerationProvider());
store.setInstanceProvider(new BlitzInstanceProvider(store.getEnumerationProvider()));
minimalStore = new OMEROMetadataStoreClient();
minimalStore.initialize(sf);
minimalStore.setEnumerationProvider(new TestEnumerationProvider());
minimalStore.setInstanceProvider(
new BlitzInstanceProvider(minimalStore.getEnumerationProvider()));
wrapper = new OMEROWrapper(config);
wrapper.setMetadataOptions(new DefaultMetadataOptions(MetadataLevel.ALL));
minimalWrapper = new OMEROWrapper(config);
minimalWrapper.setMetadataOptions(new DefaultMetadataOptions(MetadataLevel.MINIMUM));
wrapper.setMetadataStore(store);
store.setReader(wrapper.getImageReader());
minimalStore.setReader(minimalWrapper.getImageReader());
}
@Test(dependsOnMethods = {"testMetadataLevel"})
public void testEquivilentBlockRetrievalPlaneData() throws FormatException, IOException {
String fileName = wrapper.getCurrentFile();
int sizeX = wrapper.getSizeX();
int sizeY = wrapper.getSizeY();
int sizeZ = wrapper.getSizeZ();
int sizeC = wrapper.getSizeC();
int sizeT = wrapper.getSizeT();
int bytesPerPixel = wrapper.getBitsPerPixel() / 8;
int planeSize = sizeX * sizeY * bytesPerPixel;
byte[] planar = new byte[planeSize];
byte[] block = new byte[planeSize];
int planeNumber = 0;
String planarDigest;
String blockDigest;
for (int t = 0; t < sizeT; t++) {
for (int c = 0; c < sizeC; c++) {
for (int z = 0; z < sizeZ; z++) {
planeNumber = wrapper.getIndex(z, c, t);
wrapper.openPlane2D(fileName, planeNumber, planar);
planarDigest = sha1(planar);
wrapper.openPlane2D(fileName, planeNumber, block, 0, 0, sizeX, sizeY);
blockDigest = sha1(block);
assertEquals(planarDigest, blockDigest);
}
}
}
}
@AfterClass
public void tearDown() throws IOException {
wrapper.close();
minimalWrapper.close();
store.logout();
minimalStore.logout();
}
/**
* Parse the binary data to generate min/max values and allow an md to be calculated.
*
* @param series
* @return The SHA1 message digest for the binary data.
*/
public MessageDigest parseData(String fileName, int series, ImportSize size)
throws FormatException, IOException, ServerError {
reader.setSeries(series);
int maxPlaneSize = sizes.getMaxPlaneWidth() * sizes.getMaxPlaneHeight();
if (((long) reader.getSizeX() * (long) reader.getSizeY()) > maxPlaneSize) {
return null;
}
int bytesPerPixel = getBytesPerPixel(reader.getPixelType());
MessageDigest md;
try {
md = MessageDigest.getInstance("SHA-1");
} catch (NoSuchAlgorithmException e) {
throw new RuntimeException("Required SHA-1 message digest algorithm unavailable.");
}
int planeNo = 1;
for (int t = 0; t < size.sizeT; t++) {
for (int c = 0; c < size.sizeC; c++) {
for (int z = 0; z < size.sizeZ; z++) {
parseDataByPlane(size, z, c, t, bytesPerPixel, fileName, md);
notifyObservers(new ImportEvent.IMPORT_STEP(planeNo, series, reader.getSeriesCount()));
planeNo++;
}
}
}
return md;
}
/** opens the file using the {@link FormatReader} instance */
private void open(OMEROWrapper reader, OMEROMetadataStoreClient store, CheckedPath targetFile)
throws FormatException, IOException {
// reader.close(); This instance is no longer re-used
reader.setMetadataStore(store);
reader.setMinMaxStore(store);
store.setReader(reader.getImageReader());
targetFile.bfSetId(reader);
// reset series count
if (log.isDebugEnabled()) {
log.debug("Image Count: " + reader.getImageCount());
}
}
@Parameters({"target"})
@Test
public void testMetadataLevelMinimumSetId(String target) throws Exception {
minimalWrapper.setId(target);
minimalStore.postProcess();
traceMetadataStoreData(minimalStore);
}
@Parameters({"target"})
@Test
public void testMetadataLevelAllSetId(String target) throws Exception {
wrapper.setId(target);
store.postProcess();
traceMetadataStoreData(store);
}
@Test(dependsOnMethods = {"testMetadataLevel"})
public void testMetadataLevelEquivilentUsedFiles() throws FormatException, IOException {
for (int i = 0; i < minimalWrapper.getSeriesCount(); i++) {
minimalWrapper.setSeries(i);
wrapper.setSeries(i);
String[] pixelsOnlyFiles = minimalWrapper.getSeriesUsedFiles();
String[] allFiles = wrapper.getSeriesUsedFiles();
assertEquals(allFiles.length, pixelsOnlyFiles.length);
Arrays.sort(allFiles);
Arrays.sort(pixelsOnlyFiles);
for (int j = 0; j < pixelsOnlyFiles.length; j++) {
assertEquals(allFiles[j], pixelsOnlyFiles[j]);
}
}
}
/**
* Read a plane and update the pixels checksum
*
* @param size Sizes of the Pixels set.
* @param z The Z-section offset to write to.
* @param c The channel offset to write to.
* @param t The timepoint offset to write to.
* @param bytesPerPixel Number of bytes per pixel.
* @param fileName Name of the file.
* @param md Current Pixels set message digest.
* @throws FormatException If there is an error reading Pixel data via Bio-Formats.
* @throws IOException If there is an I/O error reading Pixel data via Bio-Formats.
*/
private void parseDataByPlane(
ImportSize size, int z, int c, int t, int bytesPerPixel, String fileName, MessageDigest md)
throws FormatException, IOException {
int tileHeight = reader.getOptimalTileHeight();
int tileWidth = reader.getOptimalTileWidth();
int planeNumber, x, y, w, h;
for (int tileOffsetY = 0;
tileOffsetY < (size.sizeY + tileHeight - 1) / tileHeight;
tileOffsetY++) {
for (int tileOffsetX = 0;
tileOffsetX < (size.sizeX + tileWidth - 1) / tileWidth;
tileOffsetX++) {
x = tileOffsetX * tileWidth;
y = tileOffsetY * tileHeight;
w = tileWidth;
h = tileHeight;
if ((x + tileWidth) > size.sizeX) {
w = size.sizeX - x;
}
if ((y + tileHeight) > size.sizeY) {
h = size.sizeY - y;
}
int bytesToRead = w * h * bytesPerPixel;
if (arrayBuf.length != bytesToRead) {
arrayBuf = new byte[bytesToRead];
}
planeNumber = reader.getIndex(z, c, t);
if (log.isDebugEnabled()) {
log.debug(
String.format(
"Plane:%d X:%d Y:%d TileWidth:%d TileHeight:%d " + "arrayBuf.length:%d",
planeNumber, x, y, w, h, arrayBuf.length));
}
arrayBuf = reader.openBytes(planeNumber, arrayBuf, x, y, w, h);
try {
md.update(arrayBuf);
} catch (Exception e) {
// This better not happen. :)
throw new RuntimeException(e);
}
}
}
}
private void cleanupReader() {
try {
if (reader != null) {
try {
reader.close();
} finally {
reader = null;
}
}
} catch (Throwable e) {
log.error("Failed on cleanupReader", e);
}
}
public Object pixelData(PixelDataJob pdj) throws Throwable {
if (!reader.isMinMaxSet() && !noStatsInfo) {
// Parse the binary data to generate min/max values
int seriesCount = reader.getSeriesCount();
for (int series = 0; series < seriesCount; series++) {
ImportSize size = new ImportSize(fileName, pixList.get(series), reader.getDimensionOrder());
Pixels pixels = pixList.get(series);
MessageDigest md = parseData(fileName, series, size);
if (md != null) {
final String s = Hex.encodeHexString(md.digest());
pixels.setSha1(store.toRType(s));
}
}
}
// As we're in metadata-only mode on we need to
// tell the server which Image matches which series.
int series = 0;
for (final Pixels pixels : pixList) {
store.setPixelsFile(pixels.getId().getValue(), fileName, repoUuid);
pixels.getImage().setSeries(store.toRType(series++));
}
for (final Image image : imageList) {
image.unloadAnnotationLinks();
}
store.updatePixels(pixList);
if (!reader.isMinMaxSet() && !noStatsInfo) {
store.populateMinMax();
}
return null;
}
/**
* If available, populates overlays for a given set of pixels objects.
*
* @param pixelsList Pixels objects to populate overlays for.
* @param plateIds Plate object IDs to populate overlays for.
*/
private void importOverlays(List<Pixels> pixelsList, List<Long> plateIds)
throws FormatException, IOException {
IFormatReader baseReader = reader.getImageReader().getReader();
if (baseReader instanceof MIASReader) {
try {
MIASReader miasReader = (MIASReader) baseReader;
ServiceFactoryPrx sf = store.getServiceFactory();
OverlayMetadataStore s = new OverlayMetadataStore();
s.initialize(sf, pixelsList, plateIds);
miasReader.parseMasks(s);
s.complete();
} catch (ServerError e) {
log.warn("Error while populating MIAS overlays.", e);
}
}
}
@Test(dependsOnMethods = {"testMetadataLevel"})
public void testMetadataLevelEquivilentPlaneData() throws FormatException, IOException {
for (int i = 0; i < minimalWrapper.getSeriesCount(); i++) {
minimalWrapper.setSeries(i);
wrapper.setSeries(i);
assertEquals(wrapper.getImageCount(), minimalWrapper.getImageCount());
for (int j = 0; j < minimalWrapper.getImageCount(); j++) {
byte[] pixelsOnlyPlane = minimalWrapper.openBytes(j);
String pixelsOnlySHA1 = sha1(pixelsOnlyPlane);
byte[] allPlane = wrapper.openBytes(j);
String allSHA1 = sha1(allPlane);
if (!pixelsOnlySHA1.equals(allSHA1)) {
fail(
String.format(
"MISMATCH: Series:%d Image:%d PixelsOnly%s All:%s",
i, j, pixelsOnlySHA1, allSHA1));
}
}
}
}
public void init(Helper helper) {
this.helper = helper;
helper.setSteps(5);
final ImportConfig config = new ImportConfig();
final String sessionUuid = helper.getEventContext().getCurrentSessionUuid();
if (!(location instanceof ManagedImportLocationI)) {
throw helper.cancel(
new ERR(), null, "bad-location", "location-type", location.getClass().getName());
}
ManagedImportLocationI managedLocation = (ManagedImportLocationI) location;
logPath = managedLocation.getLogFile();
logFilename = logPath.getFullFsPath();
MDC.put("fileset", logFilename);
file = ((ManagedImportLocationI) location).getTarget();
try {
sf = reg.getInternalServiceFactory(sessionUuid, "unused", 3, 1, clientUuid);
store = new OMEROMetadataStoreClient();
store.setCurrentLogFile(logFilename, token);
store.initialize(sf);
registerKeepAlive();
fileName = file.getFullFsPath();
shortName = file.getName();
format = null;
usedFiles = new String[] {fileName};
open(reader, store, file);
format = reader.getFormat();
if (reader.getUsedFiles() != null) {
usedFiles = reader.getUsedFiles();
}
if (usedFiles == null) {
throw new NullPointerException("usedFiles must be non-null");
}
// Process all information which has been passed in as annotations
detectKnownAnnotations();
// Now that we've possibly updated the settings object, copy out
// all the values needed by import.
userSpecifiedTarget = settings.userSpecifiedTarget;
userSpecifiedName =
settings.userSpecifiedName == null ? null : settings.userSpecifiedName.getValue();
userSpecifiedDescription =
settings.userSpecifiedDescription == null
? null
: settings.userSpecifiedDescription.getValue();
userPixels = settings.userSpecifiedPixels;
annotationList = settings.userSpecifiedAnnotationList;
doThumbnails = settings.doThumbnails == null ? true : settings.doThumbnails.getValue();
noStatsInfo = settings.noStatsInfo == null ? false : settings.noStatsInfo.getValue();
IFormatReader baseReader = reader.getImageReader().getReader();
if (log.isInfoEnabled()) {
log.info("File format: " + format);
log.info("Base reader: " + baseReader.getClass().getName());
}
notifyObservers(new ImportEvent.LOADED_IMAGE(shortName, 0, 0, 0));
formatString = baseReader.getClass().getSimpleName();
formatString = formatString.replace("Reader", "");
} catch (Cancel c) {
throw c;
} catch (Throwable t) {
throw helper.cancel(new ERR(), t, "error-on-init");
} finally {
MDC.clear();
}
}
@Test(dependsOnMethods = {"testMetadataLevel"})
public void testMetadataLevelEquivilentDimensions() {
assertEquals(wrapper.getSeriesCount(), minimalWrapper.getSeriesCount());
for (int i = 0; i < minimalWrapper.getSeriesCount(); i++) {
wrapper.setSeries(i);
minimalWrapper.setSeries(i);
assertEquals(wrapper.getSizeX(), minimalWrapper.getSizeX());
assertEquals(wrapper.getSizeY(), minimalWrapper.getSizeY());
assertEquals(wrapper.getSizeZ(), minimalWrapper.getSizeZ());
assertEquals(wrapper.getSizeC(), minimalWrapper.getSizeC());
assertEquals(wrapper.getSizeT(), minimalWrapper.getSizeT());
assertEquals(wrapper.getPixelType(), minimalWrapper.getPixelType());
assertEquals(wrapper.isLittleEndian(), minimalWrapper.isLittleEndian());
}
}
@Test(dependsOnMethods = {"testMetadataLevelAllSetId", "testMetadataLevelMinimumSetId"})
public void testMetadataLevel() throws FormatException, IOException {
assertEquals(MetadataLevel.MINIMUM, minimalWrapper.getMetadataOptions().getMetadataLevel());
assertEquals(MetadataLevel.ALL, wrapper.getMetadataOptions().getMetadataLevel());
assertFalse(0 == (wrapper.getSeriesMetadata().size() + wrapper.getGlobalMetadata().size()));
}
/**
* NSAUTOCLOSE causes the process to end on import completion
*
* <p>NSTARGETTEMPLATE sets a target for the import <em>if none is set</em>
*/
private void detectKnownAnnotations() throws Exception {
// PythonImporter et al. may pass a null settings.
if (settings == null || settings.userSpecifiedAnnotationList == null) {
return;
}
ome.model.annotations.CommentAnnotation lastCA = null;
long maxCAId = 0;
for (Annotation a : settings.userSpecifiedAnnotationList) {
if (a == null) {
continue;
}
ome.model.annotations.Annotation ann = null;
String ns = null;
if (a.isLoaded()) {
ns = a.getNs() == null ? null : a.getNs().getValue();
ann = (ome.model.annotations.Annotation) new IceMapper().reverse(a);
} else {
if (a.getId() == null) {
// not sure what we can do with this annotation then.
continue;
}
ann =
(ome.model.annotations.Annotation)
helper
.getSession()
.get(ome.model.annotations.Annotation.class, a.getId().getValue());
ns = ann.getNs();
}
if (NSAUTOCLOSE.value.equals(ns)) {
autoClose = true;
} else if (NSTARGETTEMPLATE.value.equals(ns)) {
ome.model.annotations.CommentAnnotation ca = (ome.model.annotations.CommentAnnotation) ann;
if (ca.getId().longValue() > maxCAId) {
maxCAId = ca.getId().longValue();
lastCA = ca;
}
}
}
if (lastCA != null) {
if (settings.userSpecifiedTarget != null) {
// TODO: Exception
String kls = settings.userSpecifiedTarget.getClass().getSimpleName();
long id = settings.userSpecifiedTarget.getId().getValue();
log.error("User-specified template target '{}' AND {}:{}", lastCA.getTextValue(), kls, id);
} else {
// Path converted to unix slashes.
String path = lastCA.getDescription();
File file = new File(path);
for (int i = 0; i < location.omittedLevels; i++) {
file = file.getParentFile();
}
path = file.toString();
log.debug("Using target path {}", path);
// Here we use the client-side (but unix-separated) path, since
// for simple imports, we don't have any context about the directory
// from the client.
ServerTemplateImportTarget target = new ServerTemplateImportTarget(path);
target.init(lastCA.getTextValue());
settings.userSpecifiedTarget = target.load(store, reader.isSPWReader());
}
}
}