/**
* Convert a 3-channel {@link MultiSpectral} image from RGB into YUV.
*
* @param rgb (Input) RGB encoded image
* @param yuv (Output) YUV encoded image
*/
public static void rgbToYuv_F32(
MultiSpectral<ImageFloat32> rgb, MultiSpectral<ImageFloat32> yuv) {
InputSanityCheck.checkSameShape(yuv, rgb);
ImageFloat32 R = rgb.getBand(0);
ImageFloat32 G = rgb.getBand(1);
ImageFloat32 B = rgb.getBand(2);
ImageFloat32 Y = yuv.getBand(0);
ImageFloat32 U = yuv.getBand(1);
ImageFloat32 V = yuv.getBand(2);
for (int row = 0; row < yuv.height; row++) {
int indexYuv = yuv.startIndex + row * yuv.stride;
int indexRgb = rgb.startIndex + row * rgb.stride;
for (int col = 0; col < yuv.width; col++, indexYuv++, indexRgb++) {
float r = R.data[indexRgb];
float g = G.data[indexRgb];
float b = B.data[indexRgb];
float y = 0.299f * r + 0.587f * g + 0.114f * b;
Y.data[indexRgb] = y;
U.data[indexRgb] = 0.492f * (b - y);
V.data[indexRgb] = 0.877f * (r - y);
}
}
}
/**
* Convert a 3-channel {@link MultiSpectral} image from YUV into RGB.
*
* @param rgb (Input) RGB encoded image
* @param yuv (Output) YUV encoded image
*/
public static void yuvToRgb_F32(
MultiSpectral<ImageFloat32> yuv, MultiSpectral<ImageFloat32> rgb) {
InputSanityCheck.checkSameShape(yuv, rgb);
ImageFloat32 Y = yuv.getBand(0);
ImageFloat32 U = yuv.getBand(1);
ImageFloat32 V = yuv.getBand(2);
ImageFloat32 R = rgb.getBand(0);
ImageFloat32 G = rgb.getBand(1);
ImageFloat32 B = rgb.getBand(2);
for (int row = 0; row < yuv.height; row++) {
int indexYuv = yuv.startIndex + row * yuv.stride;
int indexRgb = rgb.startIndex + row * rgb.stride;
for (int col = 0; col < yuv.width; col++, indexYuv++, indexRgb++) {
float y = Y.data[indexYuv];
float u = U.data[indexYuv];
float v = V.data[indexYuv];
R.data[indexRgb] = y + 1.13983f * v;
G.data[indexRgb] = y - 0.39465f * u - 0.58060f * v;
B.data[indexRgb] = y + 2.032f * u;
}
}
}
@Override
public void setImage(MultiSpectral<T> image) {
gray.reshape(image.width, image.height);
grayII.reshape(image.width, image.height);
bandII.reshape(image.width, image.height);
GConvertImage.average(image, gray);
GIntegralImageOps.transform(gray, grayII);
for (int i = 0; i < image.getNumBands(); i++)
GIntegralImageOps.transform(image.getBand(i), bandII.getBand(i));
alg.setImage(grayII, bandII);
}
public static void assertEqualsRelative(ImageBase imgA, ImageBase imgB, double tolFrac) {
// if no specialized check exists, use a slower generalized approach
if (imgA instanceof ImageSingleBand) {
GImageSingleBand a = FactoryGImageSingleBand.wrap((ImageSingleBand) imgA);
GImageSingleBand b = FactoryGImageSingleBand.wrap((ImageSingleBand) imgB);
for (int y = 0; y < imgA.height; y++) {
for (int x = 0; x < imgA.width; x++) {
double valA = a.get(x, y).doubleValue();
double valB = b.get(x, y).doubleValue();
double difference = valA - valB;
double max = Math.max(Math.abs(valA), Math.abs(valB));
if (max == 0) max = 1;
if (Math.abs(difference) / max > tolFrac)
throw new RuntimeException(
"Values not equal at (" + x + "," + y + ") " + valA + " " + valB);
}
}
} else if (imgA instanceof MultiSpectral) {
MultiSpectral a = (MultiSpectral) imgA;
MultiSpectral b = (MultiSpectral) imgB;
if (a.getNumBands() != b.getNumBands())
throw new RuntimeException("Number of bands not equal");
for (int band = 0; band < a.getNumBands(); band++) {
assertEqualsRelative(a.getBand(band), b.getBand(band), tolFrac);
}
} else {
throw new RuntimeException("Unknown image type");
}
}
public void process(final BufferedImage buffLeft, final BufferedImage buffRight) {
imageLeft.reshape(buffLeft.getWidth(), buffLeft.getHeight());
imageRight.reshape(buffRight.getWidth(), buffRight.getHeight());
grayLeft.reshape(buffLeft.getWidth(), buffLeft.getHeight());
grayRight.reshape(buffRight.getWidth(), buffRight.getHeight());
ConvertBufferedImage.convertFromMulti(buffLeft, imageLeft, true, imageType);
ConvertBufferedImage.convertFromMulti(buffRight, imageRight, true, imageType);
SwingUtilities.invokeLater(
new Runnable() {
public void run() {
panel.setImages(buffLeft, buffRight);
processedImage = true;
doRefreshAll();
}
});
}
public static void nv21ToMultiRgb_U8(byte[] dataNV, MultiSpectral<ImageUInt8> output) {
ImageUInt8 R = output.getBand(0);
ImageUInt8 G = output.getBand(1);
ImageUInt8 B = output.getBand(2);
final int yStride = output.width;
final int uvStride = output.width / 2;
final int startUV = yStride * output.height;
for (int row = 0; row < output.height; row++) {
int indexY = row * yStride;
int indexUV = startUV + (row / 2) * (2 * uvStride);
int indexOut = output.startIndex + row * output.stride;
for (int col = 0; col < output.width; col++, indexOut++) {
int y = 1191 * (dataNV[indexY++] & 0xFF) - 16;
int cr = (dataNV[indexUV] & 0xFF) - 128;
int cb = (dataNV[indexUV + 1] & 0xFF) - 128;
if (y < 0) y = 0;
int r = (y + 1836 * cr) >> 10;
int g = (y - 547 * cr - 218 * cb) >> 10;
int b = (y + 2165 * cb) >> 10;
if (r < 0) r = 0;
else if (r > 255) r = 255;
if (g < 0) g = 0;
else if (g > 255) g = 255;
if (b < 0) b = 0;
else if (b > 255) b = 255;
R.data[indexOut] = (byte) r;
G.data[indexOut] = (byte) g;
B.data[indexOut] = (byte) b;
indexUV += 2 * (col & 0x1);
}
}
}
@Test
public void rgbToLab_F32() {
MultiSpectral<ImageFloat32> input =
new MultiSpectral<ImageFloat32>(ImageFloat32.class, 20, 25, 3);
MultiSpectral<ImageFloat32> output =
new MultiSpectral<ImageFloat32>(ImageFloat32.class, 20, 25, 3);
GImageMiscOps.fillUniform(input, rand, 0, 255);
ColorLab.rgbToLab_F32(input, output);
float expected[] = new float[3];
for (int y = 0; y < input.height; y++) {
for (int x = 0; x < input.width; x++) {
float R = input.getBand(0).get(x, y);
float G = input.getBand(1).get(x, y);
float B = input.getBand(2).get(x, y);
ColorLab.srgbToLab(R / 255f, G / 255f, B / 255f, expected);
float L = output.getBand(0).get(x, y);
float A = output.getBand(1).get(x, y);
float B_ = output.getBand(2).get(x, y);
assertEquals(expected[0], L, 1e-4f);
assertEquals(expected[1], A, 1e-4f);
assertEquals(expected[2], B_, 1e-4f);
}
}
}
/**
* Conversion from YCbCr to RGB.
*
* @param yuv YCbCr encoded 8-bit image
* @param rgb RGB encoded 8-bit image
*/
public static void ycbcrToRgb_U8(MultiSpectral<ImageUInt8> yuv, MultiSpectral<ImageUInt8> rgb) {
ImageUInt8 Y = yuv.getBand(0);
ImageUInt8 U = yuv.getBand(1);
ImageUInt8 V = yuv.getBand(2);
ImageUInt8 R = rgb.getBand(0);
ImageUInt8 G = rgb.getBand(1);
ImageUInt8 B = rgb.getBand(2);
for (int row = 0; row < yuv.height; row++) {
int indexYuv = yuv.startIndex + row * yuv.stride;
int indexRgb = rgb.startIndex + row * rgb.stride;
for (int col = 0; col < yuv.width; col++, indexYuv++, indexRgb++) {
int y = 1191 * ((Y.data[indexYuv] & 0xFF) - 16);
int cb = (U.data[indexYuv] & 0xFF) - 128;
int cr = (V.data[indexYuv] & 0xFF) - 128;
if (y < 0) y = 0;
int r = (y + 1836 * cr) >> 10;
int g = (y - 547 * cr - 218 * cb) >> 10;
int b = (y + 2165 * cb) >> 10;
if (r < 0) r = 0;
else if (r > 255) r = 255;
if (g < 0) g = 0;
else if (g > 255) g = 255;
if (b < 0) b = 0;
else if (b > 255) b = 255;
R.data[indexRgb] = (byte) r;
G.data[indexRgb] = (byte) g;
B.data[indexRgb] = (byte) b;
}
}
}
@Test
public void checkRender() {
// Easier to make up a plane in this direction
Se3_F64 cameraToPlane = new Se3_F64();
ConvertRotation3D_F64.eulerToMatrix(
EulerType.XYZ, UtilAngle.degreeToRadian(0), 0, 0, cameraToPlane.getR());
cameraToPlane.getT().set(0, -5, 0);
Se3_F64 planeToCamera = cameraToPlane.invert(null);
CreateSyntheticOverheadViewMS<ImageFloat32> alg =
new CreateSyntheticOverheadViewMS<ImageFloat32>(
TypeInterpolate.BILINEAR, 3, ImageFloat32.class);
alg.configure(param, planeToCamera, centerX, centerY, cellSize, overheadW, overheadH);
MultiSpectral<ImageFloat32> input =
new MultiSpectral<ImageFloat32>(ImageFloat32.class, width, height, 3);
for (int i = 0; i < 3; i++) ImageMiscOps.fill(input.getBand(i), 10 + i);
MultiSpectral<ImageFloat32> output =
new MultiSpectral<ImageFloat32>(ImageFloat32.class, overheadW, overheadH, 3);
alg.process(input, output);
for (int i = 0; i < 3; i++) {
ImageFloat32 o = output.getBand(i);
// check parts that shouldn't be in view
assertEquals(0, o.get(0, 300), 1e-8);
assertEquals(0, o.get(5, 0), 1e-8);
assertEquals(0, o.get(5, 599), 1e-8);
// check areas that should be in view
assertEquals(10 + i, o.get(499, 300), 1e-8);
}
}
public static void nv21ToMultiYuv_F32(byte[] dataNV, MultiSpectral<ImageFloat32> output) {
ImageFloat32 Y = output.getBand(0);
ImageFloat32 U = output.getBand(1);
ImageFloat32 V = output.getBand(2);
final int uvStride = output.width / 2;
nv21ToGray(dataNV, Y);
final int startUV = output.width * output.height;
for (int row = 0; row < output.height; row++) {
int indexUV = startUV + (row / 2) * (2 * uvStride);
int indexOut = output.startIndex + row * output.stride;
for (int col = 0; col < output.width; col++, indexOut++) {
U.data[indexOut] = (dataNV[indexUV] & 0xFF) - 128;
V.data[indexOut] = (dataNV[indexUV + 1] & 0xFF) - 128;
indexUV += 2 * (col & 0x1);
}
}
}
private void undoRadialDistortion(BufferedImage image) {
ConvertBufferedImage.convertFromMulti(image, origMS, true, ImageFloat32.class);
for (int i = 0; i < origMS.getNumBands(); i++) {
ImageFloat32 in = origMS.getBand(i);
ImageFloat32 out = correctedMS.getBand(i);
undoRadial.apply(in, out);
}
if (correctedMS.getNumBands() == 3)
ConvertBufferedImage.convertTo(correctedMS, undistorted, true);
else if (correctedMS.getNumBands() == 1)
ConvertBufferedImage.convertTo(correctedMS.getBand(0), undistorted);
else throw new RuntimeException("What kind of image has " + correctedMS.getNumBands() + "???");
}
public static void assertEqualsInner(
ImageBase imgA, ImageBase imgB, double tol, int borderX, int borderY, boolean relative) {
// if no specialized check exists, use a slower generalized approach
if (imgA instanceof ImageSingleBand) {
GImageSingleBand a = FactoryGImageSingleBand.wrap((ImageSingleBand) imgA);
GImageSingleBand b = FactoryGImageSingleBand.wrap((ImageSingleBand) imgB);
for (int y = borderY; y < imgA.height - borderY; y++) {
for (int x = borderX; x < imgA.width - borderX; x++) {
double valA = a.get(x, y).doubleValue();
double valB = b.get(x, y).doubleValue();
double error = Math.abs(valA - valB);
if (relative) {
double denominator = Math.abs(valA) + Math.abs(valB);
if (denominator == 0) denominator = 1;
error /= denominator;
}
if (error > tol)
throw new RuntimeException(
"Values not equal at (" + x + "," + y + ") " + valA + " " + valB);
}
}
} else if (imgA instanceof MultiSpectral) {
MultiSpectral a = (MultiSpectral) imgA;
MultiSpectral b = (MultiSpectral) imgB;
if (a.getNumBands() != b.getNumBands())
throw new RuntimeException("Number of bands not equal");
for (int band = 0; band < a.getNumBands(); band++) {
assertEqualsInner(a.getBand(band), b.getBand(band), tol, borderX, borderY, relative);
}
} else {
throw new RuntimeException("Unknown image type");
}
}
public static void checkEquals(
BufferedImage imgA, MultiSpectral imgB, boolean boofcvBandOrder, float tol) {
if (imgA.getRaster() instanceof ByteInterleavedRaster
&& imgA.getType() != BufferedImage.TYPE_BYTE_INDEXED) {
ByteInterleavedRaster raster = (ByteInterleavedRaster) imgA.getRaster();
if (raster.getNumBands() == 1) {
GImageSingleBand band = FactoryGImageSingleBand.wrap(imgB.getBand(0));
int strideA = raster.getScanlineStride();
int offsetA = raster.getDataOffset(0) - raster.getPixelStride() + 1;
// handle a special case where the RGB conversion is screwed
for (int i = 0; i < imgA.getHeight(); i++) {
for (int j = 0; j < imgA.getWidth(); j++) {
double valB = band.get(j, i).doubleValue();
int valA = raster.getDataStorage()[offsetA + i * strideA + j];
valA &= 0xFF;
if (Math.abs(valA - valB) > tol)
throw new RuntimeException("Images are not equal: A = " + valA + " B = " + valB);
}
}
return;
}
}
int bandOrder[];
if (boofcvBandOrder) {
if (imgB.getNumBands() == 4) {
bandOrder = new int[] {1, 2, 3, 0};
} else {
bandOrder = new int[] {0, 1, 2};
}
} else {
if (imgA.getType() == BufferedImage.TYPE_INT_RGB) {
bandOrder = new int[] {0, 1, 2};
} else if (imgA.getType() == BufferedImage.TYPE_INT_BGR
|| imgA.getType() == BufferedImage.TYPE_3BYTE_BGR) {
bandOrder = new int[] {2, 1, 0};
} else if (imgA.getType() == BufferedImage.TYPE_4BYTE_ABGR) {
bandOrder = new int[] {0, 3, 2, 1};
} else if (imgA.getType() == BufferedImage.TYPE_INT_ARGB) {
bandOrder = new int[] {0, 1, 2, 3};
} else {
bandOrder = new int[] {0, 1, 2};
}
}
int expected[] = new int[4];
for (int y = 0; y < imgA.getHeight(); y++) {
for (int x = 0; x < imgA.getWidth(); x++) {
// getRGB() automatically converts the band order to ARGB
int rgb = imgA.getRGB(x, y);
expected[0] = ((rgb >>> 24) & 0xFF); // alpha
expected[1] = ((rgb >>> 16) & 0xFF); // red
expected[2] = ((rgb >>> 8) & 0xFF); // green
expected[3] = (rgb & 0xFF); // blue
if (imgB.getNumBands() == 4) {
for (int i = 0; i < 4; i++) {
double found = GeneralizedImageOps.get(imgB.getBand(bandOrder[i]), x, y);
if (Math.abs(Math.exp(expected[i] - found)) > tol) {
for (int j = 0; j < 4; j++) {
System.out.println(
expected[j] + " " + GeneralizedImageOps.get(imgB.getBand(bandOrder[j]), x, y));
}
throw new RuntimeException(
"Images are not equal: band - " + i + " type " + imgA.getType());
}
}
} else if (imgB.getNumBands() == 3) {
for (int i = 0; i < 3; i++) {
double found = GeneralizedImageOps.get(imgB.getBand(bandOrder[i]), x, y);
if (Math.abs(expected[i + 1] - found) > tol) {
for (int j = 0; j < 3; j++) {
System.out.println(
expected[j + 1]
+ " "
+ GeneralizedImageOps.get(imgB.getBand(bandOrder[j]), x, y));
}
throw new RuntimeException(
"Images are not equal: band - " + i + " type " + imgA.getType());
}
}
} else {
throw new RuntimeException("Unexpectd number of bands");
}
}
}
}
public static void checkEquals(WritableRaster imgA, MultiSpectral imgB, float tol) {
if (imgA.getNumBands() != imgB.getNumBands()) {
throw new RuntimeException("Number of bands not equals");
}
if (imgA instanceof ByteInterleavedRaster) {
ByteInterleavedRaster raster = (ByteInterleavedRaster) imgA;
byte dataA[] = raster.getDataStorage();
int strideA = raster.getScanlineStride();
int offsetA = raster.getDataOffset(0) - raster.getPixelStride() + 1;
// handle a special case where the RGB conversion is screwed
for (int y = 0; y < imgA.getHeight(); y++) {
int indexA = offsetA + strideA * y;
for (int x = 0; x < imgA.getWidth(); x++) {
for (int k = 0; k < imgB.getNumBands(); k++) {
int valueA = dataA[indexA++] & 0xFF;
double valueB = GeneralizedImageOps.get(imgB.getBand(k), x, y);
if (Math.abs(valueA - valueB) > tol)
throw new RuntimeException("Images are not equal: A = " + valueA + " B = " + valueB);
}
}
}
} else if (imgA instanceof IntegerInterleavedRaster) {
IntegerInterleavedRaster raster = (IntegerInterleavedRaster) imgA;
int dataA[] = raster.getDataStorage();
int strideA = raster.getScanlineStride();
int offsetA = raster.getDataOffset(0) - raster.getPixelStride() + 1;
// handle a special case where the RGB conversion is screwed
for (int y = 0; y < imgA.getHeight(); y++) {
int indexA = offsetA + strideA * y;
for (int x = 0; x < imgA.getWidth(); x++) {
int valueA = dataA[indexA++];
if (imgB.getNumBands() == 4) {
int found0 = (valueA >> 24) & 0xFF;
int found1 = (valueA >> 16) & 0xFF;
int found2 = (valueA >> 8) & 0xFF;
int found3 = valueA & 0xFF;
double expected0 = GeneralizedImageOps.get(imgB.getBand(0), x, y);
double expected1 = GeneralizedImageOps.get(imgB.getBand(1), x, y);
double expected2 = GeneralizedImageOps.get(imgB.getBand(2), x, y);
double expected3 = GeneralizedImageOps.get(imgB.getBand(3), x, y);
if (Math.abs(found0 - expected0) > tol)
throw new RuntimeException("Images are not equal");
if (Math.abs(found1 - expected1) > tol)
throw new RuntimeException("Images are not equal");
if (Math.abs(found2 - expected2) > tol)
throw new RuntimeException("Images are not equal");
if (Math.abs(found3 - expected3) > tol)
throw new RuntimeException("Images are not equal");
} else if (imgB.getNumBands() == 3) {
int found0 = (valueA >> 16) & 0xFF;
int found1 = (valueA >> 8) & 0xFF;
int found2 = valueA & 0xFF;
double expected0 = GeneralizedImageOps.get(imgB.getBand(0), x, y);
double expected1 = GeneralizedImageOps.get(imgB.getBand(1), x, y);
double expected2 = GeneralizedImageOps.get(imgB.getBand(2), x, y);
if (Math.abs(found0 - expected0) > tol)
throw new RuntimeException("Images are not equal");
if (Math.abs(found1 - expected1) > tol)
throw new RuntimeException("Images are not equal");
if (Math.abs(found2 - expected2) > tol)
throw new RuntimeException("Images are not equal");
} else {
throw new RuntimeException("Unexpectd number of bands");
}
}
}
} else {
throw new RuntimeException("Add support for raster type " + imgA.getClass().getSimpleName());
}
}
@Override
public void setImage(MultiSpectral<ImageUInt8> image) {
imageRed = image.getBand(0);
imageGreen = image.getBand(1);
imageBlue = image.getBand(2);
}
public <II extends ImageSingleBand> double[][] harder(BufferedImage image) {
MultiSpectral<ImageFloat32> colorImage =
ConvertBufferedImage.convertFromMulti(image, null, true, ImageFloat32.class);
// convert the color image to greyscale
ImageFloat32 greyscaleImage =
ConvertImage.average((MultiSpectral<ImageFloat32>) colorImage, null);
// SURF works off of integral images
Class<II> integralType = GIntegralImageOps.getIntegralType(ImageFloat32.class);
// define the feature detection algorithm
NonMaxSuppression extractor =
FactoryFeatureExtractor.nonmax(new ConfigExtract(2, detectThreshold, 5, true));
FastHessianFeatureDetector<II> detector =
new FastHessianFeatureDetector<II>(extractor, maxFeaturesPerScale, 2, 9, 4, 4);
// estimate orientation
OrientationIntegral<II> orientation = FactoryOrientationAlgs.sliding_ii(null, integralType);
DescribePointSurf<II> descriptor =
FactoryDescribePointAlgs.<II>surfStability(null, integralType);
// compute the integral image of the greyscale 'image'
II integralgrey =
GeneralizedImageOps.createSingleBand(
integralType, greyscaleImage.width, greyscaleImage.height);
GIntegralImageOps.transform(greyscaleImage, integralgrey);
// detect fast hessian features
detector.detect(integralgrey);
// === This is the point were the code starts deviating from the standard SURF! ===
// tell algorithms which image to process
orientation.setImage(integralgrey);
List<ScalePoint> points = detector.getFoundPoints();
double[][] descriptions = new double[points.size()][3 * descriptor.getDescriptionLength()];
double[] angles = new double[points.size()];
int l = 0;
for (ScalePoint p : points) {
orientation.setScale(p.scale);
angles[l] = orientation.compute(p.x, p.y);
l++;
}
for (int i = 0; i < 3; i++) {
// check if it is actually a greyscale image, take always the 1st band!
ImageFloat32 colorImageBand = null;
if (colorImage.getNumBands() == 1) {
colorImageBand = colorImage.getBand(0);
} else {
colorImageBand = colorImage.getBand(i);
}
// compute the integral image of the i-th band of the color 'image'
II integralband =
GeneralizedImageOps.createSingleBand(
integralType, colorImageBand.width, colorImageBand.height);
GIntegralImageOps.transform(colorImageBand, integralband);
// tell algorithms which image to process
// orientation.setImage(integralband);
descriptor.setImage(integralband);
int j = 0;
for (ScalePoint p : points) {
// estimate orientation
// orientation.setScale(p.scale);
// double angle = orientation.compute(p.x, p.y);
// extract the SURF description for this region
SurfFeature desc = descriptor.createDescription();
descriptor.describe(p.x, p.y, angles[j], p.scale, (TupleDesc_F64) desc);
double[] banddesc = desc.getValue();
if (perBandNormalization) {
banddesc = Normalization.normalizeL2(banddesc);
}
for (int k = 0; k < SURFLength; k++) {
descriptions[j][i * SURFLength + k] = banddesc[k];
}
j++;
}
}
return descriptions;
}
