@Test
public void getProductByDownloadableFilename() {
fr.gael.dhus.database.object.Collection validCollection;
fr.gael.dhus.database.object.Collection invalidCollection;
Product product;
validCollection = new Collection();
validCollection.setUUID("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa3");
validCollection.setName("Japan");
invalidCollection = new Collection();
invalidCollection.setUUID("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa4");
invalidCollection.setName("China");
product = dao.getProductByDownloadableFilename("prod0", validCollection);
Assert.assertNotNull(product);
Assert.assertEquals(product.getId().intValue(), 0);
product = dao.getProductByDownloadableFilename("prod6", validCollection);
Assert.assertNull(product);
product = dao.getProductByDownloadableFilename("prod0", invalidCollection);
Assert.assertNull(product);
product = dao.getProductByDownloadableFilename("prod6", null);
Assert.assertNotNull(product);
Assert.assertEquals(product.getId().intValue(), 6);
product = dao.getProductByDownloadableFilename(null, null);
Assert.assertNull(product);
}
@Override
public void first() {
String hql = "FROM Product WHERE processed IS FALSE ORDER BY id DESC";
Product product = dao.first(hql);
Assert.assertNotNull(product);
Assert.assertEquals(product.getId().intValue(), 4);
}
@Override
public void create() {
String identifier = "test-create-product";
String indexName = "index-name";
String indexCategory = "category";
String indexValue = "test";
MetadataIndex mi = new MetadataIndex();
mi.setName(indexName);
mi.setCategory(indexCategory);
mi.setQueryable(null);
mi.setValue(indexValue);
Product product = new Product();
product.setIdentifier(identifier);
product.setLocked(false);
product.setProcessed(true);
product.setIndexes(Arrays.asList(mi));
try {
product.setPath(new URL("file:/titi/tata"));
} catch (MalformedURLException e) {
Assert.fail(e.getMessage(), e);
}
Product createdProduct = dao.create(product);
Assert.assertNotNull(createdProduct);
Assert.assertEquals(dao.count(), (howMany() + 1));
Assert.assertEquals(createdProduct.getUuid(), product.getUuid());
List<MetadataIndex> indexes = createdProduct.getIndexes();
Assert.assertEquals(indexes.size(), 1);
Assert.assertEquals(indexes.get(0), mi);
}
@Override
public void delete() {
cancelListeners(getHibernateDao());
Long pid = Long.valueOf(6L);
Product product = dao.read(pid);
Assert.assertNotNull(product);
Set<User> authorizedUsers = product.getAuthorizedUsers();
List<MetadataIndex> indexes = product.getIndexes();
Assert.assertNotNull(indexes);
Assert.assertFalse(indexes.isEmpty());
Assert.assertFalse(authorizedUsers.isEmpty());
dao.delete(product);
getHibernateDao().getSessionFactory().getCurrentSession().flush();
Assert.assertNull(dao.read(pid));
Assert.assertEquals(countElements("METADATA_INDEXES", pid), 0);
Assert.assertEquals(countElements("CHECKSUMS", pid), 0);
for (User user : authorizedUsers) {
Assert.assertNotNull(udao.read(user.getUUID()));
}
}
@Transactional(readOnly = false, propagation = Propagation.REQUIRED)
@CacheEvict(
value = {"indexes"},
key = "#product_id")
public void setIndexes(Long product_id, List<MetadataIndex> indexes) {
Product product = productDao.read(product_id);
product.setIndexes(indexes);
productDao.update(product);
}
@Transactional(readOnly = true, propagation = Propagation.REQUIRED)
@Cacheable(
value = {"indexes"},
key = "#product_id")
public List<MetadataIndex> getIndexes(Long product_id) {
Product product = productDao.read(product_id);
Hibernate.initialize(product.getIndexes());
return product.getIndexes();
}
@Override
public void read() {
Product p = dao.read(6L);
Assert.assertNotNull(p);
Assert.assertEquals(p.getIdentifier(), "prod6");
Download dl = p.getDownload();
Map<String, String> checksums = dl.getChecksums();
Assert.assertEquals(checksums.get("MD5"), "abc");
}
// Check if product present is the DB is still present into the repository.
@Transactional(readOnly = false, propagation = Propagation.REQUIRED)
public void checkDBProducts() {
logger.info("Syncing database with repositories...");
Iterator<Product> products = productDao.getAllProducts();
while (products.hasNext()) {
Product product = products.next();
if (!ProductService.checkUrl(product.getPath())) {
logger.info("Removing Product " + product.getPath() + " not found in repository.");
products.remove();
} else logger.info("Product " + product.getPath() + " found in repository.");
}
}
@PreAuthorize("hasAnyRole('ROLE_DOWNLOAD','ROLE_SEARCH')")
@Transactional(readOnly = true, propagation = Propagation.REQUIRED)
public InputStream getProductThumbnail(Long id) {
// TODO remove method cause not used
Product product = getProduct(id);
if (!product.getThumbnailFlag()) return null;
try {
return new FileInputStream(product.getThumbnailPath());
} catch (Exception e) {
logger.warn("Cannot retrieve Thumbnail from product id #" + id, e);
}
return null;
}
/**
* Do process all products in database that their ingestion is not finished. If provided parameter
* is null, default processing consists in removing the data from database. Otherwise, the
* provided processing is launched.
*
* @param proc the processing to execute. if null, remove processing will be performed.
* @return the list of products reprocessed.
*/
@Transactional(readOnly = false, propagation = Propagation.REQUIRED)
@CacheEvict(
value = {"product_count", "product", "products"},
allEntries = true)
public void processUnprocessed(boolean recover) {
long start = new Date().getTime();
if (!recover) {
Iterator<Product> products = getUnprocessedProducts();
while (products.hasNext()) {
products.next();
products.remove();
}
logger.debug(
"Cleanup incomplete processed products in " + (new Date().getTime() - start) + "ms");
} else {
Iterator<Product> products = getUnprocessedProducts();
while (products.hasNext()) {
Product product = products.next();
// Do reporcess only already transfered products
if (product.getPath().toString().equals(product.getOrigin())) {
products.remove();
} else {
try {
String path = product.getPath().getPath();
logger.info("Recovering product from " + path);
// Check if product is still present in repository
if (!new File(path).exists()) {
throw new DataStoreException("Product " + path + " not present locally.");
}
// Retrieve owner if any
User owner = productDao.getOwnerOfProduct(product);
// Retrieve collections
List<Collection> collections = collectionDao.getCollectionsOfProduct(product.getId());
processProduct(product, owner, collections, null, null);
} catch (Exception e) {
logger.error("Error while processing: " + e.getMessage() + "- abort reprocessing.");
products.remove();
}
}
}
}
}
/**
* Adds a product in the database, the given product will not be queued for processing nor it will
* be submitted to the search engine.
*
* @param product a product to store in the database.
* @return the created product.
* @throws IllegalArgumentException incomplete products are not allowed.
*/
@Transactional(readOnly = false, propagation = Propagation.REQUIRED)
@Caching(
evict = {
@CacheEvict(value = "product", allEntries = true),
@CacheEvict(value = "products", allEntries = true)
})
@IncrementCache(name = "product_count", key = "all", value = 1)
public Product addProduct(Product product) throws IllegalArgumentException {
URL path = product.getPath();
String origin = product.getOrigin();
if (path == null || origin == null || origin.isEmpty()) {
throw new IllegalArgumentException("product must have a path and an origin");
}
// FIXME do I have to check every field? isn't it done by hibernate based on column constraints?
Product final_product = this.productDao.create(product);
return final_product;
}
@Test
public void getProductByPath() {
URL valid = null;
URL invalid = null;
try {
valid = new URL("file:/home/lambert/test/prod5");
invalid = new URL("file:/home/lambert/test/prod512");
} catch (MalformedURLException e) {
Assert.fail("Malformed URL !", e);
}
Product product = dao.getProductByPath(valid);
Assert.assertNotNull(product);
Assert.assertEquals(product.getId().intValue(), 5);
product = dao.getProductByPath(invalid);
Assert.assertNull(product);
product = dao.getProductByPath(null);
Assert.assertNull(product);
}
@Transactional(readOnly = false, propagation = Propagation.REQUIRED)
@Caching(
evict = {
@CacheEvict(value = "indexes", key = "#pid"),
@CacheEvict(value = "product", key = "#pid"),
@CacheEvict(value = "products", allEntries = true)
})
@IncrementCache(name = "product_count", key = "all", value = -1)
public void systemDeleteProduct(Long pid) {
Product product = productDao.read(pid);
if (product == null) {
throw new DataStoreException("Product #" + pid + " not found in the system.");
}
if (product.getLocked()) {
throw new DataStoreException(
"Cannot delete product #" + pid + ". Product is locked in the system.");
}
productDao.delete(product);
}
public void addProduct(
URL path,
User owner,
final List<Collection> collections,
String origin,
Scanner scanner,
FileScannerWrapper wrapper)
throws DataStoreAlreadyExistException {
if (productDao.exists(path)) {
throw new DataStoreAlreadyExistException(
"Product \"" + path.toExternalForm() + "\" already present in the system.");
}
/* **** CRITICAL SECTION *** */
/** THIS SECTION SHALL NEVER BE STOPPED BY CNTRL-C OR OTHER SIGNALS */
/* TODO: check if shutdownHook can protect this section */
Product product = new Product();
product.setPath(path);
product.setOrigin(origin);
List<User> users = new ArrayList<User>();
if (owner != null) {
product.setOwner(owner);
users.add(userDao.read(owner.getId()));
product.setAuthorizedUsers(new HashSet<User>(users));
}
product = productDao.create(product);
// FIX
product = productDao.read(product.getId());
/* **** CRITICAL SECTION *** */
processProduct(product, owner, collections, scanner, wrapper);
}
@Test
public void getProductByUuid() {
String valid = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa6";
String invalid = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb6";
User user = new User();
user.setUUID("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa0");
user.setUsername("koko");
Product product = dao.getProductByUuid(valid);
Assert.assertNotNull(product);
Assert.assertEquals(product.getId().intValue(), 6);
product = dao.getProductByUuid(invalid);
Assert.assertNull(product);
product = dao.getProductByUuid(valid);
Assert.assertNotNull(product);
Assert.assertEquals(product.getId().intValue(), 6);
product = dao.getProductByUuid(null);
Assert.assertNull(product);
}
@Test(groups = {"non-regression"})
public void testChecksumUpdate() throws MalformedURLException {
Product product = new Product();
product.setPath(new URL("file:/product/path"));
Download download = new Product.Download();
download.setPath("/no/path/file");
download.setSize(0L);
download.setType("application/octet-stream");
download.setChecksums(
Maps.newHashMap(
ImmutableMap.of(
"MD5", "54ABCDEF98765",
"SHA-1", "9876FEDCBA1234")));
product.setDownload(download);
// First create the defined product:
try {
product = dao.create(product);
} catch (Exception e) {
Assert.fail("Creation of product fails", e);
}
/** Clear/putAll feature testing */
product.getDownload().getChecksums().clear();
product
.getDownload()
.getChecksums()
.putAll(
Maps.newHashMap(
ImmutableMap.of(
"SHA-256", "4554ABCDEF98765",
"SHA-512", "ABDEFFE9876FEDCBA1234")));
try {
dao.update(product);
} catch (Exception e) {
Assert.fail("Modifying checksums with map clear/put fails", e);
}
/** Set feature testing */
product
.getDownload()
.setChecksums(
Maps.newHashMap(
ImmutableMap.of(
"MD5", "54ABCDEF98765",
"SHA-1", "9876FEDCBA1234")));
try {
dao.update(product);
} catch (Exception e) {
Assert.fail("Modifying checksums with \"set\" fails", e);
}
/** Remove residuals for this test */
cancelListeners(getHibernateDao());
dao.delete(product);
}
@Override
public void update() {
String productIdentifier = "test-prod-7";
String indexName = "updatable";
Long pid = Long.valueOf(7);
Product product = dao.read(pid);
List<MetadataIndex> indexes = product.getIndexes();
product.setIdentifier(productIdentifier);
for (MetadataIndex mi : indexes) {
mi.setName(indexName);
}
dao.setIndexes(pid, indexes);
dao.update(product);
product = dao.read(pid);
indexes = product.getIndexes();
Assert.assertNotNull(product);
Assert.assertEquals(product.getIdentifier(), productIdentifier);
for (MetadataIndex mi : indexes) {
Assert.assertEquals(mi.getName(), indexName);
}
}
@Transactional(readOnly = false, propagation = Propagation.REQUIRED)
@Caching(
evict = {
@CacheEvict(value = "product", allEntries = true),
@CacheEvict(value = "products", allEntries = true)
})
@IncrementCache(name = "product_count", key = "all", value = 1)
public void run() {
if (scanner != null && scanner.isStopped()) {
logger.info("Scanner stopped, deleting product #" + product.getId());
if (wrapper != null) {
wrapper.error(product, new InterruptedException("Processing stopped by the user"));
}
productDao.delete(product);
return;
}
logger.debug("Add product \"" + ProductDao.getPathFromProduct(product) + "\".");
try {
long processing_start = System.currentTimeMillis();
if (wrapper != null) {
wrapper.startIngestion();
}
processingManager.process(product);
productDao.update(product);
searchService.index(product);
if (collections != null) {
for (Collection c : collections) {
collectionService.systemAddProduct(c.getId(), product.getId(), true);
}
}
if (wrapper != null) {
wrapper.endIngestion();
}
long processing_end = System.currentTimeMillis();
logger.info(
"Ingestion processing complete for product "
+ product.getPath().toExternalForm()
+ " ("
+ product.getSize()
+ " bytes, "
+ product.getDownloadableSize()
+ " bytes compressed)"
+ " in "
+ (processing_end - processing_start)
+ "ms.");
actionRecordWritterDao.uploadEnd(
this.product.getPath(), this.owner.getUsername(), collections, true);
} catch (Exception excp) {
logger.warn(
"Unrecoverable error happen during ingestion of "
+ product.getPath()
+ " (removing from database)",
excp);
try {
productDao.delete(product);
} catch (Exception e) {
logger.error("Unable to remove product after ingestion failure", e);
}
if (wrapper != null) {
wrapper.error(product, excp);
}
if ((this.product.getPath() != null) && (this.owner != null)) {
actionRecordWritterDao.uploadFailed(
this.product.getPath().toString(), this.owner.getUsername());
actionRecordWritterDao.uploadEnd(
this.product.getPath(), this.owner.getUsername(), collections, false);
}
return;
}
}
