コード例 #1
0
  @Test
  public void getProductByDownloadableFilename() {
    fr.gael.dhus.database.object.Collection validCollection;
    fr.gael.dhus.database.object.Collection invalidCollection;
    Product product;

    validCollection = new Collection();
    validCollection.setUUID("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa3");
    validCollection.setName("Japan");

    invalidCollection = new Collection();
    invalidCollection.setUUID("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa4");
    invalidCollection.setName("China");

    product = dao.getProductByDownloadableFilename("prod0", validCollection);
    Assert.assertNotNull(product);
    Assert.assertEquals(product.getId().intValue(), 0);

    product = dao.getProductByDownloadableFilename("prod6", validCollection);
    Assert.assertNull(product);

    product = dao.getProductByDownloadableFilename("prod0", invalidCollection);
    Assert.assertNull(product);

    product = dao.getProductByDownloadableFilename("prod6", null);
    Assert.assertNotNull(product);
    Assert.assertEquals(product.getId().intValue(), 6);

    product = dao.getProductByDownloadableFilename(null, null);
    Assert.assertNull(product);
  }
コード例 #2
0
 @Override
 public void first() {
   String hql = "FROM Product WHERE processed IS FALSE ORDER BY id DESC";
   Product product = dao.first(hql);
   Assert.assertNotNull(product);
   Assert.assertEquals(product.getId().intValue(), 4);
 }
コード例 #3
0
  @Override
  public void create() {
    String identifier = "test-create-product";
    String indexName = "index-name";
    String indexCategory = "category";
    String indexValue = "test";

    MetadataIndex mi = new MetadataIndex();
    mi.setName(indexName);
    mi.setCategory(indexCategory);
    mi.setQueryable(null);
    mi.setValue(indexValue);

    Product product = new Product();
    product.setIdentifier(identifier);
    product.setLocked(false);
    product.setProcessed(true);
    product.setIndexes(Arrays.asList(mi));
    try {
      product.setPath(new URL("file:/titi/tata"));
    } catch (MalformedURLException e) {
      Assert.fail(e.getMessage(), e);
    }

    Product createdProduct = dao.create(product);
    Assert.assertNotNull(createdProduct);
    Assert.assertEquals(dao.count(), (howMany() + 1));
    Assert.assertEquals(createdProduct.getUuid(), product.getUuid());

    List<MetadataIndex> indexes = createdProduct.getIndexes();
    Assert.assertEquals(indexes.size(), 1);
    Assert.assertEquals(indexes.get(0), mi);
  }
コード例 #4
0
  @Override
  public void delete() {
    cancelListeners(getHibernateDao());

    Long pid = Long.valueOf(6L);
    Product product = dao.read(pid);
    Assert.assertNotNull(product);
    Set<User> authorizedUsers = product.getAuthorizedUsers();

    List<MetadataIndex> indexes = product.getIndexes();
    Assert.assertNotNull(indexes);
    Assert.assertFalse(indexes.isEmpty());
    Assert.assertFalse(authorizedUsers.isEmpty());

    dao.delete(product);
    getHibernateDao().getSessionFactory().getCurrentSession().flush();

    Assert.assertNull(dao.read(pid));
    Assert.assertEquals(countElements("METADATA_INDEXES", pid), 0);
    Assert.assertEquals(countElements("CHECKSUMS", pid), 0);

    for (User user : authorizedUsers) {
      Assert.assertNotNull(udao.read(user.getUUID()));
    }
  }
コード例 #5
0
 @Transactional(readOnly = false, propagation = Propagation.REQUIRED)
 @CacheEvict(
     value = {"indexes"},
     key = "#product_id")
 public void setIndexes(Long product_id, List<MetadataIndex> indexes) {
   Product product = productDao.read(product_id);
   product.setIndexes(indexes);
   productDao.update(product);
 }
コード例 #6
0
 @Transactional(readOnly = true, propagation = Propagation.REQUIRED)
 @Cacheable(
     value = {"indexes"},
     key = "#product_id")
 public List<MetadataIndex> getIndexes(Long product_id) {
   Product product = productDao.read(product_id);
   Hibernate.initialize(product.getIndexes());
   return product.getIndexes();
 }
コード例 #7
0
 @Override
 public void read() {
   Product p = dao.read(6L);
   Assert.assertNotNull(p);
   Assert.assertEquals(p.getIdentifier(), "prod6");
   Download dl = p.getDownload();
   Map<String, String> checksums = dl.getChecksums();
   Assert.assertEquals(checksums.get("MD5"), "abc");
 }
コード例 #8
0
 // Check if product present is the DB is still present into the repository.
 @Transactional(readOnly = false, propagation = Propagation.REQUIRED)
 public void checkDBProducts() {
   logger.info("Syncing database with repositories...");
   Iterator<Product> products = productDao.getAllProducts();
   while (products.hasNext()) {
     Product product = products.next();
     if (!ProductService.checkUrl(product.getPath())) {
       logger.info("Removing Product " + product.getPath() + " not found in repository.");
       products.remove();
     } else logger.info("Product " + product.getPath() + " found in repository.");
   }
 }
コード例 #9
0
 @PreAuthorize("hasAnyRole('ROLE_DOWNLOAD','ROLE_SEARCH')")
 @Transactional(readOnly = true, propagation = Propagation.REQUIRED)
 public InputStream getProductThumbnail(Long id) {
   // TODO remove method cause not used
   Product product = getProduct(id);
   if (!product.getThumbnailFlag()) return null;
   try {
     return new FileInputStream(product.getThumbnailPath());
   } catch (Exception e) {
     logger.warn("Cannot retrieve Thumbnail from product id #" + id, e);
   }
   return null;
 }
コード例 #10
0
  /**
   * Do process all products in database that their ingestion is not finished. If provided parameter
   * is null, default processing consists in removing the data from database. Otherwise, the
   * provided processing is launched.
   *
   * @param proc the processing to execute. if null, remove processing will be performed.
   * @return the list of products reprocessed.
   */
  @Transactional(readOnly = false, propagation = Propagation.REQUIRED)
  @CacheEvict(
      value = {"product_count", "product", "products"},
      allEntries = true)
  public void processUnprocessed(boolean recover) {
    long start = new Date().getTime();

    if (!recover) {
      Iterator<Product> products = getUnprocessedProducts();
      while (products.hasNext()) {
        products.next();
        products.remove();
      }

      logger.debug(
          "Cleanup incomplete processed products in " + (new Date().getTime() - start) + "ms");
    } else {
      Iterator<Product> products = getUnprocessedProducts();
      while (products.hasNext()) {
        Product product = products.next();
        // Do reporcess only already transfered products
        if (product.getPath().toString().equals(product.getOrigin())) {
          products.remove();
        } else {
          try {
            String path = product.getPath().getPath();
            logger.info("Recovering product from " + path);
            // Check if product is still present in repository
            if (!new File(path).exists()) {
              throw new DataStoreException("Product " + path + " not present locally.");
            }
            // Retrieve owner if any
            User owner = productDao.getOwnerOfProduct(product);

            // Retrieve collections
            List<Collection> collections = collectionDao.getCollectionsOfProduct(product.getId());

            processProduct(product, owner, collections, null, null);
          } catch (Exception e) {
            logger.error("Error while processing: " + e.getMessage() + "- abort reprocessing.");
            products.remove();
          }
        }
      }
    }
  }
コード例 #11
0
  /**
   * Adds a product in the database, the given product will not be queued for processing nor it will
   * be submitted to the search engine.
   *
   * @param product a product to store in the database.
   * @return the created product.
   * @throws IllegalArgumentException incomplete products are not allowed.
   */
  @Transactional(readOnly = false, propagation = Propagation.REQUIRED)
  @Caching(
      evict = {
        @CacheEvict(value = "product", allEntries = true),
        @CacheEvict(value = "products", allEntries = true)
      })
  @IncrementCache(name = "product_count", key = "all", value = 1)
  public Product addProduct(Product product) throws IllegalArgumentException {
    URL path = product.getPath();
    String origin = product.getOrigin();
    if (path == null || origin == null || origin.isEmpty()) {
      throw new IllegalArgumentException("product must have a path and an origin");
    }
    // FIXME do I have to check every field? isn't it done by hibernate based on column constraints?

    Product final_product = this.productDao.create(product);
    return final_product;
  }
コード例 #12
0
  @Test
  public void getProductByPath() {
    URL valid = null;
    URL invalid = null;
    try {
      valid = new URL("file:/home/lambert/test/prod5");
      invalid = new URL("file:/home/lambert/test/prod512");
    } catch (MalformedURLException e) {
      Assert.fail("Malformed URL !", e);
    }

    Product product = dao.getProductByPath(valid);
    Assert.assertNotNull(product);
    Assert.assertEquals(product.getId().intValue(), 5);

    product = dao.getProductByPath(invalid);
    Assert.assertNull(product);

    product = dao.getProductByPath(null);
    Assert.assertNull(product);
  }
コード例 #13
0
  @Transactional(readOnly = false, propagation = Propagation.REQUIRED)
  @Caching(
      evict = {
        @CacheEvict(value = "indexes", key = "#pid"),
        @CacheEvict(value = "product", key = "#pid"),
        @CacheEvict(value = "products", allEntries = true)
      })
  @IncrementCache(name = "product_count", key = "all", value = -1)
  public void systemDeleteProduct(Long pid) {
    Product product = productDao.read(pid);

    if (product == null) {
      throw new DataStoreException("Product #" + pid + " not found in the system.");
    }

    if (product.getLocked()) {
      throw new DataStoreException(
          "Cannot delete product #" + pid + ". Product is locked in the system.");
    }
    productDao.delete(product);
  }
コード例 #14
0
  public void addProduct(
      URL path,
      User owner,
      final List<Collection> collections,
      String origin,
      Scanner scanner,
      FileScannerWrapper wrapper)
      throws DataStoreAlreadyExistException {
    if (productDao.exists(path)) {
      throw new DataStoreAlreadyExistException(
          "Product \"" + path.toExternalForm() + "\" already present in the system.");
    }

    /* **** CRITICAL SECTION *** */
    /** THIS SECTION SHALL NEVER BE STOPPED BY CNTRL-C OR OTHER SIGNALS */
    /* TODO: check if shutdownHook can protect this section */
    Product product = new Product();
    product.setPath(path);
    product.setOrigin(origin);
    List<User> users = new ArrayList<User>();

    if (owner != null) {
      product.setOwner(owner);
      users.add(userDao.read(owner.getId()));
      product.setAuthorizedUsers(new HashSet<User>(users));
    }

    product = productDao.create(product);

    // FIX
    product = productDao.read(product.getId());
    /* **** CRITICAL SECTION *** */
    processProduct(product, owner, collections, scanner, wrapper);
  }
コード例 #15
0
  @Test
  public void getProductByUuid() {
    String valid = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa6";
    String invalid = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb6";
    User user = new User();
    user.setUUID("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa0");
    user.setUsername("koko");

    Product product = dao.getProductByUuid(valid);
    Assert.assertNotNull(product);
    Assert.assertEquals(product.getId().intValue(), 6);

    product = dao.getProductByUuid(invalid);
    Assert.assertNull(product);

    product = dao.getProductByUuid(valid);
    Assert.assertNotNull(product);
    Assert.assertEquals(product.getId().intValue(), 6);

    product = dao.getProductByUuid(null);
    Assert.assertNull(product);
  }
コード例 #16
0
  @Test(groups = {"non-regression"})
  public void testChecksumUpdate() throws MalformedURLException {
    Product product = new Product();
    product.setPath(new URL("file:/product/path"));

    Download download = new Product.Download();
    download.setPath("/no/path/file");
    download.setSize(0L);
    download.setType("application/octet-stream");
    download.setChecksums(
        Maps.newHashMap(
            ImmutableMap.of(
                "MD5", "54ABCDEF98765",
                "SHA-1", "9876FEDCBA1234")));

    product.setDownload(download);

    // First create the defined product:
    try {
      product = dao.create(product);
    } catch (Exception e) {
      Assert.fail("Creation of product fails", e);
    }

    /** Clear/putAll feature testing */
    product.getDownload().getChecksums().clear();
    product
        .getDownload()
        .getChecksums()
        .putAll(
            Maps.newHashMap(
                ImmutableMap.of(
                    "SHA-256", "4554ABCDEF98765",
                    "SHA-512", "ABDEFFE9876FEDCBA1234")));
    try {
      dao.update(product);
    } catch (Exception e) {
      Assert.fail("Modifying checksums with map clear/put fails", e);
    }

    /** Set feature testing */
    product
        .getDownload()
        .setChecksums(
            Maps.newHashMap(
                ImmutableMap.of(
                    "MD5", "54ABCDEF98765",
                    "SHA-1", "9876FEDCBA1234")));
    try {
      dao.update(product);
    } catch (Exception e) {
      Assert.fail("Modifying checksums with \"set\" fails", e);
    }

    /** Remove residuals for this test */
    cancelListeners(getHibernateDao());
    dao.delete(product);
  }
コード例 #17
0
  @Override
  public void update() {
    String productIdentifier = "test-prod-7";
    String indexName = "updatable";
    Long pid = Long.valueOf(7);

    Product product = dao.read(pid);
    List<MetadataIndex> indexes = product.getIndexes();
    product.setIdentifier(productIdentifier);
    for (MetadataIndex mi : indexes) {
      mi.setName(indexName);
    }
    dao.setIndexes(pid, indexes);
    dao.update(product);

    product = dao.read(pid);
    indexes = product.getIndexes();
    Assert.assertNotNull(product);
    Assert.assertEquals(product.getIdentifier(), productIdentifier);
    for (MetadataIndex mi : indexes) {
      Assert.assertEquals(mi.getName(), indexName);
    }
  }
コード例 #18
0
    @Transactional(readOnly = false, propagation = Propagation.REQUIRED)
    @Caching(
        evict = {
          @CacheEvict(value = "product", allEntries = true),
          @CacheEvict(value = "products", allEntries = true)
        })
    @IncrementCache(name = "product_count", key = "all", value = 1)
    public void run() {
      if (scanner != null && scanner.isStopped()) {
        logger.info("Scanner stopped, deleting product #" + product.getId());
        if (wrapper != null) {
          wrapper.error(product, new InterruptedException("Processing stopped by the user"));
        }
        productDao.delete(product);
        return;
      }
      logger.debug("Add product \"" + ProductDao.getPathFromProduct(product) + "\".");

      try {
        long processing_start = System.currentTimeMillis();

        if (wrapper != null) {
          wrapper.startIngestion();
        }

        processingManager.process(product);
        productDao.update(product);
        searchService.index(product);

        if (collections != null) {
          for (Collection c : collections) {
            collectionService.systemAddProduct(c.getId(), product.getId(), true);
          }
        }

        if (wrapper != null) {
          wrapper.endIngestion();
        }

        long processing_end = System.currentTimeMillis();
        logger.info(
            "Ingestion processing complete for product "
                + product.getPath().toExternalForm()
                + " ("
                + product.getSize()
                + " bytes, "
                + product.getDownloadableSize()
                + " bytes compressed)"
                + " in "
                + (processing_end - processing_start)
                + "ms.");

        actionRecordWritterDao.uploadEnd(
            this.product.getPath(), this.owner.getUsername(), collections, true);
      } catch (Exception excp) {
        logger.warn(
            "Unrecoverable error happen during ingestion of "
                + product.getPath()
                + " (removing from database)",
            excp);
        try {
          productDao.delete(product);
        } catch (Exception e) {
          logger.error("Unable to remove product after ingestion failure", e);
        }
        if (wrapper != null) {
          wrapper.error(product, excp);
        }

        if ((this.product.getPath() != null) && (this.owner != null)) {
          actionRecordWritterDao.uploadFailed(
              this.product.getPath().toString(), this.owner.getUsername());
          actionRecordWritterDao.uploadEnd(
              this.product.getPath(), this.owner.getUsername(), collections, false);
        }
        return;
      }
    }