Пример #1
0
 // Check if product present is the DB is still present into the repository.
 @Transactional(readOnly = false, propagation = Propagation.REQUIRED)
 public void checkDBProducts() {
   logger.info("Syncing database with repositories...");
   Iterator<Product> products = productDao.getAllProducts();
   while (products.hasNext()) {
     Product product = products.next();
     if (!ProductService.checkUrl(product.getPath())) {
       logger.info("Removing Product " + product.getPath() + " not found in repository.");
       products.remove();
     } else logger.info("Product " + product.getPath() + " found in repository.");
   }
 }
Пример #2
0
  /**
   * Do process all products in database that their ingestion is not finished. If provided parameter
   * is null, default processing consists in removing the data from database. Otherwise, the
   * provided processing is launched.
   *
   * @param proc the processing to execute. if null, remove processing will be performed.
   * @return the list of products reprocessed.
   */
  @Transactional(readOnly = false, propagation = Propagation.REQUIRED)
  @CacheEvict(
      value = {"product_count", "product", "products"},
      allEntries = true)
  public void processUnprocessed(boolean recover) {
    long start = new Date().getTime();

    if (!recover) {
      Iterator<Product> products = getUnprocessedProducts();
      while (products.hasNext()) {
        products.next();
        products.remove();
      }

      logger.debug(
          "Cleanup incomplete processed products in " + (new Date().getTime() - start) + "ms");
    } else {
      Iterator<Product> products = getUnprocessedProducts();
      while (products.hasNext()) {
        Product product = products.next();
        // Do reporcess only already transfered products
        if (product.getPath().toString().equals(product.getOrigin())) {
          products.remove();
        } else {
          try {
            String path = product.getPath().getPath();
            logger.info("Recovering product from " + path);
            // Check if product is still present in repository
            if (!new File(path).exists()) {
              throw new DataStoreException("Product " + path + " not present locally.");
            }
            // Retrieve owner if any
            User owner = productDao.getOwnerOfProduct(product);

            // Retrieve collections
            List<Collection> collections = collectionDao.getCollectionsOfProduct(product.getId());

            processProduct(product, owner, collections, null, null);
          } catch (Exception e) {
            logger.error("Error while processing: " + e.getMessage() + "- abort reprocessing.");
            products.remove();
          }
        }
      }
    }
  }
Пример #3
0
  /**
   * Adds a product in the database, the given product will not be queued for processing nor it will
   * be submitted to the search engine.
   *
   * @param product a product to store in the database.
   * @return the created product.
   * @throws IllegalArgumentException incomplete products are not allowed.
   */
  @Transactional(readOnly = false, propagation = Propagation.REQUIRED)
  @Caching(
      evict = {
        @CacheEvict(value = "product", allEntries = true),
        @CacheEvict(value = "products", allEntries = true)
      })
  @IncrementCache(name = "product_count", key = "all", value = 1)
  public Product addProduct(Product product) throws IllegalArgumentException {
    URL path = product.getPath();
    String origin = product.getOrigin();
    if (path == null || origin == null || origin.isEmpty()) {
      throw new IllegalArgumentException("product must have a path and an origin");
    }
    // FIXME do I have to check every field? isn't it done by hibernate based on column constraints?

    Product final_product = this.productDao.create(product);
    return final_product;
  }
Пример #4
0
    @Transactional(readOnly = false, propagation = Propagation.REQUIRED)
    @Caching(
        evict = {
          @CacheEvict(value = "product", allEntries = true),
          @CacheEvict(value = "products", allEntries = true)
        })
    @IncrementCache(name = "product_count", key = "all", value = 1)
    public void run() {
      if (scanner != null && scanner.isStopped()) {
        logger.info("Scanner stopped, deleting product #" + product.getId());
        if (wrapper != null) {
          wrapper.error(product, new InterruptedException("Processing stopped by the user"));
        }
        productDao.delete(product);
        return;
      }
      logger.debug("Add product \"" + ProductDao.getPathFromProduct(product) + "\".");

      try {
        long processing_start = System.currentTimeMillis();

        if (wrapper != null) {
          wrapper.startIngestion();
        }

        processingManager.process(product);
        productDao.update(product);
        searchService.index(product);

        if (collections != null) {
          for (Collection c : collections) {
            collectionService.systemAddProduct(c.getId(), product.getId(), true);
          }
        }

        if (wrapper != null) {
          wrapper.endIngestion();
        }

        long processing_end = System.currentTimeMillis();
        logger.info(
            "Ingestion processing complete for product "
                + product.getPath().toExternalForm()
                + " ("
                + product.getSize()
                + " bytes, "
                + product.getDownloadableSize()
                + " bytes compressed)"
                + " in "
                + (processing_end - processing_start)
                + "ms.");

        actionRecordWritterDao.uploadEnd(
            this.product.getPath(), this.owner.getUsername(), collections, true);
      } catch (Exception excp) {
        logger.warn(
            "Unrecoverable error happen during ingestion of "
                + product.getPath()
                + " (removing from database)",
            excp);
        try {
          productDao.delete(product);
        } catch (Exception e) {
          logger.error("Unable to remove product after ingestion failure", e);
        }
        if (wrapper != null) {
          wrapper.error(product, excp);
        }

        if ((this.product.getPath() != null) && (this.owner != null)) {
          actionRecordWritterDao.uploadFailed(
              this.product.getPath().toString(), this.owner.getUsername());
          actionRecordWritterDao.uploadEnd(
              this.product.getPath(), this.owner.getUsername(), collections, false);
        }
        return;
      }
    }