@Override
 @Transactional(readOnly = false)
 public Collection updateAndGetRunningCollectionStatusByUser(Long userId) throws Exception {
   Collection collection = collectionRepository.getRunningCollectionStatusByUser(userId);
   if (collection != null) {
     logger.info(
         "User with ID: '"
             + userId
             + "' has a running collection with code: '"
             + collection.getCode());
     return statusByCollection(collection, userId);
   } else {
     // logger.info("User with ID: '" + userId + "' don't have any running collections. Nothing to
     // update." );
     //            If there is no running collection there is still can be collection with status
     // 'Initializing'.
     //            This happens because we update collection information from fetcher before
     // collection was started.
     //            So we need to update from Fetcher this kind of collections as well.
     collection = collectionRepository.getInitializingCollectionStatusByUser(userId);
     if (collection != null) {
       return statusByCollection(collection, userId);
     }
   }
   return null;
 }
  // @SuppressWarnings("deprecation")
  @Transactional
  @Override
  public Collection statusByCollection(Collection collection, Long accountId) throws Exception {
    if (collection != null) {
      try {
        /** Make a call to fetcher Status Rest API */
        Client client = ClientBuilder.newBuilder().register(JacksonFeature.class).build();

        String path = "";
        if (CollectionType.Twitter.equals(collection.getProvider())
            || CollectionType.Facebook.equals(collection.getProvider())) {
          path = "/" + collection.getProvider().toString().toLowerCase() + "/status?id=";
        } else if (CollectionType.SMS.equals(collection.getProvider())) {
          path = "/sms/status?collection_code=";
        }

        WebTarget webResource =
            client.target(fetchMainUrl + path + URLEncoder.encode(collection.getCode(), "UTF-8"));
        Response clientResponse = webResource.request(MediaType.APPLICATION_JSON).get();

        String jsonString = clientResponse.readEntity(String.class);
        JSONParser parser = new JSONParser();
        JSONObject jsonResponse = (JSONObject) parser.parse(jsonString);
        collection =
            updateStatusCollection(jsonResponse.get("entity").toString(), collection, accountId);
        return collection;
      } catch (Exception e) {
        String msg = "Error while getting status for collection from Remote FetchMain Collection";
        logger.error(msg, e);
        throw new Exception(msg);
      }
    }
    return null;
  }
  @Override
  @Transactional(readOnly = false)
  public Collection stopAidrFetcher(Collection collection, Long userId) {
    try {
      /** Rest call to Fetcher */
      Client client = ClientBuilder.newBuilder().register(JacksonFeature.class).build();
      String path = "";
      if (CollectionType.SMS.equals(collection.getProvider())) {
        path = "/sms/stop?collection_code=";
      } else {
        path = "/" + collection.getProvider().toString().toLowerCase() + "/stop?id=";
      }

      WebTarget webResource =
          client.target(fetchMainUrl + path + URLEncoder.encode(collection.getCode(), "UTF-8"));

      Response clientResponse = webResource.request(MediaType.APPLICATION_JSON).get();

      String jsonString = clientResponse.readEntity(String.class);
      JSONParser parser = new JSONParser();
      JSONObject jsonResponse = (JSONObject) parser.parse(jsonString);
      collection =
          updateStatusCollection(jsonResponse.get("entity").toString(), collection, userId);

      /** Change Database Status */
      return this.collectionRepository.stop(collection.getId());
    } catch (Exception e) {
      logger.error("Error while stopping Remote FetchMain Collection", e);
    }
    return null;
  }
  @Override
  @Transactional
  public Collection create(CollectionDetailsInfo collectionDetailsInfo, UserAccount user)
      throws Exception {

    String filteredTrack = collectionDetailsInfo.getTrack();

    if (!StringUtils.isEmpty(filteredTrack)) {
      filteredTrack = getFilteredTrack(filteredTrack);

      if (StringUtils.isEmpty(filteredTrack)) {
        return null;
      }
    }

    Collection collection = adaptCollectionDetailsInfoToCollection(collectionDetailsInfo, user);
    collection.setTrack(filteredTrack);
    collection.setUsageType(UsageType.Production);
    try {
      collectionRepository.save(collection);
      collaboratorService.addCollaboratorToCollection(
          collectionDetailsInfo.getCode(), user.getId());
      return collection;
    } catch (Exception e) {

      logger.error("Error in creating collection.", e);
      return null;
    }
  }
 @Override
 @Transactional(readOnly = true)
 public Boolean isValidAPIKey(String code, String apiKey) throws Exception {
   Collection collection = findByCode(code);
   if (collection.getOwner().getApiKey().equals(apiKey)) {
     return true;
   }
   return false;
 }
 @Override
 @Transactional(readOnly = true)
 public Collection findTrashedByCode(String code) throws Exception {
   Collection temp = collectionRepository.findByCode(code);
   if (temp.getStatus().equals(CollectionStatus.TRASHED)) {
     return temp;
   }
   return null;
 }
  private CollectionBriefInfo adaptCollectionToCollectionBriefInfo(Collection collection) {

    CollectionBriefInfo briefInfo = new CollectionBriefInfo();
    briefInfo.setCollectionId(collection.getId());
    briefInfo.setCollectionCode(collection.getCode());
    briefInfo.setCollectionName(collection.getName());
    briefInfo.setMicromappersEnabled(collection.isMicromappersEnabled());
    briefInfo.setProvider(collection.getProvider().name());
    return briefInfo;
  }
  @Override
  @Transactional(readOnly = false)
  public Collection start(Long collectionId) throws Exception {

    // We are going to start new collection. Lets stop collection which is running for owner of the
    // new collection.
    Collection dbCollection = collectionRepository.findById(collectionId);
    Long userId = dbCollection.getOwner().getId();
    Collection alreadyRunningCollection =
        collectionRepository.getRunningCollectionStatusByUser(userId);
    if (alreadyRunningCollection != null) {
      this.stop(alreadyRunningCollection.getId(), userId);
    }

    return startFetcher(prepareFetcherRequest(dbCollection), dbCollection);
  }
  @Override
  @Transactional(readOnly = false)
  public Collection stop(Long collectionId, Long userId) throws Exception {
    Collection collection = collectionRepository.findById(collectionId);

    // Follwoing 2 lines added by koushik for downloadCount bug
    Collection c = this.statusByCollection(collection, userId);
    collection.setCount(c.getCount());

    Collection updateCollection = stopAidrFetcher(collection, userId);

    CollectionLog collectionLog = new CollectionLog(collection);
    collectionLog.setUpdatedBy(userId);
    collectionLogRepository.save(collectionLog);

    return updateCollection;
  }
示例#10
0
  @Override
  @Transactional(readOnly = false)
  public boolean enableClassifier(String code, UserAccount currentUser) {

    try {
      Collection collection = findByCode(code);
      if (collection != null) {
        collection.setClassifierEnabled(Boolean.TRUE);
        collection.setClassifierEnabledBy(currentUser);

        this.update(collection);
      }

      return Boolean.TRUE;
    } catch (Exception e) {
      logger.error("Error in enabling classifier for code : " + code, e);
      return Boolean.FALSE;
    }
  }
示例#11
0
  @Override
  @Transactional(readOnly = false)
  public Collection startFetcher(FetcherRequestDTO fetcherRequest, Collection collection) {
    try {
      /** Rest call to Fetcher */
      Client client = ClientBuilder.newBuilder().register(JacksonFeature.class).build();

      if (CollectionType.Twitter.equals(collection.getProvider())
          || CollectionType.Facebook.equals(collection.getProvider())) {
        WebTarget webResource =
            client.target(
                fetchMainUrl + "/" + collection.getProvider().toString().toLowerCase() + "/start");

        ObjectMapper objectMapper = JacksonWrapper.getObjectMapper();

        Response clientResponse =
            webResource
                .request(MediaType.APPLICATION_JSON)
                .post(Entity.json(objectMapper.writeValueAsString(fetcherRequest)), Response.class);

        // logger.info("ObjectMapper: " + objectMapper.writeValueAsString(fetcherRequest));
        String jsonString = clientResponse.readEntity(String.class);
        JSONParser parser = new JSONParser();
        JSONObject jsonResponse = (JSONObject) parser.parse(jsonString);
        // logger.info("NEW STRING: " + jsonResponse);
        FetcheResponseDTO response =
            objectMapper.readValue(jsonResponse.get("entity").toString(), FetcheResponseDTO.class);
        logger.info("start Response from fetchMain " + objectMapper.writeValueAsString(response));
        collection.setStatus(CollectionStatus.getByStatus(response.getStatusCode()));
      } else if (CollectionType.SMS.equals(collection.getProvider())) {
        WebTarget webResource =
            client.target(
                fetchMainUrl
                    + "/sms/start?collection_code="
                    + URLEncoder.encode(collection.getCode(), "UTF-8"));
        Response response = webResource.request(MediaType.APPLICATION_JSON).get();
        if (response.getStatus() == 200) collection.setStatus(CollectionStatus.INITIALIZING);
      }
      /** Update Status To database */
      collectionRepository.update(collection);
      return collection;
    } catch (Exception e) {
      logger.error("Error while starting Remote FetchMain Collection", e);
    }
    return null;
  }
示例#12
0
  private Collection adaptCollectionDetailsInfoToCollection(
      CollectionDetailsInfo collectionInfo, UserAccount user) {

    Collection collection = new Collection();
    collection.setDurationHours(collectionInfo.getDurationHours());
    collection.setCode(collectionInfo.getCode());
    collection.setName(collectionInfo.getName());
    collection.setClassifierEnabled(false);
    collection.setProvider(CollectionType.valueOf(collectionInfo.getProvider()));
    collection.setOwner(user);
    collection.setStatus(CollectionStatus.NOT_RUNNING);
    collection.setPubliclyListed(true); // TODO: change default behavior to user choice

    collection.setGeoR(collectionInfo.getGeoR());
    collection.setGeo(collectionInfo.getGeo());
    collection.setTrack(collectionInfo.getTrack());
    collection.setCrisisType(crisisTypeService.getById(collectionInfo.getCrisisType()));
    collection.setFollow(collectionInfo.getFollow());
    collection.setLangFilters(collectionInfo.getLangFilters());
    collection.setMicromappersEnabled(Boolean.FALSE);
    collection.setProvider(CollectionType.valueOf(collectionInfo.getProvider()));
    collection.setPurpose(collectionInfo.getPurpose());
    collection.setFetchInterval(collectionInfo.getFetchInterval());
    collection.setFetchFrom(collectionInfo.getFetchFrom());

    if (CollectionType.SMS.equals(collectionInfo.getProvider())) {
      collection.setTrack(null);
      collection.setLangFilters(null);
      collection.setGeo(null);
      collection.setFollow(null);
    }

    Timestamp now = new Timestamp(System.currentTimeMillis());
    collection.setCreatedAt(now);
    collection.setUpdatedAt(now);
    return collection;
  }
示例#13
0
  private Collection updateStatusCollection(
      String jsonResponse, Collection collection, Long accountId) throws Exception {
    ObjectMapper objectMapper = JacksonWrapper.getObjectMapper();
    FetcheResponseDTO response = objectMapper.readValue(jsonResponse, FetcheResponseDTO.class);
    if (response != null) {
      // MEGHNA: moved setting collection count to top of the method
      // to avoid individual status blocks setting collection count below
      if (response.getCollectionCount() != null
          && !response.getCollectionCount().equals(collection.getCount())) {
        collection.setCount(response.getCollectionCount());
        String lastDocument = response.getLastDocument();
        collection.setLastExecutionTime(response.getLastExecutionTime());
        if (lastDocument != null) collection.setLastDocument(lastDocument);
        collectionRepository.update(collection);
      }
      collection.setSourceOutage(response.isSourceOutage());
      if (!CollectionStatus.getByStatus(response.getStatusCode()).equals(collection.getStatus())) {

        CollectionStatus prevStatus = collection.getStatus();
        collection.setStatus(CollectionStatus.getByStatus(response.getStatusCode()));

        switch (CollectionStatus.getByStatus(response.getStatusCode())) {
          case NOT_FOUND:
            // case STOPPED:
            collection.setStatus(CollectionStatus.NOT_RUNNING);

            // Add collectionCount in collectionLog if it was not recorded.
            if (collection.getStartDate() != null
                && ((collection.getEndDate() != null
                        && collection.getStartDate().after(collection.getEndDate()))
                    || collection.getEndDate() == null)) {
              if (collectionLogRepository.countLogsStartedInInterval(
                      collection.getId(), collection.getStartDate(), new Date())
                  == 0) {
                CollectionLog collectionLog = new CollectionLog(collection);
                collectionLog.setEndDate(new Date());
                collectionLog.setUpdatedBy(accountId);
                collectionLogRepository.save(collectionLog);
              }
            }
          case RUNNING_WARNING:
            if (prevStatus == CollectionStatus.INITIALIZING) {
              collection = collectionRepository.start(collection.getId());
              break;
            }
          case WARNING:
            collectionRepository.update(collection);
            break;
          case RUNNING:
            collection = collectionRepository.start(collection.getId());
            break;
          case FATAL_ERROR:
            // collection = collectionRepository.stop(collection.getId());
            logger.warn("Fatal error, stopping collection " + collection.getId());
            if (prevStatus != CollectionStatus.FATAL_ERROR
                || prevStatus != CollectionStatus.NOT_RUNNING
                || prevStatus != CollectionStatus.STOPPED)
              this.stopFatalError(collection.getId(), accountId);
            break;
          case EXCEPTION:
            logger.warn(
                "Rejected Thread Execution Exception, restarting collection " + collection.getId());
            if (prevStatus != CollectionStatus.EXCEPTION) {
              this.stopFatalError(collection.getId(), accountId);
              this.start(collection.getId());
            }
            break;
          default:
            break;
        }
      }
    }
    return collection;
  }
示例#14
0
  @Override
  @Transactional(readOnly = true)
  public FetcherRequestDTO prepareFetcherRequest(Collection dbCollection) {
    FetcherRequestDTO dto = new FetcherRequestDTO();

    UserConnection userconnection =
        userConnectionService.fetchByCombinedUserName(dbCollection.getOwner().getUserName());
    dto.setAccessToken(userconnection.getAccessToken());
    dto.setAccessTokenSecret(userconnection.getSecret());
    dto.setCollectionName(dbCollection.getName());
    dto.setCollectionCode(dbCollection.getCode());
    if (dbCollection.getProvider() == CollectionType.Facebook) {
      dto.setToFollow(dbCollection.getFollow());
    } else {
      dto.setToFollow(
          getFollowTwitterIDs(dbCollection.getFollow(), dbCollection.getOwner().getUserName()));
    }
    dto.setToTrack(dbCollection.getTrack());
    dto.setGeoLocation(dbCollection.getGeo());
    dto.setGeoR(dbCollection.getGeoR());
    dto.setLanguageFilter(dbCollection.getLangFilters());
    dto.setSaveMediaEnabled(dbCollection.isSaveMediaEnabled());
    dto.setFetchInterval(dbCollection.getFetchInterval());
    dto.setProvider(dbCollection.getProvider().toString());
    dto.setFetchInterval(dbCollection.getFetchInterval());
    dto.setFetchFrom(dbCollection.getFetchFrom());
    dto.setLastExecutionTime(dbCollection.getLastExecutionTime());
    // Added by koushik
    accessTokenStr = dto.getAccessToken();
    accessTokenSecretStr = dto.getAccessTokenSecret();

    return dto;
  }
示例#15
0
  @Override
  @Transactional(readOnly = false)
  public boolean updateCollection(CollectionUpdateInfo collectionUpdateInfo, Long userId) {

    String filteredTrack = "";
    try {
      Collection collection = findByCode(collectionUpdateInfo.getCode());

      if (!collection.getName().equals(collectionUpdateInfo.getName())) {
        if (!existName(collectionUpdateInfo.getName())) {
          collection.setName(collectionUpdateInfo.getName());
        } else {
          return false;
        }
      }
      // if collection exists with same name

      collection.setProvider(CollectionType.valueOf(collectionUpdateInfo.getProvider()));
      collection.setFollow(collectionUpdateInfo.getFollow());
      collection.setFetchInterval(collectionUpdateInfo.getFetchInterval());
      collection.setFetchFrom(collectionUpdateInfo.getFetchFrom());
      filteredTrack = collectionUpdateInfo.getTrack();

      if (!StringUtils.isEmpty(filteredTrack)) {
        filteredTrack = getFilteredTrack(filteredTrack);

        if (StringUtils.isEmpty(filteredTrack)) {
          return false;
        }
      }
      collection.setTrack(filteredTrack);
      collection.setGeo(collectionUpdateInfo.getGeo());
      collection.setGeoR(collectionUpdateInfo.getGeoR());
      collection.setDurationHours(Integer.parseInt(collectionUpdateInfo.getDurationHours()));
      collection.setLangFilters(collectionUpdateInfo.getLangFilters());

      Long crisisTypeId = Long.parseLong(collectionUpdateInfo.getCrisisType());
      CrisisType crisisType = crisisTypeService.getById(crisisTypeId);
      if (crisisType != null) {
        collection.setCrisisType(crisisType);
      } else {
        logger.error(
            "Crisis Type Id: "
                + crisisTypeId
                + " does not exist. Can't update the collection : "
                + collectionUpdateInfo.getCode());
        return false;
      }

      if (CollectionType.SMS.equals(collection.getProvider())) {
        collection.setTrack(null);
        collection.setLangFilters(null);
        collection.setGeo(null);
        collection.setGeoR(null);
        collection.setFollow(null);
      }
      if (collection.getProvider() == CollectionType.Twitter) {
        collection.setFollow(
            this.getFollowTwitterIDs(
                collectionUpdateInfo.getFollow(), collection.getOwner().getUserName()));
      }

      collectionRepository.update(collection);
      // first make an entry in log if collection is running
      if (CollectionStatus.RUNNING_WARNING.equals(collection.getStatus())
          || CollectionStatus.RUNNING.equals(collection.getStatus())) {

        this.stop(collection.getId(), userId);
        this.startFetcher(this.prepareFetcherRequest(collection), collection);
      }
      return true;
    } catch (Exception e) {

      logger.error("Unable to update the collection : " + collectionUpdateInfo.getCode(), e);
      return false;
    }
  }
示例#16
0
  private CollectionSummaryInfo adaptCollectionToCollectionSummaryInfo(Collection collection) {

    CollectionSummaryInfo summaryInfo = new CollectionSummaryInfo();
    summaryInfo.setCode(collection.getCode());
    summaryInfo.setName(collection.getName());
    summaryInfo.setCurator(collection.getOwner().getUserName());
    summaryInfo.setStartDate(collection.getStartDate());
    summaryInfo.setEndDate(collection.getEndDate());
    summaryInfo.setCollectionCreationDate(collection.getCreatedAt());
    // TODO to fetch from collection log
    try {
      summaryInfo.setTotalCount(
          collectionLogService.countTotalDownloadedItemsForCollection(collection.getId()));
    } catch (Exception e) {
      logger.warn("Error in fetch count from collection log.", e);
      summaryInfo.setTotalCount(collection.getCount());
    }
    summaryInfo.setStatus(collection.getStatus().getStatus());

    // TODO summaryInfo.setCreatedAt(collection.getCreatedAt());
    summaryInfo.setLanguage(collection.getLangFilters());
    summaryInfo.setKeywords(collection.getTrack());
    summaryInfo.setGeo(collection.getGeo());
    summaryInfo.setLabelCount(taggerService.getLabelCount(collection.getId()));
    summaryInfo.setPubliclyListed(collection.isPubliclyListed());
    summaryInfo.setProvider(collection.getProvider().toString());
    return summaryInfo;
  }