예제 #1
0
  public static void main(String[] args) {

    AmazonDynamoDBClient client = new AmazonDynamoDBClient(new ProfileCredentialsProvider());
    client.setRegion(Region.getRegion(Regions.US_WEST_2));

    DynamoDBMapper mapper = new DynamoDBMapper(client);

    // retrieve an object using a "normal" (non-composite) id
    Movie m = new Movie();
    m.setTitle("foxy brown");
    Movie m2 = mapper.load(m);
    if (m2.getExtra() != null) {
      System.out.println("title:" + m2.getTitle() + " extra:" + m2.getExtra());
    }

    // retrieve an object using a composite key
    Artist a = new Artist();
    a.setId(5);
    a.setName("Van Gogh");
    Artist a2 = mapper.load(a);

    // scan through a bunch of entries in a table and print them out
    ScanRequest scanRequest = new ScanRequest().withTableName("Artists");
    ScanResult result = client.scan(scanRequest);
    for (Map<String, AttributeValue> item : result.getItems()) {
      String o1 = item.get("id").getN();
      String o2 = item.get("name").getS();
      Artist artist = mapper.load(Artist.class, new Integer(o1), o2);
      artist.dump();
    }
  }
  /**
   * This class relies heavily on the behavior of segmented scans with respect to which hash keys
   * are scanned by each segment. Here's a rough ASCII example to help illustrate:
   * ___________________________ |hk:A |hk:B | ---------------------------- ^segment 1 ^segment 2
   *
   * <p>Because we are scanning in segments across the entire hash key space, it is possible for the
   * same hash key to appear in two different segments. We are also running all of the scan segments
   * in parallel, so we have no control over which segment returns first.
   *
   * <p>In the example, if segment 2 was the first segment to post a result, we would store hk:B as
   * a "boundary" key. That way when segment 1 eventually reaches hk:B in its scan, we know that
   * another segment has already returned this hash key and we can safely skip returning it.
   *
   * <p>By doing this, we avoid returning a RecordIterator for the same hash key twice and we only
   * need to store at most 2 hash keys per segment.
   */
  @Override
  public List<SingleKeyRecordIterator> buildRecordIterators(ScanContext scanContext) {
    final ScanResult dynamoDbResult = scanContext.getScanResult();
    final int segment = scanContext.getScanRequest().getSegment();
    final List<Map<String, AttributeValue>> items = dynamoDbResult.getItems();
    // If the scan returned no results, we need to shortcut and just throw back an empty result set
    if (items.isEmpty()) {
      return Collections.emptyList();
    }

    List<SingleKeyRecordIterator> recordIterators = Lists.newLinkedList();

    final Iterator<Map<String, AttributeValue>> itemIterator = items.iterator();
    final Map<String, AttributeValue> firstItem = itemIterator.next();
    final StaticBuffer firstKey = new KeyBuilder(firstItem).build(Constants.TITAN_HASH_KEY);

    // Computes the full set of boundary keys up to this point. This includes the previous end key
    // for this segment.
    final ImmutableSet<StaticBuffer> boundaryKeys = aggregateBoundaryKeys();

    // The first key in this scan segment might already have been returned by a previous scan
    // segment
    if (!boundaryKeys.contains(firstKey)) {
      recordIterators.add(buildRecordIteratorForHashKey(firstKey));
    }

    StaticBuffer hashKey = firstKey;
    while (itemIterator.hasNext()) {
      final Optional<StaticBuffer> nextKey = findNextHashKey(itemIterator, hashKey);
      if (nextKey.isPresent()) {
        // Found a new hash key. Make a record iterator and look for the next unique hash key
        hashKey = nextKey.get();
        recordIterators.add(buildRecordIteratorForHashKey(hashKey));
      }
    }

    // If we've already seen the final hashKey in a previous scan segment result, we want to avoid
    // returning it again.
    if (!hashKey.equals(firstKey) && boundaryKeys.contains(hashKey)) {
      recordIterators.remove(recordIterators.size() - 1);
    }

    // Update the boundary keys for this segment
    if (scanContext.isFirstResult()) {
      setInitialBoundaryKeys(segment, firstKey, hashKey);
    } else {
      updateLastKey(segment, hashKey);
    }
    return recordIterators;
  }
예제 #3
0
  /** @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response) */
  protected void doGet(HttpServletRequest request, HttpServletResponse response)
      throws ServletException, IOException {
    String category = request.getParameter("category");
    HashMap<String, Condition> scanFilter = new HashMap<String, Condition>();
    Condition condition =
        new Condition()
            .withComparisonOperator(ComparisonOperator.NE.toString())
            .withAttributeValueList(new AttributeValue().withN("0"));
    Condition condition2 =
        new Condition()
            .withComparisonOperator(ComparisonOperator.NE.toString())
            .withAttributeValueList(new AttributeValue().withN("0"));
    Condition condition3 = new Condition().withComparisonOperator(ComparisonOperator.CONTAINS);

    scanFilter.put("geoLat", condition);
    scanFilter.put("geoLng", condition2);
    if (category != null && !category.isEmpty()) {
      condition3.withAttributeValueList(new AttributeValue().withS(category));
      scanFilter.put("category", condition3);
    }

    // String queueUrl = SQS.getQueueUrl(new GetQueueUrlRequest(SQS_QUEUE_NAME)).getQueueUrl();

    String tableName = DYNAMODB_TABLE_NAME;
    ScanRequest scanRequest = new ScanRequest(tableName).withScanFilter(scanFilter);
    ScanResult scanResult = DYNAMODB.scan(scanRequest);

    int size = scanResult.getItems().size();
    ArrayList<HashMap<String, String>> tweets = new ArrayList<HashMap<String, String>>();

    for (int i = 0; i < size; i++) {
      // Get latitude, longitude, content, username, created (long), category, sentiment
      String categorydb = "no category";
      String sentiment = "no sentiment";
      if (scanResult.getItems().get(i).get("category") != null) {
        categorydb = scanResult.getItems().get(i).get("category").getS();
        sentiment = scanResult.getItems().get(i).get("sentiment").getS();
        String lat = scanResult.getItems().get(i).get("geoLat").getN();
        String lng = scanResult.getItems().get(i).get("geoLng").getN();
        String content = scanResult.getItems().get(i).get("content").getS();
        String username = scanResult.getItems().get(i).get("username").getS();
        String created = scanResult.getItems().get(i).get("createdLong").getN();
        String createdDate = scanResult.getItems().get(i).get("createdDate").getS();

        // Format date.
        DateFormat fromFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
        DateFormat toFormat = new SimpleDateFormat("MMM dd · k:mm z");
        Date date;
        String createdstr;
        try {
          date = fromFormat.parse(createdDate);
          createdstr = toFormat.format(date);
        } catch (ParseException e) {
          createdstr = createdDate;
        }

        // Create tweet hash.
        HashMap<String, String> tweet = new HashMap<String, String>();
        tweet.put("lat", lat);
        tweet.put("lng", lng);
        tweet.put("content", content);
        tweet.put("username", username);
        tweet.put("category", categorydb);
        tweet.put("sentiment", sentiment);
        tweet.put("created", created);
        tweet.put("createdstr", createdstr);

        // Order tweet by time created. Most recent at the top of the list.
        int position = 0;
        while (position < tweets.size()
            && Long.parseLong(tweets.get(position).get("created")) > Long.parseLong(created)) {
          position++;
        }
        tweets.add(position, tweet);
      }
      // else {
      // Send tweet with blank category for sentimental processing
      //    String id = scanResult.getItems().get(i).get("id").getS();
      //    SQS.sendMessage(new SendMessageRequest(queueUrl, id));
      // }

    }

    // Log result.
    System.out.println("Successfully handled GET request.");
    if (category != null) {
      System.out.println("category:" + category);
    }
    System.out.println("size: " + tweets.size());

    // Convert object to JSON format.
    String json = new Gson().toJson(tweets);
    response.setContentType("application/json");
    response.setCharacterEncoding("UTF-8");
    response.getWriter().write(json);
  }