public static boolean checkIfb4June2015(String month) { Calendar ret = Calendar.getInstance(); GeneralHelpers.setDateByString(ret, ConstantsTruthy.b4BigIntMonth); long june2015 = ret.getTimeInMillis(); GeneralHelpers.setDateByString(ret, month); long currentMonth = ret.getTimeInMillis(); return (currentMonth > june2015); }
/** * Set start and end calendars of an interval by their string representations. * * @param calStart * @param calEnd * @param startTime * @param endTime */ public static void setStartAndEndCalendar( Calendar calStart, Calendar calEnd, String startTime, String endTime) { if (startTime.indexOf('T') >= 0) { GeneralHelpers.setDateTimeByString(calStart, startTime); } else { GeneralHelpers.setDateByString(calStart, startTime); } if (endTime.indexOf('T') >= 0) { GeneralHelpers.setDateTimeByString(calEnd, endTime); calEnd.set(Calendar.MILLISECOND, 999); } else { GeneralHelpers.setDateByString(calEnd, endTime); calEnd.set(Calendar.HOUR_OF_DAY, 23); calEnd.set(Calendar.MINUTE, 59); calEnd.set(Calendar.SECOND, 59); calEnd.set(Calendar.MILLISECOND, 999); } }
/** * Split a long time window to a sequence of small windows, each covering one month of the whole * window. * * @param startTime * @param endTime * @return */ public static Map<String, long[]> splitTimeWindowToMonths(String startTime, String endTime) { Map<String, long[]> ret = new TreeMap<String, long[]>(); Calendar calStart = Calendar.getInstance(TimeZone.getTimeZone(Constants.TIME_ZONE_GMT)); Calendar calEnd = Calendar.getInstance(TimeZone.getTimeZone(Constants.TIME_ZONE_GMT)); Calendar calTmp = Calendar.getInstance(TimeZone.getTimeZone(Constants.TIME_ZONE_GMT)); setStartAndEndCalendar(calStart, calEnd, startTime, endTime); boolean firstMonth = true; while (calStart.get(Calendar.YEAR) < calEnd.get(Calendar.YEAR) || calStart.get(Calendar.MONTH) < calEnd.get(Calendar.MONTH)) { // dateStr is like "yyyy-mm-dd" String dateStr = GeneralHelpers.getDateString(calStart); String month = dateStr.substring(0, 7); // set calTmp to the last day of the same month as calStart calTmp.setTimeInMillis(calStart.getTimeInMillis()); calTmp.set(Calendar.DAY_OF_MONTH, 1); calTmp.add(Calendar.MONTH, 1); calTmp.add(Calendar.DAY_OF_MONTH, -1); calTmp.set(Calendar.HOUR_OF_DAY, 23); calTmp.set(Calendar.MINUTE, 59); calTmp.set(Calendar.SECOND, 59); calTmp.set(Calendar.MILLISECOND, 999); long[] times = {calStart.getTimeInMillis(), calTmp.getTimeInMillis()}; ret.put(month, times); calStart.add(Calendar.MONTH, 1); if (firstMonth) { calStart.set(Calendar.DAY_OF_MONTH, 1); calStart.set(Calendar.HOUR_OF_DAY, 0); calStart.set(Calendar.MINUTE, 0); calStart.set(Calendar.SECOND, 0); calStart.set(Calendar.MILLISECOND, 0); firstMonth = false; } } String dateStr = GeneralHelpers.getDateString(calStart); String month = dateStr.substring(0, 7); long[] times = {calStart.getTimeInMillis(), calEnd.getTimeInMillis()}; ret.put(month, times); return ret; }
@Override protected void map(ImmutableBytesWritable rowKey, Result result, Context context) throws IOException, InterruptedException { long recordSize = GeneralHelpers.getHBaseResultKVSize(result); long colCount = result.size(); String rowKeyStr = null; if (rowKeyType == RowKeyType.STRING) { rowKeyStr = Bytes.toString(rowKey.get()); } else if (rowKeyType == RowKeyType.USER_ID) { rowKeyStr = TruthyHelpers.getUserIDStrFromBytes(rowKey.get()); } else { if (this.useBigInt) { rowKeyStr = TruthyHelpers.getTweetIDStrFromBigIntBytes(rowKey.get()); } else { rowKeyStr = TruthyHelpers.getTweetIDStrFromBytes(rowKey.get()); } } if (lastRowKeyStr == null) { // start counting for the first row key lastRowKeyStr = rowKeyStr; totalColCount = colCount; totalRecordSize = recordSize; } else { if (lastRowKeyStr.equals(rowKeyStr)) { // keep counting for the same row key totalColCount += colCount; totalRecordSize += recordSize; } else { // write information about last row key StringBuffer sb = new StringBuffer(); sb.append(totalColCount).append('\t'); sb.append(totalRecordSize); context.write(new Text(lastRowKeyStr), new Text(sb.toString())); // start counting for current row key lastRowKeyStr = rowKeyStr; totalColCount = colCount; totalRecordSize = recordSize; } } }