/**
* 删除昨天hbase异常的数据
*
* @param date
* @return
* @throws IOException
* @throws ParseException
*/
public boolean step2_dropHBaseData(String date) throws IOException, ParseException {
String tableName = "bi_core_city_orderinfo";
Date d = SDF_YYYY_MM_DD.parse(date);
String mdate = SDF_YYYYMMDD.format(d);
System.out.println("drop hbase table " + tableName + " date = " + date);
List<Result> rsList = dao.getRecordByRowKeyRegex(tableName, mdate);
for (Result result : rsList) {
String rowKeyString = new String(result.getRow());
System.out.println("delete - " + rowKeyString);
if (!debug) dao.deleteRecord(tableName, rowKeyString);
}
return true;
}
/**
* 插入昨天正常的数据到HBASE
*
* @return
* @throws Exception
*/
public boolean step3_doRecovery() throws Exception {
String sql =
"SELECT tmp.dateid_regn , "
+ " tmp.pro_amount,"
+ "FROM "
+ RECOVERY_TMP_TABLE
+ " tmp ";
Statement statement = null;
ResultSet rs = null;
try {
statement = con.createStatement();
System.out.println(sql);
rs = statement.executeQuery(sql);
while (rs.next()) {
double order_amt = Double.valueOf(rs.getString("prod_amount"));
String dateid_regn = rs.getString("dateid_regn");
try {
if (!debug)
dao.insertGosOrder(
"bi_core_city_orderinfo", dateid_regn, "cf", "prod_amount", order_amt + "");
System.out.println("bi_core_city_orderinfo = " + dateid_regn + ";order_amt=" + order_amt);
} catch (Exception e) {
System.out.println("write hbase error==" + e.getClass());
e.printStackTrace();
}
}
} catch (Exception e) {
e.printStackTrace();
System.out.println(e.getClass());
return false;
} finally {
}
return true;
}
@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
this._collector = collector;
try {
/** 初始化以下值 */
dao = new GosOrderDaoImpl();
curDate = Calendar.getInstance().getTime();
curTime = DateUtil.getFmtDate(curDate, DateUtil.YYYYMMDD_STR);
curHour = DateUtil.getCurHour() + 1;
rowkey_amount_last = "last_amount_" + curTime;
rowkey_amount_col_data = "col_data_" + curTime;
/** 读取Hbase获取 */
// String last_amount_value = dao.getValueByCol(Constant.TABLE_ORDER_INFO_RESULT,
// rowkey_amount_last,
// "order_amount");
// String last_amount_col_data_value =
// dao.getValue(Constant.TABLE_ORDER_INFO_RESULT,
// rowkey_amount_col_data);
// test -----------------------------------------------------
String last_amount_value =
dao.getValueByCol("bi_monitor_order_info_rslt", rowkey_amount_last, "order_amount");
String last_amount_col_data_value =
dao.getValue("bi_monitor_order_info_rslt", rowkey_amount_col_data);
// ----------------------------------------------------
if (!"".equals(last_amount_value)) {
orderAmountTotal = Double.parseDouble(last_amount_value);
}
if (!"".equals(last_amount_col_data_value)) {
sFinal = last_amount_col_data_value;
}
/** 订单数 */
rowkey_ordr_num_last = "last_ordr_num_" + curTime;
rowkey_ordr_num_col_data = "col_data_ordr_num_" + curTime;
/** 读取Hbase获取 */
String last_ordr_num_value =
dao.getValueByCol("bi_monitor_order_info_rslt", rowkey_ordr_num_last, "order_num");
String last_ordr_num_col_data_value =
dao.getValue("bi_monitor_order_info_rslt", rowkey_ordr_num_col_data);
if (!"".equals(last_ordr_num_value)) {
orderNumTotal = Long.parseLong(last_ordr_num_value);
}
if (!"".equals(last_ordr_num_col_data_value)) {
sFinalOrdrNum = last_ordr_num_col_data_value;
}
/**
* 这里补充:如果在sFinal中找不到当前小时的字符串,则进行appendNextHourStr的操作。由于重启过程中,跨小时了,这个时候curHour ==
* orderCreateHour,进行replaceNowHourStr操作,不会在当前小时的柱子上进行添加,所以需要补充一个小时的柱子
*/
if (sFinal.indexOf("[" + curHour + ",") == -1) {
sFinal = CommonUtil.appendNextHourStr(curHour, sFinal, "0.0");
sFinalOrdrNum = CommonUtil.appendNextHourStr(curHour, sFinalOrdrNum, "0");
}
logger.info("sFinal : " + sFinal);
logger.info("sFinalOrdrNum :" + sFinalOrdrNum);
/** 获取柱状图最后一个小时的数值用来作为初始值 */
orderAmountHourTotal =
Double.parseDouble(sFinal.substring(sFinal.lastIndexOf(",") + 1, sFinal.length() - 2));
orderNumHourTotal =
Long.parseLong(
sFinalOrdrNum.substring(
sFinalOrdrNum.lastIndexOf(",") + 1, sFinalOrdrNum.length() - 2));
logger.info("===curHour===" + curHour);
logger.info("amount init value is: " + orderAmountTotal);
logger.info("col data init value is: " + sFinal);
logger.info("ordr num init value is: " + orderNumTotal);
logger.info("col data ordr num init value is: " + sFinalOrdrNum);
dBConnection = new OracleConnection();
con = dBConnection.getConnection();
} catch (Exception e) {
e.printStackTrace();
}
}
@Override
public void execute(Tuple input) {
try {
// 当前坐标点信息
strLastXValue = CommonUtil.getCurFullXValueStr();
jumpMQOrderVo =
(JumpMQOrderVo) input.getValue(0); // 多线程下，input.getValue(0)的某个值不会被多个线程重复获取，即线程安全
Double orderAmount = 0.0;
// 获取订单创建时间
if (jumpMQOrderVo == null) {
logger.info("====空消息体===");
throw new Exception();
}
// 跨天判断
msgDate = jumpMQOrderVo.getMsgSendTime();
if (msgDate.getDate() != curDate.getDate()) {
if (msgDate.after(curDate)) { // 跨天
logger.info("===跨天处理===" + orderCreateDate);
curDate = msgDate;
/** 跨天处理 */
curTime = DateUtil.getFmtDate(msgDate, DateUtil.YYYYMMDD_STR);
curHour = msgDate.getHours() + 1;
/** 清空小时数据 */
orderAmountHourTotal = 0.0;
orderNumHourTotal = 0;
orderAmountTotal = 0.0;
rowkey_amount_last = "last_amount_" + curTime;
rowkey_amount_col_data = "col_data_" + curTime;
sFinal = "[]";
orderNumTotal = 0;
rowkey_ordr_num_last = "last_ordr_num_" + curTime;
rowkey_ordr_num_col_data = "col_data_ordr_num_" + curTime;
sFinalOrdrNum = "[]";
discountItemTotal.set(0);
discountPriceTotal.set(0);
dealHourFlg = 0;
} else if (msgDate.before(curDate)) { // 非当天的历史订单
return;
}
}
// 处理订单
orderAmount = processSoItem(input, jumpMQOrderVo);
Date orderDate = BusinessLogic.orderDate(jumpMQOrderVo);
if (orderDate == null) throw new IllegalArgumentException("Order Date is NULL");
orderCreateDate = DateUtil.getFmtDate(orderDate, DateUtil.YYYYMMDD_STR);
orderCreateHour = orderDate.getHours() + 1;
// 进行订单净额累加操作
logger.info("orderAmount:" + orderAmount);
orderAmountTotal += orderAmount;
logger.info("orderAmountTotal:" + orderAmountTotal);
orderNumTotal++;
// 订单创建小时数和当前系统小时数相同则进行小时订单净额累加操作,否则进行跨小时处理
if (curHour == orderCreateHour) {
// 如果sFinal为空,则初始化
if ("[]".equals(sFinal)) {
sFinal = CommonUtil.appendNextHourStr(curHour, sFinal, "0");
}
orderAmountHourTotal += orderAmount;
orderNumHourTotal++;
// 柱状图当前小时数据更新,之后需要重新设值
sFinal = CommonUtil.replaceNowHourStr(sFinal, orderAmountHourTotal.toString());
colData = sFinal;
if ("[]".equals(sFinalOrdrNum)) {
sFinalOrdrNum = CommonUtil.appendNextHourStr(curHour, sFinalOrdrNum, "0");
}
sFinalOrdrNum =
CommonUtil.replaceNowHourStr(sFinalOrdrNum, String.valueOf(orderNumHourTotal));
colDataOrdrNum = sFinalOrdrNum;
} else if (curHour < orderCreateHour) { // 需要强制入库,否则会丢失至多30秒的数据
logger.info("===跨小时处理===" + orderCreateHour);
/** 跨小时处理 */
curHour = orderCreateHour;
orderAmountHourTotal = 0.0;
// 柱状图拼接新的小时数据
sFinal = CommonUtil.appendNextHourStr(curHour, sFinal, orderAmountHourTotal.toString());
colData = sFinal;
orderNumHourTotal = 0;
sFinalOrdrNum =
CommonUtil.appendNextHourStr(curHour, sFinalOrdrNum, String.valueOf(orderNumHourTotal));
colDataOrdrNum = sFinalOrdrNum;
} else { // 当前时间大于订单创建时间,表示mq里面有之前阻塞的订单,需要特殊处理或忽略
logger.info("===处理阻塞订单===订单id:" + jumpMQOrderVo.baseInfo());
}
/** 写habse */
// 插入小时级最后累加值记录,30秒一次入库，如果期间有很多订单过来，30秒期间刚好碰到跨小时的情况的话，那部分数据就会丢失
synchronized (this) {
insertDB_last_point_endTime = System.currentTimeMillis();
// 5秒写入一次最后的点,前端5秒读取一次刷新最新的值insertDB_last_point_startTime
if (insertDB_last_point_endTime - insertDB_last_point_startTime >= Constant.SECOND_5) {
/** 订单净额 */
logger.info("orderAmountTotal : " + orderAmountTotal);
logger.info("orderNumHourTotal:" + orderNumHourTotal);
// 插入最后累加值记录,重启bolt的时候需要获取最后一次插入的金额,在此基础上再进行累加
dao.insertGosOrder(
Constant.TABLE_ORDER_INFO_RESULT,
rowkey_amount_last,
Constant.COMMON_FAMILY,
new String[] {"order_amount", "xValue", "xTitle"},
new String[] {orderAmountTotal + "", strLastXValue[1], strLastXValue[0]});
// 插入拼接的柱状图数据,单条记录,后端重启和前端展示获取历史都从这里获取
dao.insertGosOrder(
Constant.TABLE_ORDER_INFO_RESULT,
rowkey_amount_col_data,
Constant.COMMON_FAMILY,
"col_data",
colData);
/** 订单数 */
dao.insertGosOrder(
Constant.TABLE_ORDER_INFO_RESULT,
rowkey_ordr_num_last,
Constant.COMMON_FAMILY,
new String[] {"order_num", "xValue", "xTitle"},
new String[] {orderNumTotal + "", strLastXValue[1], strLastXValue[0]});
dao.insertGosOrder(
Constant.TABLE_ORDER_INFO_RESULT,
rowkey_ordr_num_col_data,
Constant.COMMON_FAMILY,
"col_data_ordr_num",
colDataOrdrNum);
insertDB_last_point_startTime = System.currentTimeMillis();
}
}
synchronized (this) {
insertDB_endTime = System.currentTimeMillis();
if (insertDB_endTime - insertDB_startTime >= Constant.SECOND_50) {
/** 线图数据处理 */
rowkey_amount = curTime + "_amount_" + strLastXValue[0].replace(".", "");
// 插入点的历史数据,线图中用来拼接历史数据线条
dao.insertGosOrder(
Constant.TABLE_ORDER_INFO_RESULT,
rowkey_amount,
Constant.COMMON_FAMILY,
new String[] {"order_amount", "xValue", "xTitle"},
new String[] {orderAmountTotal + "", strLastXValue[1], strLastXValue[0]});
rowkey_ordr_num = curTime + "_ordr_num_" + strLastXValue[0].replace(".", "");
dao.insertGosOrder(
Constant.TABLE_ORDER_INFO_RESULT,
rowkey_ordr_num,
Constant.COMMON_FAMILY,
new String[] {"order_num", "xValue", "xTitle"},
new String[] {orderNumTotal + "", strLastXValue[1], strLastXValue[0]});
logger.info("===50秒入库成功===");
insertDB_startTime = System.currentTimeMillis();
// 写Oracle ，推送数据到手机端报表
// insertOracle(con, orderAmountTotal, orderNumTotal);
}
}
} catch (Exception e) {
logger.info("ERROR. Caused by:", e);
}
}