|
@ -1,13 +1,29 @@
|
|
|
package com.yihu.quota.service.job;
|
|
|
|
|
|
import org.quartz.DisallowConcurrentExecution;
|
|
|
import org.quartz.Job;
|
|
|
import org.quartz.JobExecutionContext;
|
|
|
import org.quartz.JobExecutionException;
|
|
|
import com.google.gson.Gson;
|
|
|
import com.yihu.quota.contants.JobConstant;
|
|
|
import com.yihu.quota.kafka.Producer;
|
|
|
import org.quartz.*;
|
|
|
import org.slf4j.Logger;
|
|
|
import org.slf4j.LoggerFactory;
|
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
|
import org.springframework.context.annotation.Scope;
|
|
|
import org.springframework.jdbc.core.JdbcTemplate;
|
|
|
import org.springframework.stereotype.Component;
|
|
|
import org.springframework.web.context.support.SpringBeanAutowiringSupport;
|
|
|
|
|
|
import java.util.HashMap;
|
|
|
import java.util.List;
|
|
|
import java.util.Map;
|
|
|
|
|
|
/**
|
|
|
* 用于一个表就是一个多维数据集的情况,如组织机构表的数据采集
|
|
|
* 数据采集表要求,不符合要求的表需要先改造后进行数据采集:
|
|
|
* <p>
|
|
|
* 表必须是单字段唯一键(或主键),不支持复合唯一键(或主键)
|
|
|
* 过滤字段只支持单字段,不支持多字段过滤
|
|
|
* 过滤字段只支持时间和数字字段,不支持其他类型字段
|
|
|
*
|
|
|
* @author l4qiang
|
|
|
* @date 2018-09-18
|
|
|
*/
|
|
@ -15,8 +31,123 @@ import org.springframework.stereotype.Component;
|
|
|
@Scope("prototype")
|
|
|
@DisallowConcurrentExecution
|
|
|
public class SingleTableJob implements Job {
|
|
|
static private Logger logger = LoggerFactory.getLogger(SingleTableJob.class);
|
|
|
|
|
|
/**
|
|
|
* 数据来源表
|
|
|
*/
|
|
|
protected String table;
|
|
|
|
|
|
/**
|
|
|
* 表主键
|
|
|
*/
|
|
|
protected String primeKey;
|
|
|
|
|
|
/**
|
|
|
* 过滤字段
|
|
|
*/
|
|
|
protected String filterField;
|
|
|
/**
|
|
|
* 过滤字段类型
|
|
|
*/
|
|
|
protected String filterFieldType;
|
|
|
|
|
|
/**
|
|
|
* 过滤数据步长
|
|
|
*/
|
|
|
protected String size;
|
|
|
|
|
|
/**
|
|
|
* 开始时间
|
|
|
*/
|
|
|
protected String start;
|
|
|
|
|
|
/**
|
|
|
* 结束时间
|
|
|
*/
|
|
|
protected String end;
|
|
|
|
|
|
/**
|
|
|
* 执行动作 1 手动执行 2 周期执行
|
|
|
*/
|
|
|
protected JobConstant.ExecType execType;
|
|
|
|
|
|
@Autowired
|
|
|
private Producer producer;
|
|
|
@Autowired
|
|
|
private JdbcTemplate jdbcTemplate;
|
|
|
|
|
|
@Override
|
|
|
public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
|
|
|
prepare(jobExecutionContext);
|
|
|
|
|
|
cleanData();
|
|
|
|
|
|
List<Map<String, Object>> list;
|
|
|
do {
|
|
|
list = fetch();
|
|
|
saveData(list);
|
|
|
} while (list != null && list.size() != 0);
|
|
|
}
|
|
|
|
|
|
private void prepare(JobExecutionContext jobExecutionContext) {
|
|
|
//spring注入
|
|
|
SpringBeanAutowiringSupport.processInjectionBasedOnCurrentContext(this);
|
|
|
JobDataMap jobDataMap = jobExecutionContext.getJobDetail().getJobDataMap();
|
|
|
|
|
|
table = jobDataMap.getString("table");
|
|
|
primeKey = jobDataMap.getString("primeKey");
|
|
|
filterField = jobDataMap.getString("filterField");
|
|
|
filterFieldType = jobDataMap.getString("filterFieldType");
|
|
|
size = jobDataMap.getString("size");
|
|
|
start = jobDataMap.getString("start");
|
|
|
end = jobDataMap.getString("end");
|
|
|
}
|
|
|
|
|
|
private void cleanData() {
|
|
|
if (JobConstant.ExecType.Full.equals(execType)) {
|
|
|
Map<String, Object> dataMap = new HashMap<>(2);
|
|
|
dataMap.put("table", table);
|
|
|
dataMap.put("delAll", true);
|
|
|
|
|
|
Gson gson = new Gson();
|
|
|
String jsonData = gson.toJson(dataMap);
|
|
|
producer.sendMessage(jsonData);
|
|
|
}
|
|
|
}
|
|
|
|
|
|
private void saveData(List<Map<String, Object>> list) {
|
|
|
if (list == null) {
|
|
|
logger.warn("未获取到数据");
|
|
|
return;
|
|
|
}
|
|
|
|
|
|
list.forEach(item -> {
|
|
|
Map<String, Object> dataMap = new HashMap<>(item.size());
|
|
|
dataMap.put("table", table);
|
|
|
item.forEach((key, value) -> {
|
|
|
if (key.equals(primeKey)) {
|
|
|
dataMap.put("rowKey", value);
|
|
|
}
|
|
|
|
|
|
dataMap.put(key, value);
|
|
|
});
|
|
|
|
|
|
|
|
|
Gson gson = new Gson();
|
|
|
String jsonData = gson.toJson(dataMap);
|
|
|
producer.sendMessage(jsonData);
|
|
|
});
|
|
|
|
|
|
}
|
|
|
|
|
|
private List<Map<String, Object>> fetch() {
|
|
|
//TODO:filterFieldType过滤
|
|
|
|
|
|
String sql = "select * from " + table +
|
|
|
" where " + filterField + ">=" + start + " and " + filterField + "<" + (start + size) + " and " +
|
|
|
filterField + "<=" + end;
|
|
|
|
|
|
return jdbcTemplate.queryForList(sql);
|
|
|
}
|
|
|
}
|