多线程分页查询

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/convert/AgeConvert.java

@@ -23,6 +23,7 @@ public class AgeConvert implements Convert {
        }
        FLlabelDict one = flLabelDictList.get(0);
        Map<String,String> map = new HashMap<>();
        flLabelDictList.forEach(
                fLlabelDict -> {
                    map.put(fLlabelDict.getLabelCode(),fLlabelDict.getLabelName());
@@ -33,7 +34,7 @@ public class AgeConvert implements Convert {
                    int age = IdCardUtil.getAgeForIdcard(model.getIdcard());
                    String labelCode = getAgeLabelCode(age);
                    SaveModel saveModel = new SaveModel();
                    saveModel.setModelId(MakeModelIDUtil.makeModelID(model,""));
                    saveModel.setId(MakeModelIDUtil.makeModelID(model,""));
                    saveModel.setLabelType(one.getParentCode());
                    saveModel.setLabelCode(labelCode);
                    saveModel.setLabeName(map.get(labelCode));

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/convert/BirthConvert.java

@@ -30,7 +30,7 @@ public class BirthConvert implements Convert {
                        e.printStackTrace();
                    }
                    SaveModel saveModel = new SaveModel();
                    saveModel.setModelId(MakeModelIDUtil.makeModelID(model,birth));
                    saveModel.setId(MakeModelIDUtil.makeModelID(model,birth));
                    saveModel.setIdcard(model.getIdcard());
                    saveModel.setLabelCode(fLlabelDict.getLabelCode());
                    saveModel.setLabelType(fLlabelDict.getParentCode());

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/convert/SexConvert.java

@@ -25,7 +25,7 @@ public class SexConvert implements Convert{
                model -> {
                    String sex = IdCardUtil.getSexForIdcard(model.getIdcard());
                    SaveModel saveModel = new SaveModel();
                    saveModel.setModelId(MakeModelIDUtil.makeModelID(model,""));
                    saveModel.setId(MakeModelIDUtil.makeModelID(model,""));
                    saveModel.setLabelType(one.getParentCode());
                    saveModel.setLabelCode(sex);
                    saveModel.setLabeName(map.get(sex));

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/extract/MysqlExtracter.java

@@ -9,7 +9,9 @@ import org.springframework.jdbc.core.BeanPropertyRowMapper;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.*;
/**
 * Created by chenweida on 2018/3/7.
@@ -19,19 +21,58 @@ public class MysqlExtracter implements Extracter {
    private Logger logger = LoggerFactory.getLogger(MysqlExtracter.class);
    private int numPerPage = 100000; //一次性最多查询10万条
    @Autowired
    private JdbcTemplate jdbcTemplate;
   /* @Override
    public List<ExtractModel> extract() {
        return null;
    }*/
    @Override
    public List<DataModel> extractDataByJobConfigsql(String sql) {
        List<DataModel> datas = jdbcTemplate.query(sql,new BeanPropertyRowMapper(DataModel.class));
        logger.info("job get data counts:" + datas.size());
        List<DataModel> datas = new ArrayList<>();
        getDataByThread(sql);
        return datas;
    }
    /**
     * 根据数据量采用多线程分页查询数据
     * @param sql
     * @return
     */
    public List<DataModel> getDataByThread(String sql){
        List<DataModel> datas = new ArrayList<>();
        int size = this.getCount(sql);
        int number = size / numPerPage + 1;
        CountDownLatch countDownLatch = new CountDownLatch(number);
        ExecutorService fixedThreadPool =  Executors.newFixedThreadPool(number);
        List<Callable<List<DataModel>>> threadList = new ArrayList<>();
        try {
            for (int i = 0; i < number; i++) {
                sql = sql + " limit " + i * numPerPage + "," + (i + 1) * numPerPage;
                MutilThreadMysqlExtracter mutilThreadMysqlExtracter = new MutilThreadMysqlExtracter(jdbcTemplate, sql);
                threadList.add(mutilThreadMysqlExtracter);
            }
            List<Future<List<DataModel>>> futureList = fixedThreadPool.invokeAll(threadList);
            countDownLatch.await();
            for (Future future : futureList) {
                datas.addAll((List<DataModel>) future.get());
            }
        } catch (Exception e) {
            logger.error("MutilThreadMysqlExtracter call failed！");
        }finally {
            countDownLatch.countDown();
            fixedThreadPool.shutdown();
        }
        return datas;
    }
    /**
     * 统计有多少条数据
     * @param sql
     * @return
     */
    public Integer getCount(String sql){
        String countSql = "select count(*) from " + sql.split("from")[1];
        Integer count = jdbcTemplate.queryForObject(countSql,Integer.class);
        return count;
    }
}

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/model/SaveModel.java

@@ -10,12 +10,10 @@ import java.io.Serializable;
public class SaveModel implements Serializable{
    @JestId
    private String id;
    /**
     * 存入elasticsearch的id由 idcard，sex，parentCode，labelName，labelCode MD5加密而成。不用es自动生成的_id，实现数据插入排重
     * 存入elasticsearch的id由 idcard，parentCode，labelName，labelCode MD5加密而成。不用es自动生成的_id，实现数据插入排重
     */
    private String modelId;
    private String id;
    /**
     * 身份证
@@ -43,17 +41,8 @@ public class SaveModel implements Serializable{
    /**
     * 创建时间
     */
    private String createTime;
    public String getModelId() {
        return modelId;
    }
    public void setModelId(String modelId) {
        this.modelId = modelId;
    }
    public String getIdcard() {
        return idcard;
    }