多线程分页查询，类的属性名重命名

patient-co/patient-co-figure-label/pom.xml

@@ -83,6 +83,11 @@
            <artifactId>common-data-es-starter</artifactId>
            <version>1.1.0</version>
        </dependency>
        <dependency>
            <groupId>com.yihu</groupId>
            <artifactId>common-data-hbase-starter</artifactId>
            <version>1.0.1</version>
        </dependency>
        <dependency>
            <groupId>org.json</groupId>
            <artifactId>json</artifactId>

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/convert/AgeConvert.java

@@ -5,7 +5,7 @@ import com.yihu.wlyy.figure.label.model.DataModel;
import com.yihu.wlyy.figure.label.model.SaveModel;
import com.yihu.wlyy.figure.label.util.ConstantUtil;
import com.yihu.wlyy.figure.label.util.IdCardUtil;
import com.yihu.wlyy.figure.label.util.MakeModelIDUtil;
import com.yihu.wlyy.figure.label.util.MakeIDUtil;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@@ -34,7 +34,7 @@ public class AgeConvert implements Convert {
                    int age = IdCardUtil.getAgeForIdcard(model.getIdcard());
                    String labelCode = getAgeLabelCode(age);
                    SaveModel saveModel = new SaveModel();
                    saveModel.setId(MakeModelIDUtil.makeModelID(model,""));
                    saveModel.setId(MakeIDUtil.makeSaveModelID(model,""));
                    saveModel.setLabelType(one.getParentCode());
                    saveModel.setLabelCode(labelCode);
                    saveModel.setLabeName(map.get(labelCode));

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/convert/BirthConvert.java

@@ -5,7 +5,7 @@ import com.yihu.wlyy.figure.label.model.DataModel;
import com.yihu.wlyy.figure.label.model.SaveModel;
import com.yihu.wlyy.figure.label.util.ConstantUtil;
import com.yihu.wlyy.figure.label.util.IdCardUtil;
import com.yihu.wlyy.figure.label.util.MakeModelIDUtil;
import com.yihu.wlyy.figure.label.util.MakeIDUtil;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.springframework.util.CollectionUtils;
@@ -30,7 +30,7 @@ public class BirthConvert implements Convert {
                        e.printStackTrace();
                    }
                    SaveModel saveModel = new SaveModel();
                    saveModel.setId(MakeModelIDUtil.makeModelID(model,birth));
                    saveModel.setId(MakeIDUtil.makeSaveModelID(model,birth));
                    saveModel.setIdcard(model.getIdcard());
                    saveModel.setLabelCode(fLlabelDict.getLabelCode());
                    saveModel.setLabelType(fLlabelDict.getParentCode());

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/convert/ConvertHelper.java

@@ -6,8 +6,6 @@ import com.yihu.wlyy.figure.label.dao.FLLabelDictDao;
import com.yihu.wlyy.figure.label.model.DataModel;
import com.yihu.wlyy.figure.label.model.SaveModel;
import com.yihu.wlyy.figure.label.util.ConstantUtil;
import com.yihu.wlyy.figure.label.util.MD5Util;
import com.yihu.wlyy.figure.label.util.MakeModelIDUtil;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/convert/IsOnlineConvert.java

@@ -4,8 +4,7 @@ import com.yihu.figure_label.entity.FLlabelDict;
import com.yihu.wlyy.figure.label.model.DataModel;
import com.yihu.wlyy.figure.label.model.SaveModel;
import com.yihu.wlyy.figure.label.util.ConstantUtil;
import com.yihu.wlyy.figure.label.util.IdCardUtil;
import com.yihu.wlyy.figure.label.util.MakeModelIDUtil;
import com.yihu.wlyy.figure.label.util.MakeIDUtil;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@@ -35,7 +34,7 @@ public class IsOnlineConvert implements Convert {
        models.forEach(
                model -> {
                    SaveModel saveModel = new SaveModel();
                    saveModel.setModelId(MakeModelIDUtil.makeModelID(model,""));
                    saveModel.setId(MakeIDUtil.makeSaveModelID(model,""));
                    saveModel.setIdcard(model.getIdcard());
                    saveModel.setLabelType(one.getParentCode());
                    saveModel.setLabelCode(model.getLabelCode());

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/convert/SexConvert.java

@@ -5,7 +5,7 @@ import com.yihu.wlyy.figure.label.model.DataModel;
import com.yihu.wlyy.figure.label.model.SaveModel;
import com.yihu.wlyy.figure.label.util.ConstantUtil;
import com.yihu.wlyy.figure.label.util.IdCardUtil;
import com.yihu.wlyy.figure.label.util.MakeModelIDUtil;
import com.yihu.wlyy.figure.label.util.MakeIDUtil;
import org.apache.commons.lang3.time.DateFormatUtils;
import java.util.*;
@@ -25,7 +25,7 @@ public class SexConvert implements Convert{
                model -> {
                    String sex = IdCardUtil.getSexForIdcard(model.getIdcard());
                    SaveModel saveModel = new SaveModel();
                    saveModel.setId(MakeModelIDUtil.makeModelID(model,""));
                    saveModel.setId(MakeIDUtil.makeSaveModelID(model,""));
                    saveModel.setLabelType(one.getParentCode());
                    saveModel.setLabelCode(sex);
                    saveModel.setLabeName(map.get(sex));

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/convert/SignStateConvert.java

@@ -4,8 +4,7 @@ import com.yihu.figure_label.entity.FLlabelDict;
import com.yihu.wlyy.figure.label.model.DataModel;
import com.yihu.wlyy.figure.label.model.SaveModel;
import com.yihu.wlyy.figure.label.util.ConstantUtil;
import com.yihu.wlyy.figure.label.util.IdCardUtil;
import com.yihu.wlyy.figure.label.util.MakeModelIDUtil;
import com.yihu.wlyy.figure.label.util.MakeIDUtil;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.springframework.util.CollectionUtils;
@@ -33,7 +32,7 @@ public class SignStateConvert implements Convert{
                model -> {
                    SaveModel saveModel = new SaveModel();
                    saveModel.setIdcard(model.getIdcard());
                    saveModel.setModelId(MakeModelIDUtil.makeModelID(model,""));
                    saveModel.setId(MakeIDUtil.makeSaveModelID(model,""));
                    saveModel.setLabelCode(model.getLabelCode());
                    saveModel.setLabelType(one.getParentCode());
                    saveModel.setLabeName(map.get(model.getLabelCode()));

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/extract/MutilThreadMysqlExtracter.java

@@ -0,0 +1,39 @@
package com.yihu.wlyy.figure.label.extract;
import com.yihu.wlyy.figure.label.model.DataModel;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.BeanPropertyRowMapper;
import org.springframework.jdbc.core.JdbcTemplate;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.CountDownLatch;
/**
 * @Author lith 2018/03/23
 * 多线程分页查询mysql数据，因为mysql数据量比较大的时候，会比较慢
 */
public class MutilThreadMysqlExtracter implements Callable {
    private Logger logger = LoggerFactory.getLogger(MysqlExtracter.class);
    @Autowired
    private JdbcTemplate jdbcTemplate;
    private String sql;
    public MutilThreadMysqlExtracter(JdbcTemplate jdbcTemplate,String sql){
        this.jdbcTemplate = jdbcTemplate;
        this.sql = sql;
    }
    @Override
    public Object call() throws Exception {
        List<DataModel> datas = jdbcTemplate.query(sql,new BeanPropertyRowMapper(DataModel.class));
        logger.info("job get data counts:" + datas.size());
        return datas;
    }
}

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/extract/MysqlExtracter.java

@@ -42,24 +42,30 @@ public class MysqlExtracter implements Extracter {
        List<DataModel> datas = new ArrayList<>();
        int size = this.getCount(sql);
        int number = size / numPerPage + 1;
        CountDownLatch countDownLatch = new CountDownLatch(number);
        ExecutorService fixedThreadPool =  Executors.newFixedThreadPool(number);
        List<Callable<List<DataModel>>> threadList = new ArrayList<>();
        try {
            for (int i = 0; i < number; i++) {
                sql = sql + " limit " + i * numPerPage + "," + (i + 1) * numPerPage;
                MutilThreadMysqlExtracter mutilThreadMysqlExtracter = new MutilThreadMysqlExtracter(jdbcTemplate, sql);
                StringBuilder limitSql = new StringBuilder();
                limitSql.append(sql)
                        .append(" limit ")
                        .append((int)(i * numPerPage))
                        .append(",")
                        .append(numPerPage);
                MutilThreadMysqlExtracter mutilThreadMysqlExtracter = new MutilThreadMysqlExtracter(jdbcTemplate, limitSql.toString());
                threadList.add(mutilThreadMysqlExtracter);
            }
            long start = System.currentTimeMillis();
            logger.info("start time:" + String.valueOf(start));
            List<Future<List<DataModel>>> futureList = fixedThreadPool.invokeAll(threadList);
            countDownLatch.await();
            //取回线程执行的结果
            for (Future future : futureList) {
                datas.addAll((List<DataModel>) future.get());
            }
            logger.info("coast time:" + String.valueOf(System.currentTimeMillis() - start));
        } catch (Exception e) {
            logger.error("MutilThreadMysqlExtracter call failed！");
        }finally {
            countDownLatch.countDown();
            fixedThreadPool.shutdown();
        }
        return datas;

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/job/Mysql2ESJob.java

@@ -99,7 +99,8 @@ public class Mysql2ESJob implements Job {
        //1抽取数据
        String sql = this.flJobConfig.getSql();
        String sqlFiled = this.flJobConfig.getSqlField();
        this.finalSql = getJobConfigSql(sql,sqlFiled,sqlFiledCondition,sqlFiledValue);
//        this.finalSql = getJobConfigSql(sql,sqlFiled,sqlFiledCondition,sqlFiledValue);
        this.finalSql = getFinalSql(sql,sqlFiled,sqlFiledCondition,sqlFiledValue);
    }
    /**
@@ -178,7 +179,13 @@ public class Mysql2ESJob implements Job {
            result.append(" where ").append(sqlFiledString.toString());
        }
        /*if (sqlFiledValue.contains(",") && sql.contains("where")) {
        return result.toString();
    }
    public String getFinalSql(String sql, String sqlFiled, String sqlFiledCondition, String sqlFiledValue) {
        StringBuilder result = new StringBuilder();
        result.append(sql).append(" ");
        if (sqlFiledValue.contains(",") && sql.contains("where")) {
            String[] sqlFiledValues = sqlFiledValue.split(",");
            result.append("and ").append(sqlFiled).append(sqlFiledCondition).append("\'"+sqlFiledValues[1]+"\'").append(" and ").append(sqlFiled).append(sqlFiledCondition).append("\'"+sqlFiledValues[0]+"\'");
        }else if(sql.contains("where")){
@@ -189,9 +196,8 @@ public class Mysql2ESJob implements Job {
            result.replace(result.indexOf("$"),result.indexOf("$")+1,otherCondition.toString());
        }else {
            result.append("where ").append(sqlFiled).append(sqlFiledCondition).append("\'"+sqlFiledValue+"\'");
        }*/
        }
        return result.toString();
    }
}

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/service/JobService.java

@@ -103,9 +103,9 @@ public class JobService {
        params.put("jobConfig", flJobConfigVO.getId());
        //表里设置的增量查询类型和增量查询初始值，如果值没有配置，以当前时间为准
        if(!StringUtils.isEmpty(flJobConfigVO.getSqlFieldValue())){
            params.put(flJobConfigVO.getSqlField(),flJobConfigVO.getSqlFieldValue());
            params.put("sqlFiledValue",flJobConfigVO.getSqlFieldValue());
        }else{
            params.put(flJobConfigVO.getSqlField(), DateFormatUtils.format(new Date(),"yyyy-MM-dd HH:mm:ss"));
            params.put("sqlFiledValue", DateFormatUtils.format(new Date(),"yyyy-MM-dd HH:mm:ss"));
        }
        params.put("sqlFiledCondition", ">");
        if (!StringUtils.isEmpty(FLJobConfig.getJobClass())) {

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/storage/Store2ES.java

@@ -11,7 +11,9 @@ import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.List;
/**
 *@author lith on 2018.03.23
 */
@Component
public class Store2ES implements Storager {

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/storage/Store2Hbase.java

@@ -1,12 +1,46 @@
package com.yihu.wlyy.figure.label.storage;
import com.yihu.base.hbase.HBaseHelper;
import com.yihu.base.hbase.TableBundle;
import com.yihu.wlyy.figure.label.model.SaveModel;
import com.yihu.wlyy.figure.label.util.BeanUtil;
import com.yihu.wlyy.figure.label.util.ConstantUtil;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.util.CollectionUtils;
import java.lang.reflect.Field;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
 * lith 2018.03.23
 */
public class Store2Hbase implements Storager {
    private Logger logger = LoggerFactory.getLogger(Store2Hbase.class);
    @Autowired
    private HBaseHelper hBaseHelper;
    @Override
    public void save(List<SaveModel> modelList) {
        if(CollectionUtils.isEmpty(modelList)){
            return;
        }
        String[] fieldName = BeanUtil.getFiledName(modelList.get(0));
        TableBundle tableBundle = new TableBundle();
        for (SaveModel saveModel : modelList) {
            tableBundle.addFamily(saveModel.getId(), ConstantUtil.figure_label_familyA);
            tableBundle.addColumns(saveModel.getId(), ConstantUtil.figure_label_familyA,fieldName);
            Map<String,String> map = BeanUtil.beanToMap(saveModel);
            tableBundle.addValues(saveModel.getId(), ConstantUtil.figure_label_familyA,map);
        }
        hBaseHelper.save(ConstantUtil.figure_label_hbase_table,tableBundle);
        logger.info("save to hbase count:" + modelList.size());
    }
}
}

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/util/BeanUtil.java

@@ -0,0 +1,63 @@
package com.yihu.wlyy.figure.label.util;
import org.apache.commons.lang3.StringUtils;
import java.lang.reflect.Field;
import java.util.HashMap;
import java.util.Map;
/**
 * @project patient-co-figure-label
 * @author litaohong on 2018/3/27
 */
public class BeanUtil {
    /**
     * 获取类字段名
     * @param obj
     * @return
     */
    public static String[] getFiledName(Object obj) {
        try {
            Field[] fields = obj.getClass().getDeclaredFields();
            String[] fieldNames = new String[fields.length];
            for (int i = 0; i < fields.length; i++) {
                //id字段去掉
                if(StringUtils.equals("id",fields[i].getName())){
                    continue;
                }
                fieldNames[i] = fields[i].getName();
            }
            return fieldNames;
        } catch (SecurityException e) {
            e.printStackTrace();
        }
        return null;
    }
    /**
     * 将对象转换成map
     * @param obj
     * @return
     * @throws IllegalAccessException
     */
    public static Map<String, String> beanToMap(Object obj) {
        Map<String, String> map = new HashMap<>();
        try {
            Class<?> clazz = obj.getClass();
            for (Field field : clazz.getDeclaredFields()) {
                //id字段去掉
                if (StringUtils.equals("id", field.getName())) {
                    continue;
                }
                field.setAccessible(true);
                String fieldName = field.getName();
                Object value = field.get(obj);
                map.put(fieldName, value.toString());
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
        return map;
    }
}

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/util/ConstantUtil.java

@@ -4,4 +4,7 @@ public class ConstantUtil {
    public static final String figure_label_es_index = "figure_label_index";
    public static final String figure_label_es_type = "figure_label_type";
    public static final String date_format = "yyyy-MM-dd HH:mm:ss";
    public static final String figure_label_hbase_table = "figure_label";
    public static final String figure_label_familyA = "label_cloumn";
}

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/util/MakeModelIDUtil.java

@@ -3,7 +3,7 @@ package com.yihu.wlyy.figure.label.util;
import com.yihu.wlyy.figure.label.model.DataModel;
import org.springframework.util.StringUtils;
public class MakeModelIDUtil {
public class MakeIDUtil {
    /**
     * 生成存入es的id，确保数据不会重复
@@ -12,7 +12,7 @@ public class MakeModelIDUtil {
     * @param labelValue
     * @return
     */
    public static String makeModelID(DataModel dataModel,String labelValue) {
    public static String makeSaveModelID(DataModel dataModel, String labelValue) {
        StringBuilder idStr = new StringBuilder();
        idStr.append(dataModel.getIdcard()).append("-")
                .append(dataModel.getParentCode()).append("-")
@@ -32,8 +32,8 @@ public class MakeModelIDUtil {
        dataModel.setLabelCode("1");
        dataModel.setLabelName("健康人群");
        String MD5ID = makeModelID(dataModel,"165cm");
//        String MD5ID = makeModelID(dataModel,"");
        String MD5ID = makeSaveModelID(dataModel,"165cm");
//        String MD5ID = makeSaveModelID(dataModel,"");
        System.out.println(MD5ID);
        System.out.println(MD5ID.length());
    }