医疗云诊断疾病标签，健康问题标签

patient-co/patient-co-figure-label/pom.xml

@@ -88,6 +88,11 @@
            <artifactId>common-data-hbase-starter</artifactId>
            <version>1.0.1</version>
        </dependency>
        <dependency>
            <groupId>com.yihu</groupId>
            <artifactId>common-data-solr-starter</artifactId>
            <version>1.0.1</version>
        </dependency>
        <dependency>
            <groupId>org.json</groupId>
            <artifactId>json</artifactId>

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/controller/DataOperationController.java

@@ -38,19 +38,28 @@ public class DataOperationController extends BaseController{
    @Autowired
    ObjectMapper objectMapper;
    @ApiOperation(value = "根据dictCode删除es数据")
    @RequestMapping(value = "deteleByDictCode", method = RequestMethod.POST)
    public String deleteBydictCode( @ApiParam(name = "dictCode", value = "dictCode", required = true)@RequestParam(value = "dictCode", required = true) String dictCode) throws JobExecutionException {
       long count = deleteData(dictCode);
    @ApiOperation(value = "根据sql删除es数据")
    @RequestMapping(value = "deteleByQuery", method = RequestMethod.POST)
    public String deleteBySql( @ApiParam(name = "sql", value = "sql", required = true)@RequestParam(value = "sql", required = true) String sql) throws JobExecutionException {
       long count = deleteData(sql);
       if(count ==-1){
           return error(-1,"invalid sql");
       }
       return success("删除成功，共计"+ count +"条");
    }
    public long  deleteData(String dictCode){
    public long  deleteData(String sql){
        long start = System.currentTimeMillis();
        TimeUtil.start(logger,"start delete",start);
        String logTitle = "delete es data";
        TimeUtil.start(logger,logTitle,start);
        List<Map<String, Object>>  list = new ArrayList<>();
        String sql = "select * from " + ConstantUtil.figure_label_es_index + "/" +  ConstantUtil.figure_label_es_type + " where dictCode ='" + dictCode + "' limit 10000";
        if((!sql.contains("select") && !sql.contains("SELECT")) || (!sql.contains("from") && !sql.contains("FROM"))){
            return -1;
        }
        if(!sql.contains("limit")){
            sql = sql + " limit 10000";
        }
        list.addAll(elastricSearchHelper.excuceSQL(sql));
        long count = list.size();
        while (list.size() > 0){
@@ -58,7 +67,7 @@ public class DataOperationController extends BaseController{
            list = elastricSearchHelper.excuceSQL(sql);
            count+= list.size();
        }
        TimeUtil.finish(logger,"finish delete",start,System.currentTimeMillis());
        TimeUtil.finish(logger,logTitle,start,System.currentTimeMillis());
        return count;
    }

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/convert/AddressConvert.java

@@ -12,6 +12,7 @@ import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
/**
 * @author litaohong on 2018/4/12
@@ -24,12 +25,11 @@ public class AddressConvert implements Convert{
    private ConvertHelper convertHelper;
    @Override
    public List<SaveModel> convert(List<DataModel> modelList, List<FlLabelDict> flLabelDictList, String sourceType, String source) {
    public List<SaveModel> convert(List<DataModel> modelList, List<Map<String, Object>> hbaseDatas, List<FlLabelDict> flLabelDictList, String sourceType, String source) {
        List<SaveModel> saveModels = new ArrayList<>();
        FlLabelDict dict = flLabelDictList.get(0);
        for(DataModel dataModel : modelList){
            SaveModel saveModel = new SaveModel();
            saveModel.setId(MakeIDUtil.makeSaveModelID(dataModel,dataModel.getLabelValue()));
            saveModel.setIdcard(dataModel.getIdcard());
            saveModel.setLabelType(dict.getParentCode());
            saveModel.setLabelCode(dict.getLabelCode());
@@ -41,6 +41,8 @@ public class AddressConvert implements Convert{
                return new ArrayList<>();
            }
            saveModel.setCreateTime(DateFormatUtils.format(new Date(), ConstantUtil.date_format));
            saveModel.setId(MakeIDUtil.makeSaveModelID(saveModel));
            saveModels.add(saveModel);
        }
        return saveModels;

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/convert/AgeConvert.java

@@ -9,7 +9,6 @@ import com.yihu.wlyy.figure.label.util.MakeIDUtil;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;
import java.util.*;
@@ -25,7 +24,7 @@ public class AgeConvert implements Convert {
    private ConvertHelper convertHelper;
    @Override
    public List<SaveModel> convert(List<DataModel> models,List<FlLabelDict> flLabelDictList,String sourceType,String source) {
    public List<SaveModel> convert(List<DataModel> models, List<Map<String, Object>> hbaseDatas, List<FlLabelDict> flLabelDictList, String sourceType, String source) {
        List<SaveModel> saveModels = new ArrayList<>();
        FlLabelDict one = flLabelDictList.get(0);
        Map<String,String> map = new HashMap<>();
@@ -40,7 +39,8 @@ public class AgeConvert implements Convert {
                    int age = IdCardUtil.getAgeForIdcard(model.getIdcard());
                    String labelCode = getAgeLabelCode(age);
                    SaveModel saveModel = new SaveModel();
                    saveModel.setId(MakeIDUtil.makeSaveModelID(model,""));
                    saveModel.setDictCode(one.getDictCode());
                    saveModel.setIdcard(model.getIdcard());
                    saveModel.setLabelType(one.getParentCode());
                    saveModel.setLabelCode(labelCode);
                    saveModel.setLabelName(map.get(labelCode));
@@ -58,6 +58,7 @@ public class AgeConvert implements Convert {
                        return;
                    }
                    saveModel.setId(MakeIDUtil.makeSaveModelID(saveModel));
                    saveModels.add(saveModel);
                }
        );

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/convert/BirthConvert.java

@@ -9,12 +9,13 @@ import com.yihu.wlyy.figure.label.util.MakeIDUtil;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
/**
 * @author lith on 2018.03.14
 * 生日标签转换器
@@ -26,7 +27,7 @@ public class BirthConvert implements Convert {
    private ConvertHelper convertHelper;
    @Override
    public List<SaveModel> convert(List<DataModel> models,List<FlLabelDict> flLabelDictList,String sourceType,String source) {
    public List<SaveModel> convert(List<DataModel> models, List<Map<String, Object>> hbaseDatas, List<FlLabelDict> flLabelDictList, String sourceType, String source) {
        List<SaveModel> saveModels = new ArrayList<>();
        FlLabelDict flLabelDict = flLabelDictList.get(0);
        models.forEach(
@@ -38,8 +39,8 @@ public class BirthConvert implements Convert {
                        e.printStackTrace();
                    }
                    SaveModel saveModel = new SaveModel();
                    saveModel.setId(MakeIDUtil.makeSaveModelID(model,birth));
                    saveModel.setIdcard(model.getIdcard());
                    saveModel.setDictCode(flLabelDict.getDictCode());
                    saveModel.setLabelCode(flLabelDict.getLabelCode());
                    saveModel.setLabelType(flLabelDict.getParentCode());
                    saveModel.setLabelName(flLabelDict.getLabelName());
@@ -57,6 +58,7 @@ public class BirthConvert implements Convert {
                    if(StringUtils.isEmpty(saveModel.getSource())){
                        return;
                    }
                    saveModel.setId(MakeIDUtil.makeSaveModelID(saveModel));
                    saveModels.add(saveModel);
                }
        );

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/convert/ConsultConvert.java

@@ -8,9 +8,7 @@ import com.yihu.wlyy.figure.label.util.MakeIDUtil;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.*;
@@ -20,18 +18,18 @@ import java.util.*;
 * 咨询问题标签
 */
@Service
public class ConsultConvert implements Convert{
    @Autowired
    private ConvertHelper convertHelper;
    @Override
    public List<SaveModel> convert(List<DataModel> modelList, List<FlLabelDict> flLabelDictList, String sourceType, String source) {
    public List<SaveModel> convert(List<DataModel> modelList, List<Map<String, Object>> hbaseDatas, List<FlLabelDict> flLabelDictList, String sourceType, String source) {
        List<SaveModel> saveModels = new ArrayList<>();
        FlLabelDict dict = flLabelDictList.get(0);
        for(DataModel dataModel:modelList){
            SaveModel saveModel = new SaveModel();
            saveModel.setId(MakeIDUtil.makeSaveModelID(dataModel,dataModel.getLabelValue()));
            saveModel.setIdcard(dataModel.getIdcard());
            saveModel.setLabelType(dict.getParentCode());
            saveModel.setLabelCode(dict.getLabelCode());
@@ -43,6 +41,7 @@ public class ConsultConvert implements Convert{
            }
            saveModel.setSourceTime(dataModel.getSourceTime());
            saveModel.setCreateTime(DateFormatUtils.format(new Date(), ConstantUtil.date_format));
            saveModel.setId(MakeIDUtil.makeSaveModelID(saveModel));
            saveModels.add(saveModel);
        }
        return saveModels;

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/convert/Convert.java

@@ -5,6 +5,7 @@ import com.yihu.wlyy.figure.label.model.DataModel;
import com.yihu.wlyy.figure.label.model.SaveModel;
import java.util.List;
import java.util.Map;
/**
 * Created by chenweida on 2018/3/7.
@@ -12,5 +13,5 @@ import java.util.List;
 */
public interface Convert {
    List<SaveModel> convert(List<DataModel> modelList,List<FlLabelDict> flLabelDictList,String sourceType,String source);
    List<SaveModel> convert(List<DataModel> modelList, List<Map<String, Object>> hbaseDatas, List<FlLabelDict> flLabelDictList, String sourceType, String source);
}

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/convert/ConvertHelper.java

@@ -23,6 +23,7 @@ import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
/**
 * @author lith on 2018.03.14
@@ -40,13 +41,18 @@ public class ConvertHelper {
    /**
     * 进行具体标签生成转换
     * @param dataModels
     * @param hbaseDatas
     * @param flLabelDictJob
     * @param sourceType
     * @param source
     * @return
     * @throws Exception
     */
    public List<SaveModel> convert(List<DataModel> dataModels, FlLabelDictJob flLabelDictJob,String sourceType,String source) throws Exception {
    public List<SaveModel> convert(List<DataModel> dataModels, List<Map<String,Object>> hbaseDatas,FlLabelDictJob flLabelDictJob, String sourceType, String source) throws Exception {
        List<SaveModel> saveModels = new ArrayList<>();
        //如果源数据为空
        if(CollectionUtils.isEmpty(dataModels)){
        if(CollectionUtils.isEmpty(dataModels) && CollectionUtils.isEmpty(hbaseDatas)){
            return saveModels;
        }
        //标签job未配置
@@ -66,7 +72,7 @@ public class ConvertHelper {
            Class clazz = Class.forName(flLabelDictJob.getConvertClazz());
            Object obj = ApplicationContextHolderUtil.getContext().getBean(clazz);
            Method method = obj.getClass().getMethod("convert", List.class, List.class,String.class,String.class);
            saveModels = (List<SaveModel>) method.invoke(obj, dataModels, flLabelDictList,sourceType,source);
            saveModels = (List<SaveModel>) method.invoke(obj, dataModels, hbaseDatas,flLabelDictList,sourceType,source);
        } catch (Exception e) {
          logger.error("customized class or convert() method not found:" + flLabelDictJob.getConvertClazz());
            e.printStackTrace();
@@ -90,13 +96,13 @@ public class ConvertHelper {
        for(DataModel dataModel:dataModels){
            SaveModel saveModel = new SaveModel();
            saveModel.setIdcard(dataModel.getIdcard());
            saveModel.setId(MakeIDUtil.makeSaveModelID(dataModel,""));
            saveModel.setDictCode(dictCode);
            //只有一种类别
            if(null == basicDictItemHelper.labelDictMap.get(dictCode)){
                logger.error("label dictCode not exist :【" + dictCode+"】");
            //找不到该类别
            if (null == basicDictItemHelper.labelDictMap.get(dictCode)) {
                logger.error("label dictCode not exist :【" + dictCode + "】");
                return new ArrayList<>();
            }
            //只有一种类别
            if(basicDictItemHelper.labelDictMap.get(dictCode).size() == 1){
                List<FlLabelDict> flLabelDictList = basicDictItemHelper.labelDictMap.get(dictCode);
                saveModel.setLabelType(flLabelDictList.get(0).getParentCode());
@@ -127,6 +133,7 @@ public class ConvertHelper {
            if(StringUtils.isEmpty(dataModel.getSourceTime())){
                saveModel.setSourceTime(dataModel.getSourceTime());
            }
            saveModel.setId(MakeIDUtil.makeSaveModelID(saveModel));
            saveModels.add(saveModel);
        }
        return saveModels;

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/convert/EvaluateConvert.java

@@ -9,7 +9,6 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.*;
@@ -25,12 +24,11 @@ public class EvaluateConvert implements Convert{
    private ConvertHelper convertHelper;
    @Override
    public List<SaveModel> convert(List<DataModel> modelList, List<FlLabelDict> flLabelDictList, String sourceType, String source) {
    public List<SaveModel> convert(List<DataModel> modelList, List<Map<String, Object>> hbaseDatas, List<FlLabelDict> flLabelDictList, String sourceType, String source) {
        List<SaveModel> saveModels = new ArrayList<>();
        FlLabelDict dict = flLabelDictList.get(0);
        for(DataModel dataModel:modelList){
            SaveModel saveModel = new SaveModel();
            saveModel.setId(MakeIDUtil.makeSaveModelID(dataModel,dataModel.getLabelValue()));
            saveModel.setIdcard(dataModel.getIdcard());
            saveModel.setLabelType(dict.getParentCode());
            saveModel.setLabelCode(dict.getLabelCode());
@@ -43,6 +41,7 @@ public class EvaluateConvert implements Convert{
            }
            saveModel.setSourceTime(dataModel.getSourceTime());
            saveModel.setCreateTime(DateFormatUtils.format(new Date(), ConstantUtil.date_format));
            saveModel.setId(MakeIDUtil.makeSaveModelID(saveModel));
            saveModels.add(saveModel);
        }
        return saveModels;

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/convert/FeedbackConvert.java

@@ -31,14 +31,13 @@ public class FeedbackConvert implements Convert {
    private Map<String,String> distinctIdcardMap = new HashMap<>();
    @Override
    public List<SaveModel> convert(List<DataModel> modelList, List<FlLabelDict> flLabelDictList, String sourceType, String source) {
    public List<SaveModel> convert(List<DataModel> modelList, List<Map<String, Object>> hbaseDatas, List<FlLabelDict> flLabelDictList, String sourceType, String source) {
        List<SaveModel> saveModels = new ArrayList<>();
        FlLabelDict dict = flLabelDictList.get(0);
        Map<String,String> mapES = countESFeedbackByIdcard(modelList);
        Map<String,DataModel> modelMap = distinctIdcard(modelList);
        for(String idcard : modelMap.keySet()){
            SaveModel saveModel = new SaveModel();
            saveModel.setId(MakeIDUtil.makeSaveModelID(modelMap.get(idcard),""));
            int value = 0;
            saveModel.setIdcard(idcard);
            saveModel.setLabelType(dict.getParentCode());
@@ -57,6 +56,7 @@ public class FeedbackConvert implements Convert {
            }
            saveModel.setSourceTime(modelMap.get(idcard).getSourceTime());
            saveModel.setCreateTime(DateFormatUtils.format(new Date(), ConstantUtil.date_format));
            saveModel.setId(MakeIDUtil.makeSaveModelID(saveModel));
            saveModels.add(saveModel);
        }
        return saveModels;

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/convert/HealthConditionConvert.java

@@ -1,14 +0,0 @@
package com.yihu.wlyy.figure.label.convert;
import com.yihu.figure_label.entity.FlLabelDict;
import com.yihu.wlyy.figure.label.model.DataModel;
import com.yihu.wlyy.figure.label.model.SaveModel;
import java.util.List;
public class HealthConditionConvert implements Convert {
    @Override
    public List<SaveModel> convert(List<DataModel> models,List<FlLabelDict> flLabelDictList,String sourceType,String source) {
        return null;
    }
}

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/convert/HealthEduArticalConvert.java

@@ -9,7 +9,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.*;
/**
@@ -24,7 +24,7 @@ public class HealthEduArticalConvert implements Convert {
    private ConvertHelper convertHelper;
    @Override
    public List<SaveModel> convert(List<DataModel> modelList, List<FlLabelDict> flLabelDictList, String sourceType, String source) {
    public List<SaveModel> convert(List<DataModel> modelList, List<Map<String, Object>> hbaseDatas, List<FlLabelDict> flLabelDictList, String sourceType, String source) {
        List<SaveModel> saveModels = new ArrayList<>();
        FlLabelDict dict = flLabelDictList.get(0);
        Map<String,String> dictMap = new HashMap<>();
@@ -35,7 +35,6 @@ public class HealthEduArticalConvert implements Convert {
        );
        for(DataModel dataModel:modelList){
            SaveModel saveModel = new SaveModel();
            saveModel.setId(MakeIDUtil.makeSaveModelID(dataModel,dataModel.getLabelValue()));
            saveModel.setIdcard(dataModel.getIdcard());
            saveModel.setLabelType(dict.getParentCode());
            saveModel.setLabelCode(dataModel.getLabelCode());
@@ -48,6 +47,7 @@ public class HealthEduArticalConvert implements Convert {
            }
            saveModel.setSourceTime(dataModel.getSourceTime());
            saveModel.setCreateTime(DateFormatUtils.format(new Date(), ConstantUtil.date_format));
            saveModel.setId(MakeIDUtil.makeSaveModelID(saveModel));
            saveModels.add(saveModel);
        }
        return saveModels;

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/convert/IsConsultConvert.java

@@ -9,7 +9,6 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.*;
@@ -25,7 +24,7 @@ public class IsConsultConvert implements Convert {
    private ConvertHelper convertHelper;
    @Override
    public List<SaveModel> convert(List<DataModel> modelList, List<FlLabelDict> flLabelDictList, String sourceType, String source) {
    public List<SaveModel> convert(List<DataModel> modelList, List<Map<String, Object>> hbaseDatas, List<FlLabelDict> flLabelDictList, String sourceType, String source) {
        List<SaveModel> saveModels = new ArrayList<>();
        FlLabelDict dict = flLabelDictList.get(0);
        // 有咨询code为0，代表有咨询问题
@@ -38,7 +37,6 @@ public class IsConsultConvert implements Convert {
        );
        for(DataModel dataModel:modelList){
            SaveModel saveModel = new SaveModel();
            saveModel.setId(MakeIDUtil.makeSaveModelID(dataModel,""));
            saveModel.setIdcard(dataModel.getIdcard());
            saveModel.setLabelType(dict.getParentCode());
            saveModel.setLabelCode(hasConsult);
@@ -49,6 +47,7 @@ public class IsConsultConvert implements Convert {
            }
            saveModel.setSourceTime(dataModel.getSourceTime());
            saveModel.setCreateTime(DateFormatUtils.format(new Date(), ConstantUtil.date_format));
            saveModel.setId(MakeIDUtil.makeSaveModelID(saveModel));
            saveModels.add(saveModel);
        }
        return saveModels;

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/convert/IsDrinkingConvert.java

@@ -22,7 +22,7 @@ public class IsDrinkingConvert implements Convert{
    private ConvertHelper convertHelper;
    @Override
    public List<SaveModel> convert(List<DataModel> modelList, List<FlLabelDict> flLabelDictList, String sourceType, String source) {
    public List<SaveModel> convert(List<DataModel> modelList, List<Map<String, Object>> hbaseDatas, List<FlLabelDict> flLabelDictList, String sourceType, String source) {
        List<SaveModel> saveModels = new ArrayList<>();
        // 饮酒code为2，代表未知
        String hasDrinking = "2";

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/convert/IsOnlineConvert.java

@@ -24,7 +24,7 @@ public class IsOnlineConvert implements Convert {
    private ConvertHelper convertHelper;
    @Override
    public List<SaveModel> convert(List<DataModel> models,List<FlLabelDict> flLabelDictList,String sourceType,String source) {
    public List<SaveModel> convert(List<DataModel> models, List<Map<String, Object>> hbaseDatas, List<FlLabelDict> flLabelDictList, String sourceType, String source) {
        List<SaveModel> saveModels = new ArrayList<>();
        if(CollectionUtils.isEmpty(flLabelDictList)){
            return saveModels;
@@ -39,7 +39,6 @@ public class IsOnlineConvert implements Convert {
        models.forEach(
                model -> {
                    SaveModel saveModel = new SaveModel();
                    saveModel.setId(MakeIDUtil.makeSaveModelID(model,""));
                    saveModel.setIdcard(model.getIdcard());
                    saveModel.setLabelType(one.getParentCode());
                    saveModel.setLabelCode(model.getLabelCode());
@@ -51,6 +50,7 @@ public class IsOnlineConvert implements Convert {
                    if(StringUtils.isEmpty(saveModel.getSource())){
                        return;
                    }
                    saveModel.setId(MakeIDUtil.makeSaveModelID(saveModel));
                    saveModels.add(saveModel);
                }
        );

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/convert/IsSmokingConvert.java

@@ -22,7 +22,7 @@ public class IsSmokingConvert implements Convert{
    private ConvertHelper convertHelper;
    @Override
    public List<SaveModel> convert(List<DataModel> modelList, List<FlLabelDict> flLabelDictList, String sourceType, String source) {
    public List<SaveModel> convert(List<DataModel> modelList, List<Map<String, Object>> hbaseDatas, List<FlLabelDict> flLabelDictList, String sourceType, String source) {
        List<SaveModel> saveModels = new ArrayList<>();
        // 抽烟code为2，代表未知
        String hasSmoking = "2";

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/convert/OnLineConsultConvert.java

@@ -24,7 +24,7 @@ public class OnLineConsultConvert implements Convert {
    private ConvertHelper convertHelper;
    @Override
    public List<SaveModel> convert(List<DataModel> modelList, List<FlLabelDict> flLabelDictList, String sourceType, String source) {
    public List<SaveModel> convert(List<DataModel> modelList, List<Map<String, Object>> hbaseDatas, List<FlLabelDict> flLabelDictList, String sourceType, String source) {
        List<SaveModel> saveModels = new ArrayList<>();
        FlLabelDict dict = flLabelDictList.get(0);
        Map<String,String> dictMap = new HashMap<>();
@@ -35,7 +35,6 @@ public class OnLineConsultConvert implements Convert {
        );
        for(DataModel dataModel:modelList){
            SaveModel saveModel = new SaveModel();
            saveModel.setId(MakeIDUtil.makeSaveModelID(dataModel,""));
            saveModel.setIdcard(dataModel.getIdcard());
            saveModel.setLabelType(dict.getParentCode());
            saveModel.setLabelCode(dataModel.getLabelCode());
@@ -47,6 +46,7 @@ public class OnLineConsultConvert implements Convert {
            }
            saveModel.setSourceTime(dataModel.getSourceTime());
            saveModel.setCreateTime(DateFormatUtils.format(new Date(), ConstantUtil.date_format));
            saveModel.setId(MakeIDUtil.makeSaveModelID(saveModel));
            saveModels.add(saveModel);
        }
        return saveModels;

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/convert/OnlineAppointmentConvert.java

@@ -13,6 +13,7 @@ import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
/**
 * @author litaohong on 2018/4/12
@@ -26,12 +27,11 @@ public class OnlineAppointmentConvert implements Convert{
    private ConvertHelper convertHelper;
    @Override
    public List<SaveModel> convert(List<DataModel> modelList, List<FlLabelDict> flLabelDictList, String sourceType, String source) {
    public List<SaveModel> convert(List<DataModel> modelList, List<Map<String, Object>> hbaseDatas, List<FlLabelDict> flLabelDictList, String sourceType, String source) {
        List<SaveModel> saveModels = new ArrayList<>();
        FlLabelDict dict = flLabelDictList.get(0);
        for(DataModel dataModel : modelList){
            SaveModel saveModel = new SaveModel();
            saveModel.setId(MakeIDUtil.makeSaveModelID(dataModel,dataModel.getLabelValue()));
            saveModel.setIdcard(dataModel.getIdcard());
            saveModel.setLabelType(dict.getParentCode());
            saveModel.setLabelCode(dataModel.getParentCode() + "," + dataModel.getLabelCode());
@@ -44,6 +44,7 @@ public class OnlineAppointmentConvert implements Convert{
            }
            saveModel.setSourceTime(dataModel.getSourceTime());
            saveModel.setCreateTime(DateFormatUtils.format(new Date(), ConstantUtil.date_format));
            saveModel.setId(MakeIDUtil.makeSaveModelID(saveModel));
            saveModels.add(saveModel);
        }
        return saveModels;

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/convert/ReplyInTimeConvert.java

@@ -12,6 +12,7 @@ import org.springframework.util.StringUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
 * @author litaohong on 2018/4/8
@@ -28,12 +29,11 @@ public class ReplyInTimeConvert implements Convert{
    private ElastricSearchHelper elastricSearchHelper;
    @Override
    public List<SaveModel> convert(List<DataModel> modelList, List<FlLabelDict> flLabelDictList, String sourceType, String source) {
    public List<SaveModel> convert(List<DataModel> modelList, List<Map<String, Object>> hbaseDatas, List<FlLabelDict> flLabelDictList, String sourceType, String source) {
        List<SaveModel> saveModels = new ArrayList<>();
        FlLabelDict dict = flLabelDictList.get(0);
        for(DataModel model:modelList){
            SaveModel saveModel = new SaveModel();
            saveModel.setId(MakeIDUtil.makeSaveModelID(model,""));
            saveModel.setIdcard(model.getIdcard());
            saveModel.setLabelType(dict.getParentCode());
            saveModel.setLabelCode(dict.getLabelCode());
@@ -45,6 +45,7 @@ public class ReplyInTimeConvert implements Convert{
            if(StringUtils.isEmpty(saveModel.getSource())){
                return new ArrayList<>();
            }
            saveModel.setId(MakeIDUtil.makeSaveModelID(saveModel));
            saveModels.add(saveModel);
        }
        return saveModels;

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/convert/SexConvert.java

@@ -23,7 +23,7 @@ public class SexConvert implements Convert{
    private ConvertHelper convertHelper;
    @Override
    public List<SaveModel> convert(List<DataModel> models,List<FlLabelDict> flLabelDictList,String sourceType,String source) {
    public List<SaveModel> convert(List<DataModel> models, List<Map<String, Object>> hbaseDatas, List<FlLabelDict> flLabelDictList, String sourceType, String source) {
        List<SaveModel> saveModels = new ArrayList<>();
        FlLabelDict one = flLabelDictList.get(0);
        Map<String,String> map = new HashMap<>();
@@ -35,11 +35,11 @@ public class SexConvert implements Convert{
        models.forEach(
                model -> {
                    SaveModel saveModel = new SaveModel();
                    saveModel.setId(MakeIDUtil.makeSaveModelID(model,""));
                    saveModel.setIdcard(model.getIdcard());
                    saveModel.setLabelType(one.getParentCode());
                    String sex = IdCardUtil.getSexForIdcard(model.getIdcard());
                    saveModel.setDictCode(one.getDictCode());
                    saveModel.setLabelCode(sex);
                    saveModel.setLabelName(map.get(sex));
@@ -55,6 +55,7 @@ public class SexConvert implements Convert{
                    if(StringUtils.isEmpty(saveModel.getSource())){
                        return;
                    }
                    saveModel.setId(MakeIDUtil.makeSaveModelID(saveModel));
                    saveModels.add(saveModel);
                }
        );

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/convert/SignDoctorConvert.java

@@ -1,47 +0,0 @@
package com.yihu.wlyy.figure.label.convert;
import com.yihu.figure_label.entity.FlLabelDict;
import com.yihu.wlyy.figure.label.model.DataModel;
import com.yihu.wlyy.figure.label.model.SaveModel;
import com.yihu.wlyy.figure.label.util.ConstantUtil;
import com.yihu.wlyy.figure.label.util.MakeIDUtil;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.*;
/**
 * @author litaohong on 2018/4/9
 * @project patient-co-management
 * 签约医生
 */
@Component
public class SignDoctorConvert implements Convert {
    @Autowired
    private ConvertHelper convertHelper;
    @Override
    public List<SaveModel> convert(List<DataModel> modelList, List<FlLabelDict> flLabelDictList, String sourceType, String source) {
        List<SaveModel> saveModels = new ArrayList<>();
        FlLabelDict dict = flLabelDictList.get(0);
        for(DataModel dataModel:modelList){
            SaveModel saveModel = new SaveModel();
            saveModel.setId(MakeIDUtil.makeSaveModelID(dataModel,dataModel.getLabelValue()));
            saveModel.setIdcard(dataModel.getIdcard());
            saveModel.setLabelType(dict.getParentCode());
            saveModel.setLabelCode(dataModel.getLabelCode());
            saveModel.setLabelName(dict.getLabelName());
            saveModel.setLabelValue(dataModel.getLabelValue());
            convertHelper.generateDataSource(dataModel,saveModel,sourceType,source);
            if (StringUtils.isEmpty(saveModel.getSource())) {
                return new ArrayList<>();
            }
            saveModel.setSourceTime(dataModel.getSourceTime());
            saveModel.setCreateTime(DateFormatUtils.format(new Date(), ConstantUtil.date_format));
            saveModels.add(saveModel);
        }
        return saveModels;
    }
}

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/convert/SignStateConvert.java

@@ -1,58 +0,0 @@
package com.yihu.wlyy.figure.label.convert;
import com.yihu.figure_label.entity.FlLabelDict;
import com.yihu.wlyy.figure.label.model.DataModel;
import com.yihu.wlyy.figure.label.model.SaveModel;
import com.yihu.wlyy.figure.label.util.ConstantUtil;
import com.yihu.wlyy.figure.label.util.MakeIDUtil;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;
import java.util.*;
/**
 * zd
 * 2018-03-21
 */
@Component
public class SignStateConvert implements Convert{
    @Autowired
    private ConvertHelper convertHelper;
    @Override
    public List<SaveModel> convert(List<DataModel> models,List<FlLabelDict> flLabelDictList,String sourceType,String source) {
        List<SaveModel> saveModels = new ArrayList<>();
        if(CollectionUtils.isEmpty(flLabelDictList)){
            return saveModels;
        }
        FlLabelDict one = flLabelDictList.get(0);
        Map<String,String> map = new HashMap<>();
        flLabelDictList.forEach(
                fLlabelDict -> {
                    map.put(fLlabelDict.getLabelCode(),fLlabelDict.getLabelName());
                }
        );
        models.forEach(
                model -> {
                    SaveModel saveModel = new SaveModel();
                    saveModel.setIdcard(model.getIdcard());
                    saveModel.setId(MakeIDUtil.makeSaveModelID(model,""));
                    saveModel.setLabelCode(model.getLabelCode());
                    saveModel.setLabelType(one.getParentCode());
                    saveModel.setLabelName(map.get(model.getLabelCode()));
                    saveModel.setCreateTime(DateFormatUtils.format(new Date(), ConstantUtil.date_format));
                    convertHelper.generateDataSource(model,saveModel,sourceType,source);
                    //如果fl_job_config表配置的id和数据来源不一致，则不保存数据
                    if(StringUtils.isEmpty(saveModel.getSource())){
                        return;
                    }
                    saveModels.add(saveModel);
                }
        );
        return saveModels;
    }
}

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/convert/SignTeamConvert.java

@@ -1,49 +0,0 @@
package com.yihu.wlyy.figure.label.convert;
import com.yihu.figure_label.entity.FlLabelDict;
import com.yihu.wlyy.figure.label.model.DataModel;
import com.yihu.wlyy.figure.label.model.SaveModel;
import com.yihu.wlyy.figure.label.util.ConstantUtil;
import com.yihu.wlyy.figure.label.util.MakeIDUtil;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.*;
/**
 * @author litaohong on 2018/4/9
 * @project patient-co-management
 * 签约团队
 */
@Component
public class SignTeamConvert implements Convert {
    @Autowired
    private ConvertHelper convertHelper;
    @Override
    public List<SaveModel> convert(List<DataModel> modelList, List<FlLabelDict> flLabelDictList, String sourceType, String source) {
        List<SaveModel> saveModels = new ArrayList<>();
        FlLabelDict dict = flLabelDictList.get(0);
        Map<String,String> dictMap = new HashMap<>();
        for(DataModel dataModel:modelList){
            SaveModel saveModel = new SaveModel();
            saveModel.setId(MakeIDUtil.makeSaveModelID(dataModel,dataModel.getLabelValue()));
            saveModel.setIdcard(dataModel.getIdcard());
            saveModel.setLabelType(dict.getParentCode());
            saveModel.setLabelCode(dataModel.getLabelCode());
            saveModel.setLabelName(dict.getLabelName());
            saveModel.setLabelValue(dataModel.getLabelValue());
            convertHelper.generateDataSource(dataModel,saveModel,sourceType,source);
            if (StringUtils.isEmpty(saveModel.getSource())) {
                return new ArrayList<>();
            }
            saveModel.setSourceTime(dataModel.getSourceTime());
            saveModel.setCreateTime(DateFormatUtils.format(new Date(), ConstantUtil.date_format));
            saveModels.add(saveModel);
        }
        return saveModels;
    }
}

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/convert/WxSubscribeConvert.java

@@ -24,7 +24,7 @@ public class WxSubscribeConvert implements Convert {
    private ConvertHelper convertHelper;
    @Override
    public List<SaveModel> convert(List<DataModel> modelList, List<FlLabelDict> flLabelDictList, String sourceType, String source) {
    public List<SaveModel> convert(List<DataModel> modelList, List<Map<String, Object>> hbaseDatas, List<FlLabelDict> flLabelDictList, String sourceType, String source) {
        List<SaveModel> saveModels = new ArrayList<>();
        FlLabelDict dict = flLabelDictList.get(0);
        // 有咨询code为0，代表有咨询问题
@@ -37,7 +37,6 @@ public class WxSubscribeConvert implements Convert {
        );
        for(DataModel dataModel:modelList){
            SaveModel saveModel = new SaveModel();
            saveModel.setId(MakeIDUtil.makeSaveModelID(dataModel,""));
            saveModel.setIdcard(dataModel.getIdcard());
            saveModel.setLabelType(dict.getParentCode());
            saveModel.setLabelCode(yes);
@@ -49,6 +48,7 @@ public class WxSubscribeConvert implements Convert {
            }
            saveModel.setSourceTime(dataModel.getSourceTime());
            saveModel.setCreateTime(DateFormatUtils.format(new Date(), ConstantUtil.date_format));
            saveModel.setId(MakeIDUtil.makeSaveModelID(saveModel));
            saveModels.add(saveModel);
        }
        return saveModels;

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/extract/ESExtracter.java

@@ -25,8 +25,8 @@ public class ESExtracter implements Extracter{
    private ElasticsearchUtil elasticsearchUtil;
    @Override
    public List<DataModel> extractDataByJobConfigsql(String sql,String datasource){
        List<DataModel> saveModels = new ArrayList<>();
    public  List<Object> extractData(String sql, String datasource){
        List<Object> saveModels = new ArrayList<>();
        List<Map<String,Object>> list = elasticsearchUtil.excuteDataModel(sql);
        for(Map<String,Object> tempMap:list){
            try {

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/extract/Extracter.java

@@ -16,7 +16,7 @@ public interface Extracter {
     */
//    List<ExtractModel> extract();
    List<DataModel> extractDataByJobConfigsql(String sql,String datasource);
    List<DataModel> extractData(String sql,String datasource);
}

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/extract/HbaseExtracter.java

@@ -0,0 +1,90 @@
package com.yihu.wlyy.figure.label.extract;
import com.yihu.base.SolrHelper;
import com.yihu.base.hbase.HBaseHelper;
import com.yihu.wlyy.figure.label.model.DataModel;
import com.yihu.wlyy.figure.label.util.TimeUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
/**
 * @author litaohong on 2018/5/5
 * @project patient-co-management
 * hbase数据抽取器
 */
@Component
public class HbaseExtracter implements Extracter{
    private Logger logger = LoggerFactory.getLogger(HbaseExtracter.class);
    @Autowired
    private SolrHelper solrHelper;
    @Autowired
    private HBaseHelper hBaseHelper;
    private int numPerPage = 1000; //从solr中一次性取得1000条
    @Override
    public List<Object> extractData(String sql, String datasource) {
        return null;
    }
    public  List<Map<String,Object>>  extractData(String core, String q, String fq) {
        List<Map<String,Object>> result = new ArrayList<>();
        List<String> rowkeys = new ArrayList<>();
        rowkeys = getRowkeys(core,q,fq);
        for(String rowkey : rowkeys){
           Map<String,Object> map = hBaseHelper.getResultMap(rowkey,"HealthProfile");
            result.add(map);
        }
        return result;
    }
    public List<String> getRowkeys(String core, String q, String fq) {
        long count = 0;
        try {
            count = solrHelper.count(core, q);
            solrHelper.query(core, q, fq, null, 0, 0);
        } catch (Exception e) {
            e.printStackTrace();
            logger.error("get solr query error");
        }
        int number = (int) count / numPerPage + 1;
        List<String> rowkeys = new ArrayList<>();
        ExecutorService fixedThreadPool = Executors.newFixedThreadPool(number);
        List<Callable<List<String>>> threadList = new ArrayList<>();
        for (int i = 0; i < number; i++) {
            MutilThreadSearchSolrIndexExtracter searchSolrIndexExtracter = new MutilThreadSearchSolrIndexExtracter(solrHelper, core, q, fq, i * numPerPage, numPerPage);
            threadList.add(searchSolrIndexExtracter);
            long start = System.currentTimeMillis();
            String logTitle = "therad get data";
            TimeUtil.start(logger, logTitle, start);
            try {
                List<Future<List<String>>> futureList = fixedThreadPool.invokeAll(threadList);
                //取回线程执行的结果
                for (Future future : futureList) {
                    rowkeys.addAll((List<String>) future.get());
                }
            } catch (Exception e) {
                e.printStackTrace();
                logger.error("get futureList result error");
            }
            long finish = System.currentTimeMillis();
            TimeUtil.finish(logger, logTitle, start, finish);
        }
        return null;
    }
}

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/extract/MutilThreadSearchSolrIndexExtracter.java

@@ -0,0 +1,52 @@
package com.yihu.wlyy.figure.label.extract;
import com.yihu.base.SolrHelper;
import com.yihu.wlyy.figure.label.model.DataModel;
import org.apache.solr.common.SolrDocumentList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.BeanPropertyRowMapper;
import org.springframework.jdbc.core.JdbcTemplate;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
/**
 * @Author lith 2018/03/23
 * 多线程分页查询mysql数据，因为mysql数据量比较大的时候，会比较慢
 */
public class MutilThreadSearchSolrIndexExtracter implements Callable {
    private Logger logger = LoggerFactory.getLogger(MysqlExtracter.class);
    private SolrHelper solrHelper;
    private String core;
    private String q;
    private String fq;
    private long start;
    private long rows;
    public MutilThreadSearchSolrIndexExtracter(SolrHelper solrHelper,String core,String q,String fq,long start,long rows){
        this.solrHelper = solrHelper;
        this.core = core;
        this.q = q;
        this.start = start;
        this.rows = rows;
    }
    @Override
    public Object call() throws Exception {
        List<String> rowkeyList = new ArrayList<>();
        SolrDocumentList solrDocumentList = solrHelper.query(core,q,fq,null,start,rows);
        logger.info("job get data counts:" + solrDocumentList.size());
        solrDocumentList.forEach(
                document -> {
                    rowkeyList.add(document.get("rowkey").toString());
                }
        );
        return rowkeyList;
    }
}

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/job/Es2EsJob.java

@@ -132,7 +132,7 @@ public class Es2EsJob implements Job {
     * 提取数据，按数据库中配置的增量条件提取
     */
    public void extract(){
        this.dataModelList = esExtracter.extractDataByJobConfigsql(this.finalSql,"");
        this.dataModelList = esExtracter.extractData(this.finalSql,"");
    }
    /**

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/job/Hbase2ESJob.java

@@ -0,0 +1,168 @@
package com.yihu.wlyy.figure.label.job;
import com.yihu.figure_label.entity.FlJobConfig;
import com.yihu.figure_label.entity.FlLabelDictJob;
import com.yihu.wlyy.figure.label.controller.JobController;
import com.yihu.wlyy.figure.label.convert.ConvertHelper;
import com.yihu.wlyy.figure.label.dao.FlJobConfigDao;
import com.yihu.wlyy.figure.label.dao.FlLabelDictJobDao;
import com.yihu.wlyy.figure.label.enums.JobSqlFieldTypeEnum;
import com.yihu.wlyy.figure.label.extract.HbaseExtracter;
import com.yihu.wlyy.figure.label.extract.MysqlExtracter;
import com.yihu.wlyy.figure.label.model.DataModel;
import com.yihu.wlyy.figure.label.model.SaveModel;
import com.yihu.wlyy.figure.label.service.JobService;
import com.yihu.wlyy.figure.label.storage.Store2ES;
import org.quartz.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
 * Created by chenweida on 2018/3/7.
 */
//@Component
@Service
@Scope("prototype")
@DisallowConcurrentExecution//防止到了执行时间点前一任务还在执行中，但是这时有空闲的线程，那么马上又会执行，这样一来就会存在同一job被并行执行
public class Hbase2ESJob implements Job {
    private Logger logger = LoggerFactory.getLogger(JobController.class);
    @Autowired
    private FlJobConfigDao flJobConfigDao;
    @Autowired
    private FlLabelDictJobDao flLabelDictJobDao;
    @Autowired
    private ConvertHelper convertHelper;
    @Autowired HbaseExtracter hbaseExtracter;
    @Autowired
    Store2ES store2ES;
    @Autowired
    private JobService jobService;
    private FlJobConfig flJobConfig;
    private Long flJobConfigId;
    private String sqlFiledValue;
    private String sqlFiledCondition;
    private List<Map<String,Object>> datas;
    private String sourceType;
    private String source;
    private String datasource;
    private String q;
    private String fq;
    /**
     * 数据表的id，有些数据是按时间增量查询，有些数据是按表的主键id增量查询
     */
    private long lastDataId;
    @Override
    public void execute(JobExecutionContext context) throws JobExecutionException {
        JobDataMap paramsMap = context.getJobDetail().getJobDataMap();
        initParams(paramsMap);
        //根据obconfig里配置的sql提取数据
        extract();
        //数据转换
        List<SaveModel> list = convert();
        //转换后的结果
        if (!CollectionUtils.isEmpty(list)) {
            //数据保存
            boolean bool = save(list);
            //增量存储成功后，修改增量的czrq时间为当前时间或主键id为上次获取到的数据的最后的一条的id
            if(StringUtils.endsWithIgnoreCase(this.flJobConfig.getSqlFieldType().toString(), JobSqlFieldTypeEnum.TIME.toString())){
                jobService.updateFieldValuetoCurrentTimeOrId(this.flJobConfigId,null,bool);
            }else if(StringUtils.endsWithIgnoreCase(this.flJobConfig.getSqlFieldType().toString(), JobSqlFieldTypeEnum.NUM.toString())){
                int size = datas.size();
                DataModel dataModel = (DataModel)datas.get(size -1);
                lastDataId = dataModel.getId();
                jobService.updateFieldValuetoCurrentTimeOrId(this.flJobConfigId,this.lastDataId,bool);
            }
        }
    }
    public void initParams(JobDataMap paramsMap){
        this.flJobConfigId = (Long)paramsMap.get("jobConfig");
        this.sourceType = String.valueOf(paramsMap.get("sourceType"));
        this.source = String.valueOf(paramsMap.get("source"));
        this.flJobConfig = flJobConfigDao.findById(this.flJobConfigId);
        this.datasource = flJobConfig.getDatasource();
        this.sqlFiledCondition = paramsMap.getString("sqlFiledCondition");
        this.sqlFiledValue = flJobConfig.getSqlFieldValue();
        //没有传增量值，以数据库配置的默认值为查询条件
        if(StringUtils.isEmpty(this.sqlFiledValue)){
            this.sqlFiledValue = this.flJobConfig.getSqlFieldValue();
            this.sqlFiledCondition=">";
        }
        //1抽取数据
        this.q = this.flJobConfig.getSql();
        String sqlFiled = this.flJobConfig.getSqlField();
    }
    /**
     * 提取数据，按数据库中配置的增量条件提取
     */
    public void extract(){
        this.datas = hbaseExtracter.extractData(this.datasource,this.q,this.fq);
    }
    /**
     * 转换
     */
    public List<SaveModel> convert(){
        List<SaveModel> list = new ArrayList<>();
        FlLabelDictJob flLabelDictJob = flLabelDictJobDao.findByJobId(this.flJobConfigId);
        try {
            list = convertHelper.convert(null,this.datas, flLabelDictJob,this.sourceType,this.source);
        } catch (Exception e) {
            logger.error(e.getMessage());
        }
        return list;
    }
    /**
     * 保存
     * @param list
     */
    public boolean save(List<SaveModel> list){
        boolean bool = true;
        try {
            store2ES.save(list);
        }catch (Exception e){
            logger.error("save to elasticsearch failed，convet data count:" + list.size());
            bool = false;
        }
        return bool;
    }
}

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/job/Mysql2ESJob.java

@@ -12,21 +12,16 @@ import com.yihu.wlyy.figure.label.model.DataModel;
import com.yihu.wlyy.figure.label.model.SaveModel;
import com.yihu.wlyy.figure.label.service.JobService;
import com.yihu.wlyy.figure.label.storage.Store2ES;
import com.yihu.wlyy.figure.label.util.ConstantUtil;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.quartz.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
/**
@@ -134,7 +129,7 @@ public class Mysql2ESJob implements Job {
     */
    public void extract(){
        this.dataModelList = mysqlExtracter.extractDataByJobConfigsql(this.finalSql,this.datasource);
        this.dataModelList = mysqlExtracter.extractData(this.finalSql,this.datasource);
    }
    /**

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/util/MakeIDUtil.java

@@ -2,25 +2,27 @@ package com.yihu.wlyy.figure.label.util;
import com.yihu.figure_label.entity.FlLabelDict;
import com.yihu.wlyy.figure.label.model.DataModel;
import com.yihu.wlyy.figure.label.model.SaveModel;
import org.springframework.util.StringUtils;
public class MakeIDUtil {
    /**
     * 生成从mysql获取，后存入es的id，确保数据不会重复
     * id由 idcard，parentCode，labelName，labelCode MD5加密而成。
     * @param dataModel
     * @param labelValue
     * id由 idcard，dictCode,parentCode，labelName，labelCode MD5加密而成。
     * @param saveModel
     * @return
     */
    public static String makeSaveModelID(DataModel dataModel, String labelValue) {
    public static String makeSaveModelID(SaveModel saveModel) {
        StringBuilder idStr = new StringBuilder();
        idStr.append(dataModel.getIdcard()).append("-")
                .append(dataModel.getParentCode()).append("-")
                .append(dataModel.getLabelCode()).append("-")
                .append(dataModel.getLabelName());
       if(!StringUtils.isEmpty(labelValue)){
           idStr.append("-").append(labelValue);
        idStr.append(saveModel.getDictCode())
                .append(saveModel.getIdcard()).append("-")
                .append(saveModel.getLabelType()).append("-")
                .append(saveModel.getLabelCode()).append("-")
                .append(saveModel.getLabelName());
       if(!StringUtils.isEmpty(saveModel.getLabelValue())){
           idStr.append("-").append(saveModel.getLabelValue());
       }
        return MD5Util.GetMD5Code(idStr.toString());
    }
@@ -50,9 +52,6 @@ public class MakeIDUtil {
        dataModel.setLabelCode("1");
        dataModel.setLabelName("健康人群");
        String MD5ID = makeSaveModelID(dataModel,"165cm");
//        String MD5ID = makeSaveModelID(dataModel,"");
        System.out.println(MD5ID);
        System.out.println(MD5ID.length());
    }
}

patient-co/patient-co-figure-label/src/main/resources/application-dev.yml

@@ -30,6 +30,8 @@ spring:
        client:
          transport:
            sniff: false #开启嗅探集群  用nginx代理一层过后会出现ip解析失败问题
    solr:
      zk-host: node1.hde.h3c.com,node2.hde.h3c.com:2181/solr