标签插入elasticsearch实现排重

patient-co/patient-co-figure-label/pom.xml

@@ -81,7 +81,7 @@
        <dependency>
            <groupId>com.yihu</groupId>
            <artifactId>common-data-es-starter</artifactId>
            <version>1.0.2</version>
            <version>1.0.3</version>
        </dependency>
        <dependency>
            <groupId>org.json</groupId>

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/convert/AgeConvert.java

@@ -5,6 +5,7 @@ import com.yihu.wlyy.figure.label.model.DataModel;
import com.yihu.wlyy.figure.label.model.SaveModel;
import com.yihu.wlyy.figure.label.util.ConstantUtil;
import com.yihu.wlyy.figure.label.util.IdCardUtil;
import com.yihu.wlyy.figure.label.util.MakeModelIDUtil;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@@ -32,6 +33,7 @@ public class AgeConvert implements Convert {
                    int age = IdCardUtil.getAgeForIdcard(model.getIdcard());
                    String labelCode = getAgeLabelCode(age);
                    SaveModel saveModel = new SaveModel();
                    saveModel.setModelId(MakeModelIDUtil.makeModelID(model,""));
                    saveModel.setLabelType(one.getParentCode());
                    saveModel.setLabelCode(labelCode);
                    saveModel.setLabeName(map.get(labelCode));

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/convert/BirthConvert.java

@@ -5,6 +5,7 @@ import com.yihu.wlyy.figure.label.model.DataModel;
import com.yihu.wlyy.figure.label.model.SaveModel;
import com.yihu.wlyy.figure.label.util.ConstantUtil;
import com.yihu.wlyy.figure.label.util.IdCardUtil;
import com.yihu.wlyy.figure.label.util.MakeModelIDUtil;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.springframework.util.CollectionUtils;
@@ -29,6 +30,7 @@ public class BirthConvert implements Convert {
                        e.printStackTrace();
                    }
                    SaveModel saveModel = new SaveModel();
                    saveModel.setModelId(MakeModelIDUtil.makeModelID(model,birth));
                    saveModel.setIdcard(model.getIdcard());
                    saveModel.setLabelCode(fLlabelDict.getLabelCode());
                    saveModel.setLabelType(fLlabelDict.getParentCode());

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/convert/ConvertHelper.java

@@ -6,6 +6,8 @@ import com.yihu.wlyy.figure.label.dao.FLLabelDictDao;
import com.yihu.wlyy.figure.label.model.DataModel;
import com.yihu.wlyy.figure.label.model.SaveModel;
import com.yihu.wlyy.figure.label.util.ConstantUtil;
import com.yihu.wlyy.figure.label.util.MD5Util;
import com.yihu.wlyy.figure.label.util.MakeModelIDUtil;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/convert/SexConvert.java

@@ -5,6 +5,7 @@ import com.yihu.wlyy.figure.label.model.DataModel;
import com.yihu.wlyy.figure.label.model.SaveModel;
import com.yihu.wlyy.figure.label.util.ConstantUtil;
import com.yihu.wlyy.figure.label.util.IdCardUtil;
import com.yihu.wlyy.figure.label.util.MakeModelIDUtil;
import org.apache.commons.lang3.time.DateFormatUtils;
import java.util.*;
@@ -24,6 +25,7 @@ public class SexConvert implements Convert{
                model -> {
                    String sex = IdCardUtil.getSexForIdcard(model.getIdcard());
                    SaveModel saveModel = new SaveModel();
                    saveModel.setModelId(MakeModelIDUtil.makeModelID(model,""));
                    saveModel.setLabelType(one.getParentCode());
                    saveModel.setLabelCode(sex);
                    saveModel.setLabeName(map.get(sex));

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/model/SaveModel.java

@@ -2,14 +2,21 @@ package com.yihu.wlyy.figure.label.model;
import io.searchbox.annotations.JestId;
import java.io.Serializable;
/**
 * Created by chenweida on 2018/3/7.
 */
public class SaveModel {
public class SaveModel implements Serializable{
    @JestId
    private String id;
    /**
     * 存入elasticsearch的id由 idcard，sex，parentCode，labelName，labelCode MD5加密而成。不用es自动生成的_id，实现数据插入排重
     */
    private String modelId;
    /**
     * 身份证
     */
@@ -36,8 +43,17 @@ public class SaveModel {
    /**
     * 创建时间
     */
    private String createTime;
    public String getModelId() {
        return modelId;
    }
    public void setModelId(String modelId) {
        this.modelId = modelId;
    }
    public String getIdcard() {
        return idcard;
    }

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/storage/Store2ES.java

@@ -1,5 +1,7 @@
package com.yihu.wlyy.figure.label.storage;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.yihu.base.es.config.ElastricSearchHelper;
import com.yihu.wlyy.figure.label.model.SaveModel;
import com.yihu.wlyy.figure.label.util.ConstantUtil;
@@ -18,18 +20,24 @@ public class Store2ES implements Storager {
    @Override
    public void save(List<SaveModel> modelList) {
        List<Object> list = new ArrayList<>();
        List<String> list = new ArrayList<>();
        int saveCount = 0;
        for(SaveModel saveModel:modelList){
            list.add(saveModel);
            ObjectMapper objectMapper = new ObjectMapper();
            try {
                String str =  objectMapper.writeValueAsString(saveModel);
                list.add(str);
            } catch (JsonProcessingException e) {
                e.printStackTrace();
            }
            //防止内存溢出，一次性存储10W条
            if(list.size() >= 100000){
                elastricSearchHelper.save(ConstantUtil.figure_label_es_index, ConstantUtil.figure_label_es_type, list);
                elastricSearchHelper.saveBulkWithCustomId(ConstantUtil.figure_label_es_index, ConstantUtil.figure_label_es_type, list,"modelId");
                list.clear();
                saveCount ++;
            }else if(modelList.size() - saveCount * 100000 == list.size()){
                //剩余的零头
                elastricSearchHelper.save(ConstantUtil.figure_label_es_index, ConstantUtil.figure_label_es_type, list);
                elastricSearchHelper.saveBulkWithCustomId(ConstantUtil.figure_label_es_index, ConstantUtil.figure_label_es_type, list,"modelId");
            }
        }
    }

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/util/MD5Util.java

@@ -0,0 +1,46 @@
package com.yihu.wlyy.figure.label.util;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
public class MD5Util {
    // 全局数组
    private final static String[] strDigits = { "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f" };
    public MD5Util() {
    }
    // 返回形式为数字跟字符串
    private static String byteToArrayString(byte bByte) {
        int iRet = bByte;
        // System.out.println("iRet="+iRet);
        if (iRet < 0) {
            iRet += 256;
        }
        int iD1 = iRet / 16;
        int iD2 = iRet % 16;
        return strDigits[iD1] + strDigits[iD2];
    }
    // 转换字节数组为16进制字串
    private static String byteToString(byte[] bByte) {
        StringBuffer sBuffer = new StringBuffer();
        for (int i = 0; i < bByte.length; i++) {
            sBuffer.append(byteToArrayString(bByte[i]));
        }
        return sBuffer.toString();
    }
    public static String GetMD5Code(String strObj) {
        String resultString = null;
        try {
            resultString = new String(strObj);
            MessageDigest md = MessageDigest.getInstance("MD5");
            // md.digest() 该函数返回值为存放哈希值结果的byte数组
            resultString = byteToString(md.digest(strObj.getBytes()));
        } catch (NoSuchAlgorithmException ex) {
            ex.printStackTrace();
        }
        return resultString;
    }
}

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/util/MakeModelIDUtil.java

@@ -0,0 +1,40 @@
package com.yihu.wlyy.figure.label.util;
import com.yihu.wlyy.figure.label.model.DataModel;
import org.springframework.util.StringUtils;
public class MakeModelIDUtil {
    /**
     * 生成存入es的id，确保数据不会重复
     * d由 idcard，sex，parentCode，labelName，labelCode MD5加密而成。不用es自动生成的_id，实现数据插入排重
     * @param dataModel
     * @param labelValue
     * @return
     */
    public static String makeModelID(DataModel dataModel,String labelValue) {
        StringBuilder idStr = new StringBuilder();
        idStr.append(dataModel.getIdcard()).append("-")
                .append(dataModel.getParentCode()).append("-")
                .append(dataModel.getLabelCode()).append("-")
                .append(dataModel.getLabelName());
       if(!StringUtils.isEmpty(labelValue)){
           idStr.append("-").append(labelValue);
       }
        return MD5Util.GetMD5Code(idStr.toString());
    }
    public static void main(String[] args) {
        DataModel dataModel = new DataModel();
        dataModel.setIdcard("350881199004241228");
        dataModel.setParentCode("2");
        dataModel.setLabelCode("1");
        dataModel.setLabelName("健康人群");
        String MD5ID = makeModelID(dataModel,"165cm");
//        String MD5ID = makeModelID(dataModel,"");
        System.out.println(MD5ID);
        System.out.println(MD5ID.length());
    }
}