es mapping

src/main/java/com/yihu/quota/controller/CubeController.java

@@ -0,0 +1,111 @@
package com.yihu.quota.controller;
import com.yihu.ehr.constants.ApiVersion;
import com.yihu.ehr.constants.ServiceApi;
import com.yihu.ehr.util.rest.Envelop;
import com.yihu.quota.model.cube.Cube;
import com.yihu.quota.model.cube.CubeMapping;
import com.yihu.quota.model.cube.CubeMemberMapping;
import com.yihu.quota.model.rest.FieldInfo;
import com.yihu.quota.service.cube.CubeMappingService;
import com.yihu.quota.service.cube.CubeMemberMappingService;
import com.yihu.quota.service.cube.CubeService;
import com.yihu.quota.util.ElasticSearchHandler;
import com.yihu.quota.vo.CubeMappingModel;
import com.yihu.quota.vo.CubeMemberMappingModel;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
import org.apache.commons.lang.StringUtils;
import org.elasticsearch.client.transport.TransportClient;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.util.ArrayList;
import java.util.List;
/**
 * Created by janseny on 2018/9/19.
 */
@RestController
@RequestMapping(ApiVersion.Version1_0)
@Api(description = "OLAP 多维数据集 控制入口")
public class CubeController  extends BaseController {
    @Autowired
    private CubeService cubeService;
    @Autowired
    private CubeMappingService cubeMappingService;
    @Autowired
    private CubeMemberMappingService cubeMemberMappingService;
    @ApiOperation(value = "创建多维数据集ES的mapping结构 ")
    @RequestMapping(value = "tj/getQuotaReport", method = RequestMethod.GET)
    public Envelop getQuotaReport(
            @ApiParam(name = "id", value = "多维数据集ID", required = true)
            @RequestParam(value = "id", required = true) int id
    ) {
        Envelop envelop = new Envelop();
        try {
            List<FieldInfo> fieldInfoList = new ArrayList<>();
            Cube cube = cubeService.findOne(id);
            String index = "";
            String type = "";
            if(cube.getIndexName() != null){
                index = cube.getIndexName();
            }
            if(cube.getIndexType() != null){
                type = cube.getIndexType();
            }
            // 数据集维度
            List<CubeMappingModel> cubeMappingModels = cubeMappingService.findCubeMappingModels(id);
            for(CubeMappingModel cubeMappingModel : cubeMappingModels){
                FieldInfo fieldInfo =  new FieldInfo();
                String fieldName = cubeMappingModel.getDimensionCode();
                fieldInfo.setField(fieldName);
                fieldInfo.setDataType(cubeMappingModel.getDataType());
                if(StringUtils.isNotEmpty(cubeMappingModel.getDict())){
                    //数据字典 扩展
                    List<FieldInfo> childDictFieldiList = new ArrayList<>();
                    FieldInfo childFieldCode =  new FieldInfo();
                    childFieldCode.setField(fieldName + "Name");
                    childFieldCode.setDataType("String");
                    FieldInfo childFieldName =  new FieldInfo();
                    childFieldName.setField(fieldName + "Code");
                    childFieldName.setDataType("String");
                    childDictFieldiList.add(childFieldCode);
                    childDictFieldiList.add(childFieldName);
                    fieldInfo.setFieldInfos(childDictFieldiList);
                }else{
                    // 子成员 扩展
                    List<CubeMemberMappingModel>  cubeMemberMappingModels = cubeMemberMappingService.findCubeMappingModels(cubeMappingModel.getId());
                    if(cubeMemberMappingModels != null){
                        List<FieldInfo> childFieldiList = new ArrayList<>();
                        for(CubeMemberMappingModel cubeMemberMappingModel : cubeMemberMappingModels){
                            FieldInfo childFieldInfo =  new FieldInfo();
                            childFieldInfo.setField(cubeMemberMappingModel.getDimensionCode());
                            childFieldInfo.setDataType(cubeMemberMappingModel.getDataType());
                            childFieldiList.add(childFieldInfo);
                        }
                        fieldInfo.setFieldInfos(childFieldiList);
                    }
                }
                fieldInfoList.add(fieldInfo);
            }
            ElasticSearchHandler esHandler = new ElasticSearchHandler();
            TransportClient client = esHandler.getTransportClient();
            esHandler.createIndexAndCreateMapping(index, type, fieldInfoList, client);
            envelop.setSuccessFlg(true);
            return envelop;
        } catch (Exception e) {
            invalidUserException(e, -1, "创建失败:" + e.getMessage());
        }
        envelop.setSuccessFlg(false);
        return envelop;
    }
}

src/main/java/com/yihu/quota/kafka/ConsumerListener.java

@@ -1,6 +1,6 @@
package com.yihu.quota.kafka;
import com.yihu.quota.service.cube.ElasticSearchMappingService;
import com.yihu.quota.service.cube.ElasticSearchDataProcessService;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.annotation.KafkaListener;
@@ -11,7 +11,7 @@ import org.springframework.kafka.annotation.KafkaListener;
 */
public class ConsumerListener {
    @Autowired
    private ElasticSearchMappingService elasticSearchMappingService;
    private ElasticSearchDataProcessService elasticSearchMappingService;
    @KafkaListener(topics = "sep-hbase-data")
    public void loadData(ConsumerRecord<?, ?> record) {

src/main/java/com/yihu/quota/model/cube/Cube.java

@@ -18,6 +18,7 @@ public class Cube {
    private String code;                  //多维数据集编码
    private Integer dataSourceType;     //数据源类型 1 hbase 2 mysql
    private String indexName;           //保存的索引名称
    private String indexType;           //保存的索引类型名称
    private String note;                //备注
    @Id
@@ -63,6 +64,14 @@ public class Cube {
        this.indexName = indexName;
    }
    public String getIndexType() {
        return indexType;
    }
    public void setIndexType(String indexType) {
        this.indexType = indexType;
    }
    public String getNote() {
        return note;
    }

src/main/java/com/yihu/quota/model/cube/CubeMemberMapping.java

@@ -14,7 +14,7 @@ import java.util.List;
@Table(name = "olap_cube_member_mapping")
public class CubeMemberMapping {
    private Integer id;
    private Integer cubeMappingId;    //多维数据集ID
    private Integer cubeMappingId;    //多维数据集维度ID
    private Integer dataFieldId;      //字段名
    private Integer dimensionMemberId;//维度成员ID
    private String dimensionCode;    //维度编码

src/main/java/com/yihu/quota/model/dimension/Dimension.java

@@ -33,6 +33,11 @@ public class Dimension {
     */
    private String algorithm;
    /**
     * 数据类型
     */
    private String dataType;
    /**
     * 临时字段,存放维度成员
     */
@@ -89,6 +94,14 @@ public class Dimension {
        this.algorithm = algorithm;
    }
    public String getDataType() {
        return dataType;
    }
    public void setDataType(String dataType) {
        this.dataType = dataType;
    }
    @Transient
    public List<DimensionMember> getDimensionMembers() {
        return dimensionMembers;

src/main/java/com/yihu/quota/model/dimension/DimensionMember.java

@@ -35,6 +35,11 @@ public class DimensionMember {
     */
    private String dimensionName;
    /**
     * 数据类型
     */
    private String dataType;
    @Id
    @GeneratedValue(strategy = GenerationType.IDENTITY)
    @Column(name = "id", unique = true, nullable = false)
@@ -79,6 +84,14 @@ public class DimensionMember {
        this.level = level;
    }
    public String getDataType() {
        return dataType;
    }
    public void setDataType(String dataType) {
        this.dataType = dataType;
    }
    @Formula("(SELECT odm.name FROM olap_dimension_member odm LEFT JOIN olap_dimension od ON od.id = odm.dimension_id WHERE odm.id  = id )")
    public String getDimensionName() {
        return dimensionName;

src/main/java/com/yihu/quota/model/rest/FieldInfo.java

@@ -9,7 +9,7 @@ import java.util.List;
public class FieldInfo {
    private String field;//名称
    private String dataType;//数据类型
    private int participle;//分词 1 string  not_analyzed 不分词  2 分词   其他 keyword  not_analyzed
    private int participle;//分词 1 string  not_analyzed 不分词  2 分词
    private List<FieldInfo> fieldInfos ;//子属性集合

src/main/java/com/yihu/quota/service/cube/CubeMappingService.java

@@ -5,9 +5,13 @@ import com.yihu.quota.dao.cube.CubeDao;
import com.yihu.quota.dao.cube.CubeMappingDao;
import com.yihu.quota.model.cube.Cube;
import com.yihu.quota.model.cube.CubeMapping;
import com.yihu.quota.vo.CubeMappingModel;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.BeanPropertyRowMapper;
import org.springframework.stereotype.Service;
import java.util.List;
/**
 * Created by janseny on 2018/9/10.
@@ -31,4 +35,18 @@ public class CubeMappingService extends BaseJpaService<CubeMapping, CubeMappingD
    public void deleteOlapcubeMapping(Integer id) {
        cubeMappingDao.delete(id);
    }
    public CubeMapping findOne(Integer id) {
       return cubeMappingDao.findOne(id);
    }
    public List<CubeMappingModel> findCubeMappingModels(int cubeId) {
        String sql = "SELECT ocm.*,od.data_type,od.dict from olap_cube_mapping ocm ,olap_dimension od " +
                " where ocm.dimension_id = od.id AND ocm.id = ? order by ocm.id asc";
        List<CubeMappingModel> cubeMappingModels = jdbcTemplate.query(sql, new BeanPropertyRowMapper(CubeMappingModel.class), cubeId);
        return cubeMappingModels;
    }
}

src/main/java/com/yihu/quota/service/cube/CubeMemberMappingService.java

@@ -6,9 +6,13 @@ import com.yihu.quota.dao.cube.CubeMemberMappingDao;
import com.yihu.quota.model.cube.Cube;
import com.yihu.quota.model.cube.CubeMapping;
import com.yihu.quota.model.cube.CubeMemberMapping;
import com.yihu.quota.vo.CubeMemberMappingModel;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.BeanPropertyRowMapper;
import org.springframework.stereotype.Service;
import java.util.List;
/**
 * Created by janseny on 2018/9/10.
@@ -32,4 +36,11 @@ public class CubeMemberMappingService extends BaseJpaService<CubeMemberMapping,
    public void deleteOlapcubeMemberMapping(Integer id) {
        cubeMemberMappingDao.delete(id);
    }
    public List<CubeMemberMappingModel> findCubeMappingModels(Integer id) {
        String sql = "SELECT ocmm.*,odm.data_type from olap_cube_member_mapping ocmm ,olap_dimension_member odm" +
                " where ocmm.dimension_member_id = odm.id order by ocmm.id asc";
        List<CubeMemberMappingModel> cubeMemberMappingModels = jdbcTemplate.query(sql, new BeanPropertyRowMapper(CubeMemberMappingModel.class), id);
        return cubeMemberMappingModels;
    }
}

src/main/java/com/yihu/quota/service/cube/CubeService.java

@@ -29,4 +29,8 @@ public class CubeService extends BaseJpaService<Cube, CubeDao> {
    public void deleteOlapCube(Integer id) {
        cubeDao.delete(id);
    }
    public Cube findOne(Integer id) {
        return cubeDao.findOne(id);
    }
}

src/main/java/com/yihu/quota/service/cube/ElasticSearchMappingService.java

@@ -5,13 +5,11 @@ import com.google.gson.Gson;
import com.yihu.ehr.elasticsearch.ElasticSearchPool;
import com.yihu.ehr.elasticsearch.ElasticSearchUtil;
import com.yihu.quota.util.ElasticSearchHandler;
import org.elasticsearch.client.transport.TransportClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.text.ParseException;
import java.util.HashMap;
import java.util.Map;
@@ -20,8 +18,8 @@ import java.util.Map;
 * Created by janseny on 2018/9/18.
 */
@Service
public class ElasticSearchMappingService {
    private static Logger logger = LoggerFactory.getLogger(ElasticSearchMappingService.class);
public class ElasticSearchDataProcessService {
    private static Logger logger = LoggerFactory.getLogger(ElasticSearchDataProcessService.class);
    private static String dataSource_hbase = "hbase";
    private static String dataSource_mysql = "mysql";
    private static String action_put = "Put";//添加和修改单个字段值
@@ -85,6 +83,9 @@ public class ElasticSearchMappingService {
        try {
            if(action.contains(action_put)){
                //保存数据
                //TODO 子属性数据 更新 ，需要将库中数据查询出来，然后合并在保存到库中
                //TODO 扩展的属性值 更新  如 字典 子成员  更加数据类型转换
                elasticSearchUtil.index(index, type, dataMap);
            }else if (action.contains(action_del)){
                for(String key : dataMap.keySet()){

src/main/java/com/yihu/quota/util/ElasticSearchHandler.java

@@ -165,13 +165,26 @@ public class ElasticSearchHandler {
                dataType = dataType.toLowerCase();
                Integer participle = info.getParticiple();
                if("string".equals(dataType)){
                    if(participle == 1) {
                        mapping.startObject(field).field("type", "string").field("index", "not_analyzed").endObject();
                    }else if(participle == 2) {
                        mapping.startObject(field).field("type", "string").endObject();
                    }
                    mapping.startObject(field).field("type", "string").field("index", "not_analyzed").endObject();
                }else if("date".equals(dataType)){
                    mapping.startObject(field).field("type", "date").field("format", "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis").endObject();
                }else if("object".equals(dataType)){//对象属性
                    mapping.startObject(field).startObject("properties");
                    //子属性
                    List<FieldInfo> childFieldList = info.getFieldInfos();
                    for(FieldInfo child : childFieldList){
                        String childField = child.getField();
                        String childDataType = child.getDataType();
                        int childParticiple = child.getParticiple();
                        if("string".equals(childDataType)){
                            mapping.startObject(childField).field("type", "string").field("index", "not_analyzed").endObject();
                        }else if("date".equals(dataType)){
                            mapping.startObject(childField).field("type", "date").field("format", "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis").endObject();
                        }else {
                            mapping.startObject(childField) .field("type", childDataType).endObject();
                        }
                    }
                    mapping.endObject().endObject();
                }else if("nested".equals(dataType)){//子集属性
                    mapping.startObject(field).field("type", "nested").startObject("properties");
                    //子属性
@@ -181,11 +194,7 @@ public class ElasticSearchHandler {
                        String childDataType = child.getDataType();
                        int childParticiple = child.getParticiple();
                        if("string".equals(childDataType)){
                            if(childParticiple == 1) {
                                mapping.startObject(childField).field("type", "string").field("index", "not_analyzed").endObject();
                            }else if(childParticiple == 2) {
                                mapping.startObject(childField).field("type", "string").endObject();
                            }
                            mapping.startObject(childField).field("type", "string").field("index", "not_analyzed").endObject();
                        }else if("date".equals(dataType)){
                            mapping.startObject(childField).field("type", "date").field("format", "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis").endObject();
                        }else {

src/main/java/com/yihu/quota/vo/CubeMappingModel.java

@@ -0,0 +1,98 @@
package com.yihu.quota.vo;
/**
 * Created by janseny on 2018/9/19.
 */
public class CubeMappingModel {
    private Integer id;
    private Integer cubeId;            //多维数据集ID
    private Integer dataFieldId;      //字段名
    private Integer dimensionId;      //维度ID
    private String dimensionCode;    //维度编码
    private String fieldName;        //字段名称
    private Integer saveType;        //成员保存时映射格式类型: 1 列 2 对象 3 子集
    private String dataType;        //数据类型
    private String dict;            //数据字典
    private String childPrimaryKay; //子集唯一字段
    public Integer getId() {
        return id;
    }
    public void setId(Integer id) {
        this.id = id;
    }
    public Integer getCubeId() {
        return cubeId;
    }
    public void setCubeId(Integer cubeId) {
        this.cubeId = cubeId;
    }
    public Integer getDataFieldId() {
        return dataFieldId;
    }
    public void setDataFieldId(Integer dataFieldId) {
        this.dataFieldId = dataFieldId;
    }
    public Integer getDimensionId() {
        return dimensionId;
    }
    public void setDimensionId(Integer dimensionId) {
        this.dimensionId = dimensionId;
    }
    public String getDimensionCode() {
        return dimensionCode;
    }
    public void setDimensionCode(String dimensionCode) {
        this.dimensionCode = dimensionCode;
    }
    public String getFieldName() {
        return fieldName;
    }
    public void setFieldName(String fieldName) {
        this.fieldName = fieldName;
    }
    public Integer getSaveType() {
        return saveType;
    }
    public void setSaveType(Integer saveType) {
        this.saveType = saveType;
    }
    public String getDataType() {
        return dataType;
    }
    public String getDict() {
        return dict;
    }
    public void setDict(String dict) {
        this.dict = dict;
    }
    public void setDataType(String dataType) {
        this.dataType = dataType;
    }
    public String getChildPrimaryKay() {
        return childPrimaryKay;
    }
    public void setChildPrimaryKay(String childPrimaryKay) {
        this.childPrimaryKay = childPrimaryKay;
    }
}

src/main/java/com/yihu/quota/vo/CubeMemberMappingModel.java

@@ -0,0 +1,71 @@
package com.yihu.quota.vo;
/**
 * Created by janseny on 2018/9/19.
 */
public class CubeMemberMappingModel {
    private Integer id;
    private Integer cubeMappingId;    //多维数据集维度ID
    private Integer dataFieldId;      //字段名
    private Integer dimensionMemberId;//维度成员ID
    private String dimensionCode;    //维度编码
    private String fieldName;        //字段名称
    private String dataType;        //数据类型
    public Integer getId() {
        return id;
    }
    public void setId(Integer id) {
        this.id = id;
    }
    public Integer getCubeMappingId() {
        return cubeMappingId;
    }
    public void setCubeMappingId(Integer cubeMappingId) {
        this.cubeMappingId = cubeMappingId;
    }
    public Integer getDataFieldId() {
        return dataFieldId;
    }
    public void setDataFieldId(Integer dataFieldId) {
        this.dataFieldId = dataFieldId;
    }
    public Integer getDimensionMemberId() {
        return dimensionMemberId;
    }
    public void setDimensionMemberId(Integer dimensionMemberId) {
        this.dimensionMemberId = dimensionMemberId;
    }
    public String getDimensionCode() {
        return dimensionCode;
    }
    public void setDimensionCode(String dimensionCode) {
        this.dimensionCode = dimensionCode;
    }
    public String getFieldName() {
        return fieldName;
    }
    public void setFieldName(String fieldName) {
        this.fieldName = fieldName;
    }
    public String getDataType() {
        return dataType;
    }
    public void setDataType(String dataType) {
        this.dataType = dataType;
    }
}