数据解析 增加 数据关联查询

src/main/java/com/yihu/quota/dao/cube/JdbcBasicDao.java

@@ -21,7 +21,7 @@ public class JdbcBasicDao {
    @Value("${spring.datasource.password}")
    private String datasourcePassword;
    public String getConnection(String sql ,String database,String cloumnCode){
    public String getConnection(String sql ,String database,String cloumnCode) throws Exception{
        try {
            String URL = datasourceUrl;
            String oldDatabase = datasourceUrl.substring(datasourceUrl.lastIndexOf("/")+1,datasourceUrl.indexOf("?"));

src/main/java/com/yihu/quota/service/cube/CubeMappingService.java

@@ -66,7 +66,7 @@ public class CubeMappingService extends BaseJpaService<CubeMapping, CubeMappingD
                " LEFT JOIN olap_data_sources_table_field odstf ON ocm.data_field_id = odstf.id "+
                " LEFT JOIN olap_data_sources_table odst ON odst.id = odstf.table_id " +
                " LEFT JOIN olap_cube oc ON oc.id = ocm.cube_id " +
                " LEFT JOIN olap_dimension od ON ocm.dimension_id = od.id where odstf.field_code = ? and odst.table_code = ? ";
                " LEFT JOIN olap_dimension od ON ocm.dimension_id = od.id where odstf.field_code = ? and odst.table_code = ? and ocm.child_save_type is null  ";
        String param[] = {fieldCode,tableCode};
        List<CubeMappingModel> cubeMappingModels = jdbcTemplate.query(sql, new BeanPropertyRowMapper(CubeMappingModel.class), param);
        return cubeMappingModels;
@@ -91,21 +91,21 @@ public class CubeMappingService extends BaseJpaService<CubeMapping, CubeMappingD
        return null;
    }
    /**
     * 根据表编码和关联字段编码 查询对应的维度信息
     * @param RelationFieldCode 关联字段
     * @param tableCode
     * @return
     */
    public List<CubeMappingModel> findRelationMappingModelsByFieldCode(String tableCode,String RelationFieldCode) {
        String sql = "SELECT ocm.*,oc.index_name,oc.index_type,odstf.field_code,od.dict,od.data_type,od.`algorithm`,od.parm from olap_cube_mapping ocm  " +
                " LEFT JOIN olap_data_sources_table_field odstf ON ocm.relation_field_id = odstf.id "+
                " LEFT JOIN olap_data_sources_table odst ON odst.id = odstf.table_id " +
                " LEFT JOIN olap_cube oc ON oc.id = ocm.cube_id " +
                " LEFT JOIN olap_dimension od ON ocm.dimension_id = od.id where ocm.child_save_type is null and  odstf.field_code = ? and odst.table_code = ? ";
        String param[] = {RelationFieldCode,tableCode};
        List<CubeMappingModel> cubeMappingModels = jdbcTemplate.query(sql, new BeanPropertyRowMapper(CubeMappingModel.class), param);
        return cubeMappingModels;
    }
//    /**
//     * 根据表编码和关联字段编码 查询对应的维度信息
//     * @param RelationFieldCode 关联字段
//     * @param tableCode
//     * @return
//     */
//    public List<CubeMappingModel> findRelationMappingModelsByFieldCode(String tableCode,String RelationFieldCode) {
//        String sql = "SELECT ocm.*,oc.index_name,oc.index_type,odstf.field_code,od.dict,od.data_type,od.`algorithm`,od.parm from olap_cube_mapping ocm  " +
//                " LEFT JOIN olap_data_sources_table_field odstf ON ocm.relation_field_id = odstf.id "+
//                " LEFT JOIN olap_data_sources_table odst ON odst.id = odstf.table_id " +
//                " LEFT JOIN olap_cube oc ON oc.id = ocm.cube_id " +
//                " LEFT JOIN olap_dimension od ON ocm.dimension_id = od.id where ocm.child_save_type is null and  odstf.field_code = ? and odst.table_code = ? ";
//        String param[] = {RelationFieldCode,tableCode};
//        List<CubeMappingModel> cubeMappingModels = jdbcTemplate.query(sql, new BeanPropertyRowMapper(CubeMappingModel.class), param);
//        return cubeMappingModels;
//    }
}

src/main/java/com/yihu/quota/service/cube/ElasticSearchDataProcessService.java

@@ -12,6 +12,7 @@ import com.yihu.quota.model.source.DataSourcesTableField;
import com.yihu.quota.service.source.DataSourcesTableFieldService;
import com.yihu.quota.vo.CubeMappingModel;
import com.yihu.quota.vo.CubeMemberMappingModel;
import com.yihu.quota.vo.DataSourcesTableModel;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -32,6 +33,7 @@ public class ElasticSearchDataProcessService {
    private static String dataSource_mysql = "mysql";
    private static String action_del = "DeleteFamily";//删除整行
    private static String action_put = "Put";                //添加和修改单个字段值
    private static String action_delAll = "delAll";
    private static String dataSource_k = "dataSource";
    private static String database_k = "database";
@@ -40,7 +42,6 @@ public class ElasticSearchDataProcessService {
    private static String rowKey_k = "rowkey";
    private static String profileId_k = "profile_id";
    private static String action_k = "action";
    private static String action_delAll = "delAll";
    @Autowired
    private JdbcBasicService jdbcBasicService;
@@ -67,9 +68,9 @@ public class ElasticSearchDataProcessService {
                String dataSource = dataMap.get(dataSource_k).toString();
                dataMap.remove(dataSource_k);
                if(dataSource.toLowerCase().equals(dataSource_hbase)){
                     dataProcess(dataMap);
                    dataProcess(dataMap);
                }else if(dataSource.toLowerCase().equals(dataSource_mysql)){
                     dataProcess(dataMap);
                    dataProcess(dataMap);
                }
            }
        } catch (Exception e) {
@@ -84,7 +85,6 @@ public class ElasticSearchDataProcessService {
     * 如果是 删除 子表中的一行数据，es 这边就无法确定删除哪个数据
     */
    public void dataProcess(Map<String, Object> dataMap) throws Exception{
        Map<String, Object> source  = new HashMap<>();
        String table = "";
        String rowKey = "";
        String profileId = "";
@@ -98,8 +98,6 @@ public class ElasticSearchDataProcessService {
        }
        if(dataMap.containsKey(rowKey_k)){
            rowKey = dataMap.get(rowKey_k).toString();
            source.put(id_k,rowKey);
            source.put(rowKey_k,rowKey);
        }
        if(dataMap.containsKey(profileId_k)){
            profileId = dataMap.get(profileId_k).toString();
@@ -111,20 +109,28 @@ public class ElasticSearchDataProcessService {
            dataMap.remove(table_k);
            dataMap.remove(rowKey_k);
            dataMap.remove(action_k);
            String baseCloumnValue = "";
            String baseCloumnValue = null;
            if(action.contains(action_put)){
                for(String baseCloumnCode : dataMap.keySet()){
                    if(dataMap.get(baseCloumnCode)!= null){
                        baseCloumnValue = dataMap.get(baseCloumnCode).toString();
                    }else {
                        baseCloumnValue = null;
                    }
                    System.out.println("开始解析数据：" + baseCloumnCode + " ,值为 = " + baseCloumnValue);
                    List<CubeMappingModel> cubeMappingModels = cubeMappingService.findCubeMappingModelsByFieldCode(table, baseCloumnCode);
                    if(cubeMappingModels != null && cubeMappingModels.size() > 0){
                        for(CubeMappingModel cubeMappingModel :cubeMappingModels){
                            Map<String, Object> source  = new HashMap<>();
                            source.put(id_k,rowKey);
                            source.put(rowKey_k,rowKey);
                            String index = cubeMappingModel.getIndexName();
                            String type = cubeMappingModel.getIndexType();
                            String cloumnCode = cubeMappingModel.getDimensionCode();
                            System.out.println("维度code = " + cloumnCode +  ",维度类型：" + cubeMappingModel.getDataType());
                            //维度数据扩展保存
                            Map<String, Object> esDataMap = dimensionDataExtendToMap(cubeMappingModel,cloumnCode,baseCloumnValue);
                            Map<String, Object> esDataMap = new HashMap<>();
                            esDataMap = dimensionDataExtendToMap(cubeMappingModel,cloumnCode,baseCloumnValue);
                            source.putAll(esDataMap);
                            saveElasticSearchData(index, type,rowKey,source);
                        }
@@ -142,46 +148,38 @@ public class ElasticSearchDataProcessService {
                            String subRowKey = rowKey;
                            if(action.contains(action_put)){
                                //维度成员数据扩展保存
                                Map<String, Object> source  = new HashMap<>();
                                source = dimensionMemberDataExtendToMap(cubeMemberMappingModel,baseCloumnValue,subRowKey,profileId);
                                saveElasticSearchData(index, type,profileId,source);
                            }
                        }
                    }
                    //是否为关联字段 start
                    List<CubeMappingModel> relationMappingModels = cubeMappingService.findRelationMappingModelsByFieldCode(table, baseCloumnCode);
                    if(relationMappingModels != null && relationMappingModels.size() > 0) {
                        for (CubeMappingModel cubeMappingModel : relationMappingModels) {
                            DataSourcesTableField dataSourcesTableField = dataSourcesTableFieldService.findById(cubeMappingModel.getDataFieldId());
                            Object object = jdbcBasicService.getEntityByRelationId(database ,table, dataSourcesTableField.getFieldCode(), cubeMappingModel.getRelationFieldId());
                            String value = converMapObject(object);
                            String index = cubeMappingModel.getIndexName();
                            String type = cubeMappingModel.getIndexType();
                            String cloumnCode = cubeMappingModel.getDimensionCode();
                            //维度数据扩展保存
                            Map<String, Object> esDataMap = dimensionDataExtendToMap(cubeMappingModel,cloumnCode,value);
                            source.putAll(esDataMap);
                            saveElasticSearchData(index, type,rowKey,source);
                        }
                    }
//                    //主表字段关联出相关表数据 start
                    List<CubeMemberMappingModel> relationMemberMappingModels = cubeMemberMappingService.findRelationMemberMappingModels(table,baseCloumnCode);
                    if(relationMemberMappingModels != null && relationMemberMappingModels.size() > 0) {
                        profileId = rowKey;
                        for (CubeMemberMappingModel cubeMemberMappingModel : relationMemberMappingModels) {
                            //父级对象
                            if(cubeMemberMappingModel.getChildSaveType() == 1){
                                DataSourcesTableField dataSourcesTableField = dataSourcesTableFieldService.findById(cubeMemberMappingModel.getDataFieldId());
                                String value = jdbcBasicService.getEntityByRelationId(database ,table, dataSourcesTableField.getFieldCode(), cubeMemberMappingModel.getRelationFieldId());
                                String index = cubeMemberMappingModel.getIndexName();
                                String type = cubeMemberMappingModel.getIndexType();
                                String subRowKey = rowKey;
                                if(action.contains(action_put)){
                                    //维度成员数据扩展保存
                                    source = dimensionMemberDataExtendToMap(cubeMemberMappingModel,value,subRowKey,profileId);
                                    saveElasticSearchData(index, type,profileId,source);
                            if(cubeMemberMappingModel.getChildSaveType() == 1 && cubeMemberMappingModel.getRelationSubFieldId() != null){
                                DataSourcesTableModel dataSourcesTableModel = dataSourcesTableFieldService.findDataSourcesTableModel(cubeMemberMappingModel.getDataFieldId());
                                DataSourcesTableModel subdataSourcesTableModel = dataSourcesTableFieldService.findDataSourcesTableModel(cubeMemberMappingModel.getRelationSubFieldId());
                                if(dataSourcesTableModel != null && subdataSourcesTableModel != null){
                                    String value = jdbcBasicService.getEntityByRelationId(subdataSourcesTableModel.getDatabaseName() ,subdataSourcesTableModel.getTableCode(), dataSourcesTableModel.getFieldCode(),subdataSourcesTableModel.getFieldCode(),subdataSourcesTableModel.getFieldType(),baseCloumnValue);
                                    if(StringUtils.isNotEmpty(value)){
                                        String index = cubeMemberMappingModel.getIndexName();
                                        String type = cubeMemberMappingModel.getIndexType();
                                        if(action.contains(action_put)){
                                            //维度成员数据扩展保存
                                            Map<String, Object> source  = new HashMap<>();
                                            source = dimensionMemberDataExtendToMap(cubeMemberMappingModel,value,null,profileId);
                                            saveElasticSearchData(index, type,profileId,source);
                                        }
                                    }
                                }
                            }
                        }
                    }
                    //是否为关联字段 end
                    //主表字段关联出相关表数据 end
                }
            }else if(action.contains(action_del)){
                //一个表只能对应到一个 索引type
@@ -191,7 +189,7 @@ public class ElasticSearchDataProcessService {
                }else {
                    throw new Exception("视图，表不存在");
                }
            } else if (action.contains(action_delAll)) {
            }else if (action.contains(action_delAll)) {
                String cubeId = "";
                if (dataMap.containsKey("cubeId")) {
                    cubeId = dataMap.remove("cubeId").toString();
@@ -442,31 +440,37 @@ public class ElasticSearchDataProcessService {
     * @param dataType
     * @param keyValue
     */
    public Object dataConver(String dataType,String keyValue){
        NumberFormat nf = NumberFormat.getInstance();
    public Object dataConver(String dataType,String keyValue) throws Exception{
        Object value = null;
        dataType = dataType.toLowerCase();
        if(dataType.equals("string")){
            value = keyValue;
        }else if(dataType.equals("integer")){
            int intValue = Integer.valueOf(keyValue);
            value = intValue;
        }else if(dataType.equals("double")){
            nf.setGroupingUsed(false);
            nf.setMaximumFractionDigits(2);
            double doubleValue = Double.valueOf(keyValue);
            value = doubleValue;
        }else if(dataType.equals("date")){
            String dateFormat = "yyyy-MM-dd";
            if(keyValue.length() != 10){
                dateFormat = "yyyy-MM-dd'T'HH:mm:ssZ";
        NumberFormat nf = NumberFormat.getInstance();
        try {
            if(StringUtils.isNotEmpty(keyValue)){
                dataType = dataType.toLowerCase();
                if(dataType.equals("string")){
                    value = keyValue;
                }else if(dataType.equals("integer")){
                    int intValue = Integer.valueOf(keyValue);
                    value = intValue;
                }else if(dataType.equals("double")){
                    nf.setGroupingUsed(false);
                    nf.setMaximumFractionDigits(2);
                    double doubleValue = Double.valueOf(keyValue);
                    value = doubleValue;
                }else if(dataType.equals("date")){
                    String dateFormat = DateUtil.DEFAULT_DATE_YMD_FORMAT;
                    if(keyValue.length() > 10){
                        dateFormat = DateUtil.DEFAULT_YMDHMSDATE_FORMAT;;
                    }
                    Date dateValue = DateUtil.parseDate(keyValue, dateFormat);
                    Calendar ca = Calendar.getInstance();
                    ca.setTime(dateValue);
                    ca.add(Calendar.HOUR_OF_DAY, 8);
                    value = ca.getTime();
                }
            }
            //比中国所在地区东八区少8小时.
            Date dateValue = DateUtil.parseDate(keyValue, dateFormat);
            Calendar ca = Calendar.getInstance();
            ca.setTime(dateValue);
            ca.add(Calendar.HOUR_OF_DAY, 8);
            value = ca.getTime();
        }catch (Exception e){
            e.printStackTrace();
            logger.debug("数据转换异常！");
        }
        return value;
    }
@@ -492,4 +496,5 @@ public class ElasticSearchDataProcessService {
        }
        return index;
    }
}

src/main/java/com/yihu/quota/service/cube/JdbcBasicService.java

@@ -16,9 +16,13 @@ public class JdbcBasicService {
    private JdbcBasicDao jdbcBasicDao;
    public String getEntityByRelationId(String database ,String table, String cloumnCode ,String relationId)
    {
        String sql = " SELECT " + cloumnCode + " FROM " + table + " WHERE " + relationId + "= " + relationId;
    public String getEntityByRelationId(String database ,String table, String cloumnCode ,String relationCode,String relationColDataType ,String relationId) throws Exception {
        String sql = "";
        if(relationColDataType.toLowerCase().equals("string")){
            sql = " SELECT " + cloumnCode + " FROM " + table + " WHERE " + relationCode + "= '" + relationId + "' ";
        }else {
            sql = " SELECT " + cloumnCode + " FROM " + table + " WHERE " + relationCode + "= " + relationId;
        }
        return jdbcBasicDao.getConnection(sql,database,cloumnCode);
    }

src/main/java/com/yihu/quota/service/source/DataSourcesTableFieldService.java

@@ -3,7 +3,9 @@ package com.yihu.quota.service.source;
import com.yihu.ehr.query.BaseJpaService;
import com.yihu.quota.dao.source.DataSourcesTableFieldDao;
import com.yihu.quota.model.source.DataSourcesTableField;
import com.yihu.quota.vo.DataSourcesTableModel;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.BeanPropertyRowMapper;
import org.springframework.stereotype.Service;
import java.util.Arrays;
@@ -42,4 +44,21 @@ public class DataSourcesTableFieldService extends BaseJpaService<DataSourcesTabl
    public DataSourcesTableField findById(int id){
        return dataSourcesTableFieldDao.findOne(id);
    }
    /**
     * 根据 表编码和关联字段编码 查询对应的维度信息
     * @param fieldId 关联字段
     * @return
     */
    public DataSourcesTableModel findDataSourcesTableModel(int fieldId) {
        String sql = "SELECT odstf.*,odst.table_code,odst.database_name from olap_data_sources_table_field odstf  " +
                " LEFT JOIN olap_data_sources_table odst on odst.id = odstf.table_id " +
                " WHERE odstf.id = ? ";
        String param[] = {String.valueOf(fieldId)};
        List<DataSourcesTableModel> cubeMemberMappingModels = jdbcTemplate.query(sql, new BeanPropertyRowMapper(DataSourcesTableModel.class), param);
        if(cubeMemberMappingModels != null && cubeMemberMappingModels.size() > 0){
            return  cubeMemberMappingModels.get(0);
        }
        return null;
    }
}

src/main/java/com/yihu/quota/vo/CubeMemberMappingModel.java

@@ -23,6 +23,9 @@ public class CubeMemberMappingModel {
    private Integer isPrimarykey;  //是否指定唯一字段 1是 0 否
    private String dataGetType;        //数据获取方式 1 直接获取 2 关联获取（一对一关联不支持一对多关联） 默认为一
    private String relationFieldId;    //关联主键字段
    private Integer relationSubFieldId;    //关联细表主键字段
    private String basedata;
    private String table;
    public String getDataGetType() {
        return dataGetType;
@@ -167,4 +170,28 @@ public class CubeMemberMappingModel {
    public void setIsPrimarykey(Integer isPrimarykey) {
        this.isPrimarykey = isPrimarykey;
    }
    public Integer getRelationSubFieldId() {
        return relationSubFieldId;
    }
    public void setRelationSubFieldId(Integer relationSubFieldId) {
        this.relationSubFieldId = relationSubFieldId;
    }
    public String getBasedata() {
        return basedata;
    }
    public void setBasedata(String basedata) {
        this.basedata = basedata;
    }
    public String getTable() {
        return table;
    }
    public void setTable(String table) {
        this.table = table;
    }
}