es数据解析 解析

src/main/java/com/yihu/quota/controller/ViewController.java

@@ -1,6 +1,7 @@
package com.yihu.quota.controller;
import com.fasterxml.jackson.core.type.TypeReference;
import com.github.abel533.echarts.Option;
import com.yihu.ehr.constants.ApiVersion;
import com.yihu.ehr.constants.ServiceApi;
import com.yihu.ehr.controller.EnvelopRestEndPoint;
@@ -11,13 +12,22 @@ import com.yihu.quota.service.cube.CubeService;
import com.yihu.quota.service.view.ViewDimensionService;
import com.yihu.quota.service.view.ViewQuotaService;
import com.yihu.quota.service.view.ViewService;
import com.yihu.quota.util.AggregationBuildHandler;
import com.yihu.quota.util.ElasticSearchHandler;
import com.yihu.quota.util.ReportOption;
import com.yihu.quota.vo.ViewQuotaFilterModel;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
@@ -25,7 +35,9 @@ import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
/**
 * @author janseny
@@ -41,11 +53,8 @@ public class ViewController extends EnvelopRestEndPoint {
    @Autowired
    private CubeService cubeService;
    @Autowired
    private ViewDimensionService viewDimensionService;
    @Autowired
    private ViewQuotaService viewQuotaService;
    @Autowired
    private ElasticSearchHandler elasticSearchHandler;
    private AggregationBuildHandler aggregationBuildHandler;
    @ApiOperation(value = "添加视图")
    @RequestMapping(value = ServiceApi.StatisticView.View.Add, method = RequestMethod.GET)
@@ -112,10 +121,29 @@ public class ViewController extends EnvelopRestEndPoint {
                filterModelList = objectMapper.readValue(filters, new TypeReference<List<ViewQuotaFilterModel>>() {
                });
            }
            View view = viewService.findByCode(viewCode);
            if(view == null){
                logger.debug("视图不存在");
                return null;
            }
            Aggregations aggregations = viewService.statViewResult(viewCode, filterModelList);
            if(view.getDisplayType().equals("1")){
                //数值型
                Map<String, Object> resultMap = aggregationBuildHandler.numericalDataParsing(view, aggregations.getAsMap());
                envelop.setObj(resultMap);
            }else if(view.getDisplayType().equals("2")){
                //普通表格
                List<Map<String, Object>> resultList = aggregationBuildHandler.tableDataParsing(view, aggregations.getAsMap());
                envelop.setObj(resultList);
            }else if(view.getDisplayType().equals("3")){
                //树形表格
            // TODO 将统计结果转换成展示的数据格式
            }else {
                //图表
                Option option = aggregationBuildHandler.ehartDataParsing(view, aggregations.getAsMap());
                String echartJson = option.toString();
                envelop.setObj(echartJson);
            }
        } catch (Exception e) {
            e.printStackTrace();
            envelop.setSuccessFlg(false);

src/main/java/com/yihu/quota/service/view/ViewService.java

@@ -86,6 +86,10 @@ public class ViewService extends BaseJpaService<View, ViewDao> {
        return viewDao.findOne(id);
    }
    public View findByCode(String viewCode){
        return viewDao.findByCode(viewCode);
    }
    /**
     * 获取视图完整规则
     *

src/main/java/com/yihu/quota/util/AggregationBuildHandler.java

@@ -53,94 +53,94 @@ public class AggregationBuildHandler {
    @Autowired
    private CubeService cubeService;
    /**
     * 简单聚合
     * @param aggType 聚合类型
     * @param aggName 聚合名称
     * @param fieldName 聚合字段
     * @return
     */
    public AbstractAggregationBuilder addAggregationBuilder(String aggType,String aggName, String fieldName){
        AbstractAggregationBuilder builder = null;
        if(aggType.equals("sum")){
            builder = AggregationBuilders.sum(aggName).field(fieldName);
        }else if(aggType.equals("count")){
            builder = AggregationBuilders.count(aggName).field(fieldName);
        }else if(aggType.equals("avg")){
            builder = AggregationBuilders.avg(aggName).field(fieldName);
        }else if(aggType.equals("max")){
            builder = AggregationBuilders.max(aggName).field(fieldName);
        }else if(aggType.equals("min")){
            builder = AggregationBuilders.min(aggName).field(fieldName);
        }
        return  builder;
    }
    /**
     * 分组后，简单聚合
     * @param aggType 聚合类型
     * @param aggName 聚合名称
     * @param fieldName 聚合字段
     * @return
     */
    public AbstractAggregationBuilder addTermAggregationBuilder(String termName, String termFieldName,String aggType,String aggName, String fieldName){
        AbstractAggregationBuilder builder = null;
        TermsBuilder termsBuilder = addTermsBuilder(termName,termFieldName);
        if(aggType.equals("sum")){
            builder = termsBuilder.subAggregation(AggregationBuilders.sum(aggName).field(fieldName));
        }else if(aggType.equals("count")){
            builder = termsBuilder.subAggregation(AggregationBuilders.count(aggName).field(fieldName));
        }else if(aggType.equals("avg")){
            builder = termsBuilder.subAggregation(AggregationBuilders.avg(aggName).field(fieldName));
        }else if(aggType.equals("max")){
            builder = termsBuilder.subAggregation(AggregationBuilders.max(aggName).field(fieldName));
        }else if(aggType.equals("min")){
            builder = termsBuilder.subAggregation(AggregationBuilders.min(aggName).field(fieldName));
        }
        return  builder;
    }
    /**
     * 添加分组
     * @param termName
     * @param fieldName
     * @return
     */
    public TermsBuilder addTermsBuilder(String termName, String fieldName){
        TermsBuilder termBuilder = AggregationBuilders.terms(termName).field(fieldName);
        return termBuilder;
    }
    /**
     *
     * @param client
     * @param boolQueryBuilder  查询的过滤条件
     * @param aggBuilderList  聚合组
     *  聚合组中成员是 单个的聚合查询，其中单个的聚合查询可以嵌套子聚合查询 如：
     *  单个聚合 查询 ：TermsBuilder firstAgg= AggregationBuilders.terms("player_count ").field("team");
     *  带有子聚合的聚合查询 ：
     *    TermsBuilder secondAgg= AggregationBuilders.terms("pos_count").field("position")
     *      .subAggregation(
     *          AggregationBuilders.dateHistogram("by_year").field("dateOfBirth").interval((DateHistogramInterval.YEAR))
     *              .subAggregation(
     *                  AggregationBuilders.avg("avg_children").field("children")
     *              )
     *          );
     * @return 聚合结果集
     */
    public Map<String, Aggregation> structAggregationQuery(TransportClient client,String index,String type,
                                                            BoolQueryBuilder boolQueryBuilder,
                                                            LinkedList<AbstractAggregationBuilder> aggBuilderList){
        SearchRequestBuilder searchRequestBuilder = client.prepareSearch(index).setTypes(type).setQuery(boolQueryBuilder);
        for(AbstractAggregationBuilder aggBuilder : aggBuilderList){
            searchRequestBuilder.addAggregation(aggBuilder);
        }
        SearchResponse response = searchRequestBuilder.execute().actionGet();
        Map<String, Aggregation> map = response.getAggregations().getAsMap();
        client.close();
        return map;
    }
//    /**
//     * 简单聚合
//     * @param aggType 聚合类型
//     * @param aggName 聚合名称
//     * @param fieldName 聚合字段
//     * @return
//     */
//    public AbstractAggregationBuilder addAggregationBuilder(String aggType,String aggName, String fieldName){
//        AbstractAggregationBuilder builder = null;
//        if(aggType.equals("sum")){
//            builder = AggregationBuilders.sum(aggName).field(fieldName);
//        }else if(aggType.equals("count")){
//            builder = AggregationBuilders.count(aggName).field(fieldName);
//        }else if(aggType.equals("avg")){
//            builder = AggregationBuilders.avg(aggName).field(fieldName);
//        }else if(aggType.equals("max")){
//            builder = AggregationBuilders.max(aggName).field(fieldName);
//        }else if(aggType.equals("min")){
//            builder = AggregationBuilders.min(aggName).field(fieldName);
//        }
//        return  builder;
//    }
//    /**
//     * 分组后，简单聚合
//     * @param aggType 聚合类型
//     * @param aggName 聚合名称
//     * @param fieldName 聚合字段
//     * @return
//     */
//    public AbstractAggregationBuilder addTermAggregationBuilder(String termName, String termFieldName,String aggType,String aggName, String fieldName){
//        AbstractAggregationBuilder builder = null;
//        TermsBuilder termsBuilder = addTermsBuilder(termName,termFieldName);
//        if(aggType.equals("sum")){
//            builder = termsBuilder.subAggregation(AggregationBuilders.sum(aggName).field(fieldName));
//        }else if(aggType.equals("count")){
//            builder = termsBuilder.subAggregation(AggregationBuilders.count(aggName).field(fieldName));
//        }else if(aggType.equals("avg")){
//            builder = termsBuilder.subAggregation(AggregationBuilders.avg(aggName).field(fieldName));
//        }else if(aggType.equals("max")){
//            builder = termsBuilder.subAggregation(AggregationBuilders.max(aggName).field(fieldName));
//        }else if(aggType.equals("min")){
//            builder = termsBuilder.subAggregation(AggregationBuilders.min(aggName).field(fieldName));
//        }
//        return  builder;
//    }
//    /**
//     * 添加分组
//     * @param termName
//     * @param fieldName
//     * @return
//     */
//    public TermsBuilder addTermsBuilder(String termName, String fieldName){
//        TermsBuilder termBuilder = AggregationBuilders.terms(termName).field(fieldName);
//        return termBuilder;
//    }
//    /**
//     *
//     * @param client
//     * @param boolQueryBuilder  查询的过滤条件
//     * @param aggBuilderList  聚合组
//     *  聚合组中成员是 单个的聚合查询，其中单个的聚合查询可以嵌套子聚合查询 如：
//     *  单个聚合 查询 ：TermsBuilder firstAgg= AggregationBuilders.terms("player_count ").field("team");
//     *  带有子聚合的聚合查询 ：
//     *    TermsBuilder secondAgg= AggregationBuilders.terms("pos_count").field("position")
//     *      .subAggregation(
//     *          AggregationBuilders.dateHistogram("by_year").field("dateOfBirth").interval((DateHistogramInterval.YEAR))
//     *              .subAggregation(
//     *                  AggregationBuilders.avg("avg_children").field("children")
//     *              )
//     *          );
//     * @return 聚合结果集
//     */
//    public Map<String, Aggregation> structAggregationQuery(TransportClient client,String index,String type,
//                                                            BoolQueryBuilder boolQueryBuilder,
//                                                            LinkedList<AbstractAggregationBuilder> aggBuilderList){
//        SearchRequestBuilder searchRequestBuilder = client.prepareSearch(index).setTypes(type).setQuery(boolQueryBuilder);
//        for(AbstractAggregationBuilder aggBuilder : aggBuilderList){
//            searchRequestBuilder.addAggregation(aggBuilder);
//        }
//        SearchResponse response = searchRequestBuilder.execute().actionGet();
//        Map<String, Aggregation> map = response.getAggregations().getAsMap();
//        client.close();
//        return map;
//    }
    /**
     * 表格型 - 数据结果解析
@@ -149,15 +149,8 @@ public class AggregationBuildHandler {
     * @return
     */
    public List<Map<String, Object>> tableDataParsing(View view,Map<String, Aggregation> map){
//        List<String> rowList = new LinkedList<>();
//        rowList.add("town_terms");
//        rowList.add("total");
//        List<String> cloumnList = new ArrayList<>();
//        cloumnList.add("total_count_result");
//        cloumnList.add("sex_terms");
        List<String> rowList = getRowDimensionList(view.getId());
        List<String> cloumnList = getColumnDimensionList(view.getId());
        List<ViewQuota> viewQuotas = viewQuotaDao.findByViewId(view.getId());
        Map<String, Object> quotaMap = new HashMap<>();
        for(ViewQuota viewQuota :viewQuotas){
@@ -282,32 +275,22 @@ public class AggregationBuildHandler {
     * @param map 聚合查询结果
     * @return
     */
    public Option ehartDataParsing(View view,Map<String, Aggregation> map){
    public Option ehartDataParsing(View view,Map<String, Aggregation> map) throws  Exception{
        Option option = null;
        ReportOption reportOption = new ReportOption();
        List<String> xDataList = new LinkedList<>();
        List<String> charTypes = new LinkedList<>();
        List<String> lineNames = new LinkedList<>();
        List<List<Object>> lineDataList = new LinkedList<>();
        List<String> rowList = new LinkedList<>();
        rowList.add("town_terms");
//        rowList = getRowDimensionList(view.getId());
        List<String> rowList = getRowDimensionList(view.getId());
        Map<String,List<Object>> quotasDataMap = new LinkedHashMap<>();
        if(rowList.size() != 1){
            logger.debug("图表型指标维度配置有误");
            return null;
            throw new Exception("图表型指标维度配置有误");
        }
        String rowAggCode = rowList.get(0);
        List<String> quotaList = new LinkedList<>();
        Map<String,List<Object>> quotasDataMap = new LinkedHashMap<>();
        List<ViewQuota> viewQuotas = new ArrayList<>();
        ViewQuota vq = new ViewQuota();
        vq.setChartType("1");
        vq.setCode("total");
        vq.setName("各区县总人数");
        viewQuotas.add(vq);
//        viewQuotas = viewQuotaDao.findByViewId(view.getId());
        List<ViewQuota> viewQuotas = viewQuotaDao.findByViewId(view.getId());
        for(ViewQuota viewQuota :viewQuotas){
            quotaList.add(viewQuota.getCode());
            charTypes.add(viewQuota.getChartType());
@@ -318,14 +301,14 @@ public class AggregationBuildHandler {
            Terms terms = (Terms)aggregation;
            String key = "";
            List<Terms.Bucket> buckets =  terms.getBuckets();
            for(String quotaCode: quotaList){
            for(ViewQuota viewQuota: viewQuotas){
                List<Object> quotaData = new ArrayList<>();
                for(Terms.Bucket bucket : buckets){
                    key = bucket.getKeyAsString() != null ? bucket.getKeyAsString() : bucket.getKey().toString();
                    xDataList.add(key);
                    List<Aggregation> aggregationList = bucket.getAggregations().asList();
                    for(Aggregation subAgg : aggregationList){
                        if(subAgg.getName().contains(quotaCode)){
                        if(subAgg.getName().contains(viewQuota.getBasicFormulaType())){
                            Map<String, Object> subDataMap = getInternalAggValue(subAgg.getName(),subAgg);
                            if(subDataMap != null || subDataMap.size() > 0){
                                quotaData.add(subDataMap.get(subAgg.getName()));
@@ -333,7 +316,7 @@ public class AggregationBuildHandler {
                        }
                    }
                }
                quotasDataMap.put(quotaCode, quotaData);
                quotasDataMap.put(viewQuota.getCode(), quotaData);
            }
        }
        for (Map.Entry<String,List<Object>> entry : quotasDataMap.entrySet()) {