|
@ -3,6 +3,7 @@ package com.yihu.quota.service.view;
|
|
|
import com.fasterxml.jackson.core.type.TypeReference;
|
|
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
|
import com.yihu.ehr.elasticsearch.ElasticSearchPool;
|
|
|
import com.yihu.ehr.exception.ApiException;
|
|
|
import com.yihu.ehr.query.BaseJpaService;
|
|
|
import com.yihu.quota.dao.view.ViewDao;
|
|
|
import com.yihu.quota.dao.view.ViewDimensionDao;
|
|
@ -13,7 +14,11 @@ import com.yihu.quota.model.view.View;
|
|
|
import com.yihu.quota.model.view.ViewDimension;
|
|
|
import com.yihu.quota.model.view.ViewQuota;
|
|
|
import com.yihu.quota.model.view.ViewQuotaFilter;
|
|
|
import com.yihu.quota.service.cube.CubeService;
|
|
|
import com.yihu.quota.vo.ViewQuotaFilterModel;
|
|
|
import io.searchbox.core.search.aggregation.DateHistogramAggregation;
|
|
|
import io.searchbox.core.search.aggregation.FilterAggregation;
|
|
|
import io.searchbox.core.search.aggregation.ValueCountAggregation;
|
|
|
import org.apache.commons.lang3.StringUtils;
|
|
|
import org.apache.log4j.Logger;
|
|
|
import org.elasticsearch.action.search.SearchRequestBuilder;
|
|
@ -22,9 +27,25 @@ import org.elasticsearch.client.transport.TransportClient;
|
|
|
import org.elasticsearch.index.query.BoolQueryBuilder;
|
|
|
import org.elasticsearch.index.query.QueryBuilder;
|
|
|
import org.elasticsearch.index.query.QueryBuilders;
|
|
|
import org.elasticsearch.search.aggregations.AggregationBuilder;
|
|
|
import org.elasticsearch.search.aggregations.AggregationBuilders;
|
|
|
import org.elasticsearch.search.aggregations.Aggregations;
|
|
|
import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
|
|
|
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramBuilder;
|
|
|
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
|
|
|
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
|
|
|
import org.elasticsearch.search.aggregations.bucket.terms.LongTerms;
|
|
|
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
|
|
|
import org.elasticsearch.search.aggregations.bucket.terms.TermsBuilder;
|
|
|
import org.elasticsearch.search.aggregations.metrics.MetricsAggregationBuilder;
|
|
|
import org.elasticsearch.search.aggregations.metrics.avg.AvgBuilder;
|
|
|
import org.elasticsearch.search.aggregations.metrics.cardinality.Cardinality;
|
|
|
import org.elasticsearch.search.aggregations.metrics.cardinality.CardinalityBuilder;
|
|
|
import org.elasticsearch.search.aggregations.metrics.max.MaxBuilder;
|
|
|
import org.elasticsearch.search.aggregations.metrics.min.MinBuilder;
|
|
|
import org.elasticsearch.search.aggregations.metrics.sum.SumBuilder;
|
|
|
import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCountBuilder;
|
|
|
import org.elasticsearch.search.sort.SortBuilder;
|
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
|
import org.springframework.stereotype.Service;
|
|
|
|
|
@ -56,6 +77,8 @@ public class ViewService extends BaseJpaService<View, ViewDao> {
|
|
|
private ElasticSearchPool elasticSearchPool;
|
|
|
@Autowired
|
|
|
private ObjectMapper objectMapper;
|
|
|
@Autowired
|
|
|
private CubeService cubeService;
|
|
|
|
|
|
public View findOne(String id) {
|
|
|
return viewDao.findOne(id);
|
|
@ -95,12 +118,12 @@ public class ViewService extends BaseJpaService<View, ViewDao> {
|
|
|
/**
|
|
|
* 获取视图统计结果
|
|
|
*
|
|
|
* @param viewCode
|
|
|
* @param filterModelList
|
|
|
* @param viewCode 视图编码
|
|
|
* @param filterModelList 页面的过滤条件模型集合
|
|
|
*/
|
|
|
public Aggregations statViewResult(String viewCode, List<ViewQuotaFilterModel> filterModelList) throws IOException {
|
|
|
public Aggregations statViewResult(String viewCode, List<ViewQuotaFilterModel> filterModelList) throws Exception {
|
|
|
// 获取视图规则
|
|
|
View view = viewDao.findByCode(viewCode);
|
|
|
View view = this.getViewRule(viewCode);
|
|
|
String[] esIndices = view.getEsIndex().split(",");
|
|
|
String[] esTypes = view.getEsType().split(",");
|
|
|
|
|
@ -109,22 +132,98 @@ public class ViewService extends BaseJpaService<View, ViewDao> {
|
|
|
|
|
|
// 组装页面传入的过滤条件
|
|
|
BoolQueryBuilder viewBoolQueryBuilder = QueryBuilders.boolQuery();
|
|
|
joinViewFilters(viewBoolQueryBuilder, filterModelList);
|
|
|
this.joinFilters(viewBoolQueryBuilder, filterModelList);
|
|
|
// 组装视图默认过滤条件
|
|
|
String viewFilterStr = objectMapper.writeValueAsString(view.getViewFilterList());
|
|
|
List<ViewQuotaFilterModel> viewFilterModelList = objectMapper.readValue(viewFilterStr, new TypeReference<List<ViewQuotaFilterModel>>() {
|
|
|
});
|
|
|
joinViewFilters(viewBoolQueryBuilder, viewFilterModelList);
|
|
|
this.joinFilters(viewBoolQueryBuilder, viewFilterModelList);
|
|
|
searchRequestBuilder.setQuery(viewBoolQueryBuilder);
|
|
|
|
|
|
// 组装聚合
|
|
|
// 顶层行维度
|
|
|
List<ViewDimension> groupTopRowDimensionList = view.getGroupTopRowDimensionList();
|
|
|
if (groupTopRowDimensionList.size() != 0) {
|
|
|
// 组内其他行维度
|
|
|
Map<Integer, List<ViewDimension>> groupOtherRowDimensionsMap = view.getGroupOtherRowDimensionsMap();
|
|
|
if (groupTopRowDimensionList.size() == 0) {
|
|
|
if (!"1".equals(view.getDisplayType())) {
|
|
|
throw new ApiException("请为视图至少配置一个行维度。");
|
|
|
} else {
|
|
|
// 视图为数值展示类型的场合,没有行/列维度,直接对指标聚合。
|
|
|
this.gatherMetricsAgg(searchRequestBuilder, view.getViewQuotaList());
|
|
|
}
|
|
|
} else {
|
|
|
int indexCount = esIndices.length;
|
|
|
for (ViewDimension viewDimension : groupTopRowDimensionList) {
|
|
|
// 有多个多维度数据集时,设定顶层聚合的数据来自哪个多维数据集
|
|
|
// 遍历顶层行维度
|
|
|
for (ViewDimension topRowDimension : groupTopRowDimensionList) {
|
|
|
String topRowDimensionCode = topRowDimension.getDimensionCode();
|
|
|
String topOrderType = topRowDimension.getMemberOrderType();
|
|
|
boolean isTopAsc = "asc".equals(topRowDimension.getMemberOrderStrategy()) ? true : false;
|
|
|
|
|
|
// 有多个多维度数据集时,设定顶层聚合的数据来自哪个多维数据集。
|
|
|
FilterAggregationBuilder topFilterAgg = null;
|
|
|
if (indexCount > 1) {
|
|
|
AggregationBuilders.filter(viewDimension.getDimensionCode());
|
|
|
String aggName = topRowDimension.getDimensionCode() + "-index-filter";
|
|
|
QueryBuilder queryBuilder = QueryBuilders.termQuery("_index", topRowDimension.getEsIndex());
|
|
|
topFilterAgg = AggregationBuilders.filter(aggName).filter(queryBuilder);
|
|
|
}
|
|
|
|
|
|
// TODO 当顶层行维度是合计维度时,合计维度与列维度组合的聚合都在聚合节点顶层。
|
|
|
|
|
|
// 顶层行维度聚合
|
|
|
AggregationBuilder topAgg = null;
|
|
|
String fieldType = cubeService.findDimensionDataType(topRowDimension.getCubeCode(), topRowDimension.getDimensionCode());
|
|
|
if (!"date".equals(fieldType)) {
|
|
|
String aggName = topRowDimensionCode + "-terms";
|
|
|
topAgg = AggregationBuilders.terms(aggName).field(topRowDimensionCode);
|
|
|
|
|
|
// 设置排序
|
|
|
if (StringUtils.isEmpty(topOrderType) || "name".equals(topOrderType)) {
|
|
|
((TermsBuilder) topAgg).order(Terms.Order.term(isTopAsc));
|
|
|
} else {
|
|
|
// TODO 按子聚合结果排序,需拼接子聚合名称
|
|
|
((TermsBuilder) topAgg).order(Terms.Order.aggregation("", isTopAsc));
|
|
|
}
|
|
|
|
|
|
// 设置分组个数
|
|
|
int memberCount = topRowDimension.getMemberCount();
|
|
|
if (memberCount > 0) {
|
|
|
((TermsBuilder) topAgg).size(memberCount);
|
|
|
} else {
|
|
|
// TODO 动态获取全部的分组个数进行设置
|
|
|
((TermsBuilder) topAgg).size(1000);
|
|
|
}
|
|
|
} else {
|
|
|
String aggName = topRowDimensionCode + "-date_histogram";
|
|
|
topAgg = AggregationBuilders.dateHistogram(aggName).field(topRowDimensionCode);
|
|
|
((DateHistogramBuilder) topAgg).minDocCount(0);
|
|
|
((DateHistogramBuilder) topAgg).format(topRowDimension.getMemberDateFormat());
|
|
|
((DateHistogramBuilder) topAgg).interval(new DateHistogramInterval(topRowDimension.getMemberDateInterval()));
|
|
|
if (StringUtils.isEmpty(topOrderType) || "name".equals(topOrderType)) {
|
|
|
Histogram.Order order = isTopAsc ? Histogram.Order.KEY_ASC : Histogram.Order.KEY_DESC;
|
|
|
((DateHistogramBuilder) topAgg).order(order);
|
|
|
} else {
|
|
|
// TODO 按子聚合结果排序,需拼接子聚合名称
|
|
|
((DateHistogramBuilder) topAgg).order(Histogram.Order.aggregation("", isTopAsc));
|
|
|
}
|
|
|
// TODO 需动态根据条件范围设置,或设置默认值
|
|
|
((DateHistogramBuilder) topAgg).extendedBounds("", "");
|
|
|
}
|
|
|
|
|
|
// 遍历同组其他行维度,其聚合按顺序嵌套于同组上层行维度聚合下。
|
|
|
AggregationBuilder secondRowAgg = null; // 第二个行维度聚合
|
|
|
AggregationBuilder lastRowAgg = null; // 最后一个行维度聚合
|
|
|
List<ViewDimension> otherRowDimensionList = groupOtherRowDimensionsMap.get(topRowDimension.getGroupRow());
|
|
|
int otherRowDimensionCount = otherRowDimensionList.size();
|
|
|
for (int i = 0; i < otherRowDimensionCount; i++) {
|
|
|
ViewDimension otherRowDimension = otherRowDimensionList.get(i);
|
|
|
String otherRowDimensionCode = otherRowDimension.getDimensionCode();
|
|
|
|
|
|
|
|
|
ViewDimension nextOtherRowDimension = null;
|
|
|
if (i < otherRowDimensionCount - 1) {
|
|
|
nextOtherRowDimension = otherRowDimensionList.get(i + 1);
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
}
|
|
@ -137,17 +236,18 @@ public class ViewService extends BaseJpaService<View, ViewDao> {
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
* 拼接视图的过滤条件
|
|
|
* 拼接视图/指标的过滤条件
|
|
|
*
|
|
|
* @param boolQueryBuilder
|
|
|
* @param viewFilterModelList
|
|
|
* @param boolQueryBuilder ES布尔查询构建器
|
|
|
* @param filterModelList 视图/指标的过滤条件模型集合
|
|
|
*/
|
|
|
private void joinViewFilters(BoolQueryBuilder boolQueryBuilder, List<ViewQuotaFilterModel> viewFilterModelList) {
|
|
|
if (viewFilterModelList.size() != 0) {
|
|
|
for (ViewQuotaFilterModel filterModel : viewFilterModelList) {
|
|
|
private void joinFilters(BoolQueryBuilder boolQueryBuilder, List<ViewQuotaFilterModel> filterModelList) {
|
|
|
if (filterModelList.size() != 0) {
|
|
|
for (ViewQuotaFilterModel filterModel : filterModelList) {
|
|
|
String field = filterModel.getDimensionCode();
|
|
|
String value = filterModel.getFilterValue();
|
|
|
// 属于与否,TODO 需要判断字段类型,转换 value 的类型。
|
|
|
Object value = filterModel.getFilterValue();
|
|
|
|
|
|
// 属于与否
|
|
|
if ("belong".equals(filterModel.getCompareType())) {
|
|
|
QueryBuilder queryBuilder = QueryBuilders.termQuery(field, value);
|
|
|
joinBoolQuery(boolQueryBuilder, queryBuilder, filterModel.getRelationType());
|
|
@ -171,7 +271,7 @@ public class ViewService extends BaseJpaService<View, ViewDao> {
|
|
|
QueryBuilder queryBuilder = QueryBuilders.matchQuery(field, value);
|
|
|
joinBoolQuery(boolQueryBuilder, queryBuilder, "not");
|
|
|
}
|
|
|
// 范围,TODO 需要判断字段类型,转换 value 的类型。
|
|
|
// 范围
|
|
|
else if ("gt".equals(filterModel.getCompareType())) {
|
|
|
QueryBuilder queryBuilder = QueryBuilders.rangeQuery(field).gt(value);
|
|
|
joinBoolQuery(boolQueryBuilder, queryBuilder, filterModel.getRelationType());
|
|
@ -189,6 +289,13 @@ public class ViewService extends BaseJpaService<View, ViewDao> {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
* 组装且/或/否查询
|
|
|
*
|
|
|
* @param boolQueryBuilder ES布尔查询构建器
|
|
|
* @param queryBuilder ES查询构建器
|
|
|
* @param relationType 且/或/非关系类型
|
|
|
*/
|
|
|
private void joinBoolQuery(BoolQueryBuilder boolQueryBuilder, QueryBuilder queryBuilder, String relationType) {
|
|
|
if ("and".equals(relationType)) {
|
|
|
boolQueryBuilder.must(queryBuilder);
|
|
@ -199,4 +306,79 @@ public class ViewService extends BaseJpaService<View, ViewDao> {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
* 汇总视图为数值展示类型的各种度量聚合
|
|
|
*
|
|
|
* @param searchRequestBuilder ES请求构建器
|
|
|
* @param viewQuotaList 视图的指标
|
|
|
* @throws IOException
|
|
|
*/
|
|
|
private void gatherMetricsAgg(SearchRequestBuilder searchRequestBuilder, List<ViewQuota> viewQuotaList) throws IOException {
|
|
|
for (ViewQuota quota : viewQuotaList) {
|
|
|
if ("basic".equals(quota.getFormulaMode())) {
|
|
|
if ("sum".equals(quota.getBasicFormulaType())) {
|
|
|
// 求和
|
|
|
String aggName = "sum-result";
|
|
|
SumBuilder sumAgg = AggregationBuilders.sum(aggName).field(quota.getDimensionCode());
|
|
|
this.joinMetricAgg(searchRequestBuilder, sumAgg, quota);
|
|
|
} else if ("count".equals(quota.getBasicFormulaType())) {
|
|
|
// 计数
|
|
|
String aggName = "count-result";
|
|
|
ValueCountBuilder countAgg = AggregationBuilders.count(aggName).field(quota.getDimensionCode());
|
|
|
this.joinMetricAgg(searchRequestBuilder, countAgg, quota);
|
|
|
} else if ("avg".equals(quota.getBasicFormulaType())) {
|
|
|
// 均值
|
|
|
String aggName = "avg-result";
|
|
|
AvgBuilder avgAgg = AggregationBuilders.avg(aggName).field(quota.getDimensionCode());
|
|
|
this.joinMetricAgg(searchRequestBuilder, avgAgg, quota);
|
|
|
} else if ("max".equals(quota.getBasicFormulaType())) {
|
|
|
// 最大值
|
|
|
String aggName = "max-result";
|
|
|
MaxBuilder maxAgg = AggregationBuilders.max(aggName).field(quota.getDimensionCode());
|
|
|
this.joinMetricAgg(searchRequestBuilder, maxAgg, quota);
|
|
|
} else if ("min".equals(quota.getBasicFormulaType())) {
|
|
|
// 最小值
|
|
|
String aggName = "min-result";
|
|
|
MinBuilder minAgg = AggregationBuilders.min(aggName).field(quota.getDimensionCode());
|
|
|
this.joinMetricAgg(searchRequestBuilder, minAgg, quota);
|
|
|
} else if ("cardinality".equals(quota.getBasicFormulaType())) {
|
|
|
// 去重计数
|
|
|
String aggName = "cardinality-result";
|
|
|
CardinalityBuilder cardinalityAgg = AggregationBuilders.cardinality(aggName).field(quota.getDimensionCode());
|
|
|
this.joinMetricAgg(searchRequestBuilder, cardinalityAgg, quota);
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
* 组装视图为数值展示类型的聚合
|
|
|
*
|
|
|
* @param searchRequestBuilder ES请求构建器
|
|
|
* @param metricAgg ES度量聚合构建器
|
|
|
* @param quota 视图指标
|
|
|
* @throws IOException
|
|
|
*/
|
|
|
private void joinMetricAgg(SearchRequestBuilder searchRequestBuilder, MetricsAggregationBuilder metricAgg, ViewQuota quota) throws IOException {
|
|
|
List<ViewQuotaFilter> quotaFilterList = quota.getQuotaFilterList();
|
|
|
FilterAggregationBuilder filterAgg = null;
|
|
|
if (quotaFilterList != null && quotaFilterList.size() > 0) {
|
|
|
String quotaFilterStr = objectMapper.writeValueAsString(quotaFilterList);
|
|
|
List<ViewQuotaFilterModel> quotaFilterModelList = objectMapper.readValue(quotaFilterStr, new TypeReference<List<ViewQuotaFilterModel>>() {
|
|
|
});
|
|
|
BoolQueryBuilder quotaBoolQueryBuilder = QueryBuilders.boolQuery();
|
|
|
joinFilters(quotaBoolQueryBuilder, quotaFilterModelList);
|
|
|
|
|
|
String aggName = quota.getDimensionCode() + "-filter";
|
|
|
filterAgg = AggregationBuilders.filter(aggName).filter(quotaBoolQueryBuilder);
|
|
|
filterAgg.subAggregation(metricAgg);
|
|
|
}
|
|
|
|
|
|
if (filterAgg == null) {
|
|
|
searchRequestBuilder.addAggregation(metricAgg);
|
|
|
} else {
|
|
|
searchRequestBuilder.addAggregation(filterAgg);
|
|
|
}
|
|
|
}
|
|
|
|
|
|
}
|