svr-pack-analyzer/src/main/java/com/yihu/ehr/analyze/service/dataQuality/DataQualityHomeService.java

package com.yihu.ehr.analyze.service.dataQuality;


import com.yihu.ehr.analyze.dao.DqPaltformReceiveWarningDao;
import com.yihu.ehr.analyze.service.pack.PackQcReportService;
import com.yihu.ehr.elasticsearch.ElasticSearchPool;
import com.yihu.ehr.elasticsearch.ElasticSearchUtil;
import com.yihu.ehr.entity.quality.DqPaltformReceiveWarning;
import com.yihu.ehr.profile.qualilty.DqDataType;
import com.yihu.ehr.query.BaseJpaService;
import com.yihu.ehr.redis.client.RedisClient;
import com.yihu.ehr.util.datetime.DateUtil;
import com.yihu.ehr.util.rest.Envelop;
import org.apache.commons.lang.StringUtils;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.RangeQueryBuilder;
import org.elasticsearch.search.aggregations.AggregationBuilder;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.metrics.cardinality.CardinalityBuilder;
import org.elasticsearch.search.aggregations.metrics.cardinality.InternalCardinality;
import org.elasticsearch.search.aggregations.metrics.sum.InternalSum;
import org.elasticsearch.search.aggregations.metrics.sum.SumBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.ApplicationContext;
import org.springframework.data.domain.Page;
import org.springframework.stereotype.Service;

import javax.annotation.PostConstruct;
import java.io.IOException;
import java.sql.ResultSet;
import java.text.DecimalFormat;
import java.util.*;

/**
 *  质控管理- 首页逻辑类
 * @author HZY
 * @created 2018/8/17 11:24
 */
@Service
public class DataQualityHomeService extends BaseJpaService {

    private final static Logger logger = LoggerFactory.getLogger(DataQualityHomeService.class);
    @Autowired
    private ElasticSearchUtil elasticSearchUtil;
    @Autowired
    private DqPaltformReceiveWarningDao dqPaltformReceiveWarningDao;
    @Autowired
    private PackQcReportService packQcReportService;
    @Value("${quality.orgCode}")
    private String defaultOrgCode;
    @Value("${quality.cloud}")
    private String cloud;
    @Value("${quality.cloudName}")
    private String cloudName;
    @Autowired
    private RedisClient redisClient;
    @Autowired
    private ApplicationContext context;
    @Autowired
    private ElasticSearchPool elasticSearchPool;

    private Map<DqDataType, DataQualityBaseService> dqBaseServiceMap;
    @PostConstruct
    private void init() {
        dqBaseServiceMap = new HashMap<>();
        dqBaseServiceMap.put(DqDataType.complete, context.getBean(DataCompleteService.class));
        dqBaseServiceMap.put(DqDataType.imTime, context.getBean(DataInTimeService.class));
        dqBaseServiceMap.put(DqDataType.correct, context.getBean(DataCorrectService.class));
    }

    public DataQualityBaseService getInstance(DqDataType type){
        DataQualityBaseService dataQualityBaseService;
        switch (type) {
            case complete:
                dataQualityBaseService = dqBaseServiceMap.get(DqDataType.complete);
                break;
            case imTime:
                dataQualityBaseService = dqBaseServiceMap.get(DqDataType.imTime);
                break;
            case correct:
                dataQualityBaseService = dqBaseServiceMap.get(DqDataType.correct);
                break;
            default:
                throw new RuntimeException("Failed to identify dataQualityBaseService type");
        }
        return dataQualityBaseService;
    }

    /**
     * 平台就诊人数 去重复(完整人数)  档案完整性
     * @param dateField 时间区间查询字段
     * @param start
     * @param end
     * @param orgCode
     * @return
     */
    public void getPatientCount(String dateField,String start, String end, String orgCode, Map<String, Object> map) throws Exception {
        try {
            long starttime = System.currentTimeMillis();
            String sql0 = "";
            String sql1 = "";
            String sql2 = "";
            String sql3 = "";
            if (StringUtils.isNotEmpty(orgCode)) {
                sql0 = "SELECT COUNT(DISTINCT event_no) FROM json_archives WHERE event_type=2 AND pack_type=1 AND org_code='" + orgCode + "' AND " + dateField + " BETWEEN" +
                        " '" + start + " 00:00:00' AND '" + end + " 23:59:59'";

                sql1 = "SELECT COUNT(DISTINCT event_no) FROM json_archives WHERE event_type=1 AND pack_type=1 AND org_code='" + orgCode + "' AND " + dateField + " BETWEEN" +
                        " '" + start + " 00:00:00' AND '" + end + " 23:59:59'";

                sql2 = "SELECT COUNT(DISTINCT event_no) FROM json_archives WHERE event_type=0 AND pack_type=1 AND org_code='" + orgCode + "' AND " + dateField + " BETWEEN " +
                        "'" + start + " 00:00:00' AND '" + end + " 23:59:59'";

                sql3 = "SELECT COUNT(DISTINCT event_no) FROM json_archives WHERE pack_type=1 AND org_code='" + orgCode + "' AND " + dateField + " BETWEEN " +
                        "'" + start + " 00:00:00' AND '" + end + " 23:59:59'";
            } else {
                sql0 = "SELECT COUNT(DISTINCT event_no) FROM json_archives WHERE event_type=2 AND pack_type=1 AND " + dateField +
                        " BETWEEN '" + start + " 00:00:00' AND '" + end + " 23:59:59'";

                sql1 = "SELECT COUNT(DISTINCT event_no) FROM json_archives WHERE event_type=1 AND pack_type=1 AND " + dateField +
                        " BETWEEN '" + start + " 00:00:00' AND '" + end + " 23:59:59'";

                sql2 = "SELECT COUNT(DISTINCT event_no) FROM json_archives WHERE event_type=0 AND pack_type=1 AND " + dateField +
                        " BETWEEN '" + start + " 00:00:00' AND '" + end + " 23:59:59'";

                sql3 = "SELECT COUNT(DISTINCT event_no) FROM json_archives WHERE pack_type=1 AND " + dateField +
                        " BETWEEN '" + start + " 00:00:00' AND '" + end + " 23:59:59'";
            }
            ResultSet resultSet0 = elasticSearchUtil.findBySql(sql0);
            ResultSet resultSet1 = elasticSearchUtil.findBySql(sql1);
            ResultSet resultSet2 = elasticSearchUtil.findBySql(sql2);
            ResultSet resultSet3 = elasticSearchUtil.findBySql(sql3);
            resultSet0.next();
            resultSet1.next();
            resultSet2.next();
            resultSet3.next();
            map.put("peIntegrity", new Double(resultSet0.getObject("COUNT(DISTINCT event_no)").toString()).intValue());//体检
            map.put("hospitalIntegrity", new Double(resultSet1.getObject("COUNT(DISTINCT event_no)").toString()).intValue());//住院
            map.put("outpatientIntegrity", new Double(resultSet2.getObject("COUNT(DISTINCT event_no)").toString()).intValue());//门诊
            map.put("visitIntegrity", new Double(resultSet3.getObject("COUNT(DISTINCT event_no)").toString()).intValue());//就诊
            logger.info("平台就诊人数 去重复：" + (System.currentTimeMillis() - starttime) + "ms");
        } catch (Exception e) {
            if (!"Error".equals(e.getMessage())) {
                e.printStackTrace();
            }
        }
    }

    /**
     * 百分比计算
     *
     * @param molecular   分子
     * @param denominator 分母
     * @return
     */
    public String calRate(double molecular, double denominator) {
        if (molecular == 0) {
            return "0.00%";
        } else if (denominator == 0) {
            return "--";
        }
        DecimalFormat decimalFormat = new DecimalFormat("0.00%");
        return decimalFormat.format(molecular / denominator);
    }


    /**
     * 质控情况 -  总计 （及时率，完整率，准确率）
     *
     * @param start 就诊时间- 起始
     * @param end   就诊时间- 截止
     *              【目前取的是平台全部机构，若做多区域平台的话，需要添加区域参数】
     */
    public Map<String, Object> getQuailyDetail(String start, String end) throws Exception {
        Map<String, Object> totalMap = new HashMap<>();
        List<Map<String, Object>> archiveMapList = new ArrayList<>();
        String dateStr = DateUtil.toString(new Date());
        if (StringUtils.isBlank(start)) {
            start = dateStr;
        }
        if (StringUtils.isBlank(end)) {
            end = dateStr;
        }

        int totalHospitalNum = 0;//医院总就诊数
        double hospitalOutpatientNum = 0;//总门诊数
        double hospitalExamNum = 0;//总体检数
        double hospitalInpatientNum = 0;//总住院数
        double totalInTime = 0;//总及时数
        double totalVisitNum = 0;//总完整数（平台总就诊数）
        double totalCorrect = 0;//总准确数

        //1. 获取医院档案量;
        Envelop envelop = packQcReportService.dailyReport("event_date", start + "T00:00:00Z", end + "T23:59:59Z", null);
        Map<String, Object> hospitalDataMap = (Map<String, Object>) envelop.getDetailModelList().get(0);
        //医院总数据量
        totalHospitalNum = (int) hospitalDataMap.get("total");
        hospitalOutpatientNum = (int) hospitalDataMap.get("oupatient");
        hospitalInpatientNum = (int) hospitalDataMap.get("inpatient");
        hospitalExamNum = (int) hospitalDataMap.get("physical");
        Map<String, Object> dataMap = new HashMap<>();
        //2. 平台就诊完整数
        getPatientCount("event_date",start, end, null, dataMap);
        totalVisitNum = Double.valueOf(dataMap.get("visitIntegrity").toString());//就诊完整数
        //3. 及时数
        totalInTime = getInTimeNum("event_date" ,start, end);
        // 3. 去重准确数【】
        totalCorrect = getErrorDataSetData("event_date",start, end, null);
        //4. 数据集总量
        double dataSetsMun = getDataSetsMap(start, end, null);

        totalMap.put("orgCode", cloud);//机构code
        totalMap.put("orgName", cloudName);//机构名称
        totalMap.put("totalInTime", totalInTime);//及时数
        totalMap.put("totalComplete", totalVisitNum);//总就诊完整数
        totalMap.put("totalCorrect", totalCorrect);//准确数
        totalMap.put("totalHospital", totalHospitalNum);//医院总就诊数
        totalMap.put("totalDataSet", dataSetsMun);//数据集总行数
        //计算及时率及完整率,准确性
        totalMap.put("inTimeRate", calRate(totalInTime, totalHospitalNum));//及时率
        totalMap.put("completeRate", calRate(totalVisitNum, totalHospitalNum));//完整率
        totalMap.put("correctRate", calRate(totalCorrect, dataSetsMun));//数据集准确率

        Map<String, Object> outPatientMap = genVisitMap("outPatient", hospitalOutpatientNum, totalHospitalNum);
        Map<String, Object> inPatientMap = genVisitMap("inPatient", hospitalInpatientNum, totalHospitalNum);
        Map<String, Object> examPatientMap = genVisitMap("exam", hospitalExamNum, totalHospitalNum);
        archiveMapList.add(outPatientMap);
        archiveMapList.add(inPatientMap);
        archiveMapList.add(examPatientMap);
        //档案包采集情况
        totalMap.put("rate", archiveMapList);
        return totalMap;
    }


    /**
     * 质控分析 -  完整性统计
     *
     * @param start 就诊时间- 起始
     * @param end   就诊时间- 截止
     *              【目前取的是平台全部机构，若做多区域平台的话，需要添加区域参数】
     */
    public Map<String, Object> getCompleteStatistics(String start, String end) throws Exception {
        Map<String, Object> totalMap = new HashMap<>();
        String dateStr = DateUtil.toString(new Date());
        if (StringUtils.isBlank(start)) {
            start = dateStr;
        }
        if (StringUtils.isBlank(end)) {
            end = dateStr;
        }

        int totalHospitalNum = 0;//医院总就诊数
        double totalVisitNum = 0;//总完整数（平台总就诊数）

        //1. 获取医院档案量;
        Envelop envelop = packQcReportService.dailyReport("event_date", start + "T00:00:00Z", end + "T23:59:59Z", null);
        Map<String, Object> hospitalDataMap = (Map<String, Object>) envelop.getDetailModelList().get(0);
        //医院总数据量
        totalHospitalNum = (int) hospitalDataMap.get("total");
        Map<String, Object> dataMap = new HashMap<>();
        //2. 平台就诊完整数
        getPatientCount("event_date",start, end, null, dataMap);
        totalVisitNum = Double.valueOf(dataMap.get("visitIntegrity").toString());
        totalMap.put("totalComplete", totalVisitNum);//总就诊完整数
        totalMap.put("totalHospital", totalHospitalNum);//医院总就诊数
        //计算完整率,
        totalMap.put("completeRate", calRate(totalVisitNum, totalHospitalNum));//完整率
        return totalMap;
    }

    /**
     * 质控分析 -  准确率统计
     *
     * @param start 就诊时间- 起始
     * @param end   就诊时间- 截止
     */
    public Map<String, Object> getCorrectStatistics(String start, String end) throws Exception {
        long startlong = System.currentTimeMillis();
        Map<String, Object> totalMap = new HashMap<>();
        String dateStr = DateUtil.toString(new Date());
        if (StringUtils.isBlank(start)) {
            start = dateStr;
        }
        if (StringUtils.isBlank(end)) {
            end = dateStr;
        }

        double totalCorrect = 0;//总准确数
        long starttime1 = System.currentTimeMillis();
        //1. 去重准确数
        totalCorrect = getErrorDataSetData("event_date",start, end, null);
        long endTime1 = System.currentTimeMillis();
        logger.info("去重准确数耗时："+ (endTime1-starttime1));
        //2. 数据集总量
        double dataSetsMun = getDataSetsMap(start, end, null);
        long endTime2= System.currentTimeMillis();
        logger.info("总准确数据集数耗时："+ (endTime2-endTime1));
        totalMap.put("totalCorrect", totalCorrect);//准确数
        totalMap.put("totalDataSet", dataSetsMun);//数据集总行数
        //3  计算准确性
        totalMap.put("correctRate", calRate(totalCorrect, dataSetsMun));//数据集准确率
        long endlong = System.currentTimeMillis();
        logger.info("总耗时："+ (endlong-startlong));
        return totalMap;
    }

    /**
     * 质控分析 -  及时率统计
     *
     * @param start 就诊时间- 起始
     * @param end   就诊时间- 截止
     */
    public Map<String, Object> getInTimeStatistics(String start, String end) throws Exception {
        Map<String, Object> totalMap = new HashMap<>();
        String dateStr = DateUtil.toString(new Date());
        if (StringUtils.isBlank(start)) {
            start = dateStr;
        }
        if (StringUtils.isBlank(end)) {
            end = dateStr;
        }

        int totalHospitalNum = 0;//医院总就诊数
        double totalInTime = 0;//总及时数
        //1. 获取医院档案量;
        Envelop envelop = packQcReportService.dailyReport("event_date", start + "T00:00:00Z", end + "T23:59:59Z", null);
        Map<String, Object> hospitalDataMap = (Map<String, Object>) envelop.getDetailModelList().get(0);
        //医院总数据量
        totalHospitalNum = (int) hospitalDataMap.get("total");
        Map<String, Object> dataMap = new HashMap<>();
        //2. 及时数
        totalInTime = getInTimeNum("event_date" ,start, end);
        totalMap.put("totalInTime", totalInTime);//及时数
        totalMap.put("totalHospital", totalHospitalNum);//医院总就诊数
        //3 计算及时率
        totalMap.put("inTimeRate", calRate(totalInTime, totalHospitalNum));//及时率
        return totalMap;
    }





    /**
     * 获取质控错误 - 数据集总条数
     * @param dateField  时间区间查询字段
     * @param start
     * @param end
     * @param orgCode
     * @return
     */
    public double getErrorDataSetData(String dateField,String start, String end, String orgCode) {
        long starttime1 = System.currentTimeMillis();
        List< Map<String, Object>> resultList = new ArrayList<>();
        double num1 = 0;
        try {
            String dateStr = DateUtil.toString(new Date());
            if (StringUtils.isBlank(start)) {
                start = dateStr;
            }
            if (StringUtils.isBlank(end)) {
                end = dateStr;
            }

            // 原生es
            TransportClient transportClient = elasticSearchPool.getClient();
            StringBuilder stringBuilder = new StringBuilder();
            stringBuilder.append("event_date>=" + start + " 00:00:00;");
            stringBuilder.append("event_date<=" + end + " 23:59:59;");
            stringBuilder.append("qc_step=1||qc_step=2;");
            if (StringUtils.isNotEmpty(orgCode)) {
                stringBuilder.append("org_code=" + orgCode);
            }

            SearchRequestBuilder builder = transportClient.prepareSearch("json_archives_qc");
            builder.setTypes("qc_metadata_info");
            builder.setSearchType(SearchType.DFS_QUERY_THEN_FETCH);
            builder.setQuery(elasticSearchUtil.getQueryBuilder(stringBuilder.toString()));
            AggregationBuilder terms = AggregationBuilders.terms("dataset").field("dataset").size(200);
            CardinalityBuilder childTerms = AggregationBuilders.cardinality("count").field("pack_id").precisionThreshold(40000);
            terms.subAggregation(childTerms);
            builder.addAggregation(terms);
            builder.setSize(0);
            builder.setFrom(0);
            SearchResponse response = builder.get();
            StringTerms longTerms = response.getAggregations().get("dataset");
            for (Terms.Bucket item : longTerms.getBuckets()) {
                InternalCardinality extendedStats = item.getAggregations().get("count");
//                Map<String, Object> temp = new HashMap<>();
//                temp.put("dataset", item.getKeyAsString());
//                temp.put("count", extendedStats.getValue());
//                resultList.add(temp);
                num1 += extendedStats.getValue();
            }



        } catch (Exception e) {
            e.printStackTrace();
            num1 = 0;
        }
            return num1;
    }


    /**
     * 获取数据集总量
     *
     * @param start
     * @param end
     * @param orgCode
     * @return
     */
    public double getDataSetsMap(String start, String end, String orgCode) throws IOException {
        //  数据集总量
        double totalNum = 0;
        // 默认时间
        String dateStr = DateUtil.toString(new Date());
        if (StringUtils.isBlank(start)) {
            start = dateStr;
        }
        if (StringUtils.isBlank(end)) {
            end = dateStr;
        }
        TransportClient transportClient = elasticSearchPool.getClient();
        //  条件设置
        StringBuilder stringBuilder = new StringBuilder();
        stringBuilder.append("event_date>=" + start + " 00:00:00;");
        stringBuilder.append("event_date<=" + end + " 23:59:59;");
        if (StringUtils.isNotEmpty(orgCode)) {
            stringBuilder.append("org_code=" + orgCode);
        }

        SearchRequestBuilder builder = transportClient.prepareSearch("json_archives_qc");
        builder.setTypes("qc_dataset_detail");
        builder.setSearchType(SearchType.DFS_QUERY_THEN_FETCH);
        builder.setQuery(elasticSearchUtil.getQueryBuilder(stringBuilder.toString()));
        SumBuilder terms = AggregationBuilders.sum("sumCount").field("count");
        builder.addAggregation(terms);
        builder.setSize(0);
        builder.setFrom(0);
        SearchResponse response = builder.get();
        InternalSum longTerms = response.getAggregations().get("sumCount");
        totalNum = longTerms.getValue();
        return totalNum;
    }

    /**
     *  获取所有及时上传数
     * @param dateField  时间区间查询字段
     * @param start
     * @param end
     * @return
     */
    public double getInTimeNum(String dateField,String start, String end) {
        double totalInTime = 0;
        Map<String, Object> resMap = new HashMap<>();
        DataQualityBaseService dataQualityBaseService = getInstance(DqDataType.create(1));
        //获取有数据的医院code列表
        List<String> orgList = dataQualityBaseService.hasDataHospital(dateField, start, end);
        for (String orgCode : orgList) {
            //初始化 及时率预警信息
            DqPaltformReceiveWarning warning = dqPaltformReceiveWarningDao.findByOrgCode(orgCode);
            if (warning == null) {
                warning = dqPaltformReceiveWarningDao.findByOrgCode(defaultOrgCode);
            }
            long starttime = System.currentTimeMillis();
            String sql0 = "";
            List<String> fields = new ArrayList<String>();
            fields.add("count");
            try {
                sql0 = "SELECT  COUNT(DISTINCT event_no) as count FROM json_archives WHERE  pack_type=1 AND org_code='" + orgCode + "' AND " + dateField +
                        " BETWEEN '" + start + " 00:00:00' AND '" + end + " 23:59:59' and delay <=" + warning.getPeInTime() ;

                List<Map<String, Object>> resultSet0 = elasticSearchUtil.findBySql(fields, sql0);
                for (Map<String, Object> orgData : resultSet0) {
                    double newValue = (double) orgData.get("count");
                    totalInTime +=newValue;
                }

                logger.info("平台就诊及时人数 去重复：" + (System.currentTimeMillis() - starttime) + "ms");
            } catch (Exception e) {
                if (!"Error".equals(e.getMessage())) {
                    e.printStackTrace();
                }
            }
        }
        return totalInTime;
    }

    public Map<String, Object> genVisitMap(String typeField, double value, double total) {
        Map<String, Object> map = new HashMap<>();
        map.put("name", typeField);
        map.put("value", value);
        map.put("rate", calRate(value,total));
        return map;
    }


    /**
     *  批量更新es中的区域编码org_area
     *  (通过机构编码org_code 更新org_area）
     */
    public long bulkUpdateOrgArea(String index,String type,String filters){
        long page = 0;
        long count = elasticSearchUtil.count(index, type, filters);
        if (count >10000) {
            page = count/10000 + 1;
        }else {
            page = 1;
        }

        for (int i = 1;i<=page;i++) {
            Page<Map<String, Object>> result = elasticSearchUtil.page(index, type, filters, i, 10000);
            List<Map<String, Object>> updateSourceList = new ArrayList<>();
            result.forEach(item -> {
                Map<String, Object> updateSource = new HashMap<>();
                updateSource.put("_id", item.get("_id"));
                String orgCode = (String) item.get("org_code");
                String orgArea = redisClient.get("organizations:" + orgCode + ":area");
                updateSource.put("org_area", orgArea);
                updateSourceList.add(updateSource);
            });
            if (!updateSourceList.isEmpty()) {
                elasticSearchUtil.bulkUpdate(index, type, updateSourceList);
            }
        }
        return count;

    }


             /* ******************************** 区域层级模块相关 ***********************************/

    /**
     *  获取市区域的下级区域质控情况
     * @param dataType 数据维度  （0: 完整性，1:及时性，2:准确性）
     * @param start
     * @param end
     * @return
     */
    public List<Map<String,Object>> findAreaData(Integer dataType, String start, String end) throws Exception {
        List<Map<String,Object>> list = new ArrayList<>();
        String dateStr = DateUtil.toString(new Date());
        if (StringUtils.isBlank(start)) {
            start = dateStr;
        }
        if (StringUtils.isBlank(end)) {
            end = dateStr;
        }
        DataQualityBaseService dataQualityBaseService = getInstance(DqDataType.create(dataType));
        list = dataQualityBaseService.getAreaDataQuality(0,start,end);
        return list;
    }

    /* ******************************** 机构 层级模块相关 ***********************************/

    /**
     *  获取区县下的机构质控情况
     * @param dataType 数据维度  （0: 完整性，1:及时性，2:准确性）
     * @param areaCode 上区域编码
     * @param start
     * @param end
     * @return
     */
    public List<Map<String,Object>> findOrgData(Integer dataType , String areaCode, String start, String end) throws Exception {
        List<Map<String,Object>> list = new ArrayList<>();
        String dateStr = DateUtil.toString(new Date());
        if (StringUtils.isBlank(start)) {
            start = dateStr;
        }
        if (StringUtils.isBlank(end)) {
            end = dateStr;
        }
        DataQualityBaseService dataQualityBaseService = getInstance(DqDataType.create(dataType));
        list = dataQualityBaseService.getOrgDataQuality(1,areaCode,start,end);
        return list;
    }

    /**
     * 首页错误数据集
     * @param orgCode
     * @param dataType
     * @param start
     * @param end
     * @return
     * @throws Exception
     */
    public List<Map<String,Object>> homeDatasetError(String orgArea, String orgCode, Integer dataType, String start, String end) throws Exception {
        List<String> fileds = new ArrayList<>();
        fileds.add("dataset");
        fileds.add("count");
        fileds.add("version");
        StringBuffer sql = new StringBuffer();
        sql.append("SELECT dataset,COUNT(DISTINCT pack_id) count ,version FROM json_archives_qc/qc_metadata_info ");
        sql.append(" WHERE event_date>='" + start + " 00:00:00' and (qc_step=1 or qc_step=2) and  event_date<='" + end + " 23:59:59' ");

        if(StringUtils.isNotEmpty(orgArea)){
            sql.append(" AND org_area = '"+orgArea+"'");
        }

        if(StringUtils.isNotEmpty(orgCode)){
            sql.append(" AND org_code = '"+orgCode+"'");
        }
        if(dataType == 0){
            sql.append(" AND qc_error_type=1");
        }else{
            sql.append(" AND qc_error_type<>1");
        }
        sql.append(" group by dataset,version");
        List<Map<String,Object>> list = elasticSearchUtil.findBySql(fileds,sql.toString());
        for(Map<String,Object> map:list){
            map.put("dataset_name", redisClient.get("std_data_set_" + map.get("version") + ":" + map.get("dataset") + ":name"));
        }
        DataQualityBaseService dataQualityBaseService = getInstance(DqDataType.create(1));
        dataQualityBaseService.comparator(list,"count",1);//降序排序
        return list;
    }


    /**
     * 首页错误数据元
     * @param dataset
     * @param dataType
     * @param start
     * @param end
     * @return
     * @throws Exception
     */
    public List<Map<String,Object>> homeMetadataError(String orgArea, String orgCode, String dataset ,Integer dataType,String start,String end) throws Exception {
        List<String> fileds = new ArrayList<>();
        fileds.add("dataset");
        fileds.add("metadata");
        fileds.add("count");
        fileds.add("version");
        StringBuffer sql = new StringBuffer();
        sql.append("SELECT dataset, metadata,COUNT(metadata) count ,version FROM json_archives_qc/qc_metadata_info ");
        sql.append(" WHERE event_date>='" + start + " 00:00:00' and (qc_step=1 or qc_step=2) and event_date<='" + end + " 23:59:59'");
        if(StringUtils.isNotEmpty(orgArea)){
            sql.append(" AND org_area = '"+orgArea+"'");
        }

        if(StringUtils.isNotEmpty(orgCode)){
            sql.append(" AND org_code = '"+orgCode+"'");
        }
        if(StringUtils.isNotEmpty(dataset)){
            sql.append(" AND dataset = '"+dataset+"'");
        }
        if(dataType == 0){
            sql.append(" AND qc_error_type=1");
        }else{
            sql.append(" AND qc_error_type<>1");
        }
        sql.append(" group by dataset,metadata,version");
        List<Map<String,Object>> list = elasticSearchUtil.findBySql(fileds,sql.toString());
        for(Map<String,Object> map:list){
            map.put("metadata_name", redisClient.get("std_meta_data_" + map.get("version") + ":" + map.get("dataset")+"."+ map.get("metadata")+ ":name"));
        }
        DataQualityBaseService dataQualityBaseService = getInstance(DqDataType.create(1));
        dataQualityBaseService.comparator(list,"count",1);//降序排序
        return list;
    }

    /**
     * 完整率、准确率、及时率趋势
     * @param flag 10按月 20按日
     * @param dataType 0: 完整性，1:及时性，2:准确性
     * @param month
     * @return
     * @throws Exception
     */
    public Map<String, Object> homeTrend(String flag, int dataType, String month) throws Exception {
        Map<String, Object> map  = new HashMap<>();
        List<Double> data = new ArrayList<>();
        List<String> name = getName(flag, month);
        if(dataType==0){
            Map<String,Object> hospitalDataMap = packQcReportService.getHospitalNum(flag,month);
            Map<String,Object> map1  =getTotalVisitNum(flag, month);
            for(int i=0;i<name.size();i++){
                if(map1.get(name.get(i))!=null&&hospitalDataMap.get(name.get(i))!=null){
                    if(Double.parseDouble(hospitalDataMap.get(name.get(i))+"")!=0){
                        double rate = (long)map1.get(name.get(i))/(double)hospitalDataMap.get(name.get(i))*100;
                        data.add((double) Math.round(rate * 100) / 100);
                    }else{
                        data.add(0.0);
                    }
                }else{
                    data.add(0.0);
                }
            }
        }else if(dataType==1){
            Map<String,Object> hospitalDataMap = packQcReportService.getHospitalNum(flag,month);
            for(int i=0;i<name.size();i++){
                //2. 及时数
                String start="";
                String end="";
                if("10".equals(flag)){
                    start = name.get(i)+"-01";
                    end =  DateUtil.getLastDate(start, "yyyy-MM-dd", "yyyy-MM-dd");
                }else{
                    start=name.get(i);
                    end=name.get(i);
                }
                double totalInTime = getInTimeNum("event_date" ,start, end);
                if(hospitalDataMap.get(name.get(i))!=null&&Double.parseDouble(hospitalDataMap.get(name.get(i))+"")!=0){
                    double rate = totalInTime/(double)hospitalDataMap.get(name.get(i))*100;
                    data.add((double) Math.round(rate * 100) / 100);
                }else{
                    data.add(0.0);
                }
            }
        }else if(dataType==2){
            Map<String,Object> map1 = getErrorDataSetData(flag, month);
            Map<String,Object> map2 = getTotalDataSets(flag, month);
            for(int i=0;i<name.size();i++){
                if(map1.get(name.get(i))!=null&&map2.get(name.get(i))!=null){
                    if(Double.parseDouble(map2.get(name.get(i))+"")!=0){
                        double rate = (double)map1.get(name.get(i))/(double)map2.get(name.get(i))*100;
                        data.add((double) Math.round(rate * 100) / 100);
                    }else{
                        data.add(0.0);
                    }
                }else{
                    data.add(0.0);
                }
            }
        }
        map.put("data", data);
        map.put("name", name);
        return map;
    }

    private List<String> getName(String flag ,String month) throws Exception{
        List<String> name = new ArrayList<>();
        if("10".equals(flag)){//按月
            for(int i=6;i>0;i--){
                String start =  DateUtil.formatDate(DateUtil.addMonth(1-i, DateUtil.formatCharDateYMD(month+"-01")), "yyyy-MM");
                name.add(start);
            }
        }else{//按日
            //当前月份显示最近30天，否则显示当月的数据
            if(month.equals(DateUtil.toString(new Date(), "yyyy-MM"))){
                for(int i=0;i<30;i++){
                    String date = DateUtil.toString(DateUtil.addDate(0-i,new Date()));
                    name.add(0,date);
                }
            }else{
                String end =  DateUtil.getLastDate (month+"-01", "yyyy-MM-dd", "yyyy-MM-dd");
                for(int i=0;i<32;i++){
                    String date = DateUtil.toString(DateUtil.addDate(i,DateUtil.strToDate(month+"-01")));
                    name.add(date);
                    if(date.equals(end)){
                        break;
                    }
                }
            }
        }
        return name;
    }

    public Map<String, Object> getHealthyArchiveTrend(String flag,String month) throws Exception {
        Map<String, Object> resMap  = new HashMap<>();
        List<String> date = new ArrayList<>();
        List<Long> receiveCount = new ArrayList<>();
        List<Long> storageCount = new ArrayList<>();
       //采集量统计
        Map<String, Object> receive = getHealthyTrend(flag, "receive_date", month);
        //入库量统计
        Map<String, Object> storage  = getHealthyTrend(flag, "finish_date", month);
        //前端趋势图 格式生成
        for (String key : receive.keySet()) {
            Long receiveValue = receive.get(key) == null ? 0 : (Long) receive.get(key);
            Long storageValue = storage.get(key) == null ? 0 : (Long) storage.get(key);
            receiveCount.add(receiveValue);
            storageCount.add(storageValue);
            date.add(key);
        }
        resMap.put("name",date);
        resMap.put("receiveData",receiveCount);
        resMap.put("storageData",storageCount);
        return resMap;
    }

    /**
     * 健康档案建档数
     * @param flag 10按月 20按日
     * @param dateField 分组时间字段  receive_date:采集维度，finish_date：入库维度
     * @param month
     * @return
     * @throws Exception
     */
    public Map<String, Object> getHealthyTrend(String flag, String dateField, String month) throws Exception {
        TreeMap<String, Object> map  = new TreeMap<>();
        //先设置值,防止某几个月份没值,前端直接不展示
        for(int i=6;i>0;i--){
            String data =  DateUtil.formatDate(DateUtil.addMonth(-i, DateUtil.formatCharDateYMD(month+"-01")), "yyyy-MM");
            map.put(data,0L);
        }
        TransportClient transportClient = elasticSearchPool.getClient();
        String start =  "";
        String end =  "";
        AggregationBuilder aggregation =null;
        if("10".equals(flag)){//按月
             start =  DateUtil.formatDate(DateUtil.addMonth(-6, DateUtil.formatCharDateYMD(month+"-01")), "yyyy-MM") + "-01";
             end =  DateUtil.getLastDate (month + "-01", "yyyy-MM-dd", "yyyy-MM-dd");
             aggregation = AggregationBuilders.dateHistogram("agg").field("receive_date")
                    .interval(DateHistogramInterval.MONTH).format("yyyy-MM");

        }else{//按日
            //当前月份显示最近30天，否则显示当月的数据
            if(month.equals(DateUtil.toString(new Date(), "yyyy-MM"))){
                 start = DateUtil.toString(DateUtil.addDate(-30,new Date()));
                end =  DateUtil.getLastDate (month+"-01", "yyyy-MM-dd", "yyyy-MM-dd");
            }else{
                start = month+"-01";
                end =  DateUtil.getLastDate (month+"-01", "yyyy-MM-dd", "yyyy-MM-dd");
            }
            aggregation = AggregationBuilders.dateHistogram("agg").field(dateField)
                    .interval(DateHistogramInterval.DAY).format("yyyy-MM-dd");
        }

        // 声明where 条件
        BoolQueryBuilder qbs = QueryBuilders.boolQuery();
        RangeQueryBuilder startRange = QueryBuilders.rangeQuery(dateField);
        startRange.gte(start + " 00:00:00");
        qbs.must(startRange);

        RangeQueryBuilder endRange = QueryBuilders.rangeQuery(dateField);
        endRange.lt(end+" 23:59:59");
        qbs.must(endRange);

        SearchRequestBuilder requestBuilder = transportClient.prepareSearch("json_archives")
                .setTypes("info");
        requestBuilder.setQuery(qbs);
        requestBuilder.addAggregation(aggregation);
        requestBuilder.addSort(dateField, SortOrder.DESC);
        SearchResponse response = requestBuilder.execute().actionGet();
        Histogram agg = response.getAggregations().get("agg");

// For each entry
        for (Histogram.Bucket entry : agg.getBuckets()) {
            String key = entry.getKey().toString();
            String keyAsString = entry.getKeyAsString();
            long docCount = entry.getDocCount();
            map.put(keyAsString,docCount);
            logger.info("key [{" + keyAsString + "}]");
            logger.info("date [{" + key + "}]");
            logger.info("doc_count [{" + docCount + "}]");
        }
        return map;
    }


    /**
     * 平台人数 去重复
     * @param flag 10按月 20按日
     * @param month
     * @return
     * @throws Exception
     */
    public Map<String, Object> getTotalVisitNum(String flag, String month) throws Exception {
        TreeMap<String, Object> map  = new TreeMap<>();
        TransportClient transportClient = elasticSearchPool.getClient();
        String start =  "";
        String end =  "";
        AggregationBuilder aggregation =null;
        if("10".equals(flag)){//按月
            start =  DateUtil.formatDate(DateUtil.addMonth(-6, DateUtil.formatCharDateYMD(month+"-01")), "yyyy-MM") + "-01";
            end =  DateUtil.getLastDate (month + "-01", "yyyy-MM-dd", "yyyy-MM-dd");
            aggregation = AggregationBuilders.dateHistogram("agg").field("event_date")
                    .interval(DateHistogramInterval.MONTH).format("yyyy-MM");

        }else{//按日
            //当前月份显示最近30天，否则显示当月的数据
            if(month.equals(DateUtil.toString(new Date(), "yyyy-MM"))){
                start = DateUtil.toString(DateUtil.addDate(-30,new Date()));
                end =  DateUtil.getLastDate (month+"-01", "yyyy-MM-dd", "yyyy-MM-dd");
            }else{
                start = month+"-01";
                end =  DateUtil.getLastDate (month+"-01", "yyyy-MM-dd", "yyyy-MM-dd");
            }
            aggregation = AggregationBuilders.dateHistogram("agg").field("event_date")
                    .interval(DateHistogramInterval.DAY).format("yyyy-MM-dd");
        }

        StringBuilder stringBuilder = new StringBuilder();
        stringBuilder.append("pack_type=1;");
        stringBuilder.append("event_date>=" + start + " 00:00:00;");
        stringBuilder.append("event_date<" + end + " 23:59:59;");
        CardinalityBuilder childTerms = AggregationBuilders.cardinality("count").field("event_no").precisionThreshold(40000);

        SearchRequestBuilder requestBuilder = transportClient.prepareSearch("json_archives")
                .setTypes("info");
        requestBuilder.setQuery(elasticSearchUtil.getQueryBuilder(stringBuilder.toString()));
        requestBuilder.addAggregation(aggregation.subAggregation(childTerms));
        requestBuilder.addSort("event_date", SortOrder.DESC);
        SearchResponse response = requestBuilder.execute().actionGet();
        Histogram agg = response.getAggregations().get("agg");

        for (Histogram.Bucket entry : agg.getBuckets()) {
            String keyAsString = entry.getKeyAsString();
            long docCount = entry.getDocCount();
            InternalCardinality extendedStats = entry.getAggregations().get("count");
            map.put(keyAsString,extendedStats.getValue());
        }
        return map;
    }


    /**
     * 获取数据集总量
     *
     * @return
     */
    public Map<String, Object> getTotalDataSets(String flag, String month) throws Exception {
        TreeMap<String, Object> map  = new TreeMap<>();
        TransportClient transportClient = elasticSearchPool.getClient();
        String start =  "";
        String end =  "";
        AggregationBuilder aggregation =null;
        if("10".equals(flag)){//按月
            start =  DateUtil.formatDate(DateUtil.addMonth(-6, DateUtil.formatCharDateYMD(month+"-01")), "yyyy-MM") + "-01";
            end =  DateUtil.getLastDate (month + "-01", "yyyy-MM-dd", "yyyy-MM-dd");
            aggregation = AggregationBuilders.dateHistogram("agg").field("event_date")
                    .interval(DateHistogramInterval.MONTH).format("yyyy-MM");

        }else{//按日
            //当前月份显示最近30天，否则显示当月的数据
            if(month.equals(DateUtil.toString(new Date(), "yyyy-MM"))){
                start = DateUtil.toString(DateUtil.addDate(-30,new Date()));
                end =  DateUtil.getLastDate (month+"-01", "yyyy-MM-dd", "yyyy-MM-dd");
            }else{
                start = month+"-01";
                end =  DateUtil.getLastDate (month+"-01", "yyyy-MM-dd", "yyyy-MM-dd");
            }
            aggregation = AggregationBuilders.dateHistogram("agg").field("event_date")
                    .interval(DateHistogramInterval.DAY).format("yyyy-MM-dd");
        }

        StringBuilder stringBuilder = new StringBuilder();
        stringBuilder.append("event_date>=" + start + " 00:00:00;");
        stringBuilder.append("event_date<=" + end + " 23:59:59;");
        SumBuilder terms = AggregationBuilders.sum("sumCount").field("count");

        SearchRequestBuilder requestBuilder = transportClient.prepareSearch("json_archives_qc")
                .setTypes("qc_dataset_detail");
        requestBuilder.setQuery(elasticSearchUtil.getQueryBuilder(stringBuilder.toString()));
        requestBuilder.addAggregation(aggregation.subAggregation(terms));
        requestBuilder.addSort("event_date", SortOrder.DESC);
        SearchResponse response = requestBuilder.execute().actionGet();
        Histogram agg = response.getAggregations().get("agg");

        for (Histogram.Bucket entry : agg.getBuckets()) {
            String keyAsString = entry.getKeyAsString();
            InternalSum longTerms = entry.getAggregations().get("sumCount");
            map.put(keyAsString,longTerms.getValue());
        }
        return map;
    }


    /**
     * 获取质控错误 - 数据集总条数
     * @return
     */
    public Map<String,Object> getErrorDataSetData(String flag, String month)throws Exception {
        TreeMap<String, Object> map  = new TreeMap<>();
        TransportClient transportClient = elasticSearchPool.getClient();
        String start =  "";
        String end =  "";
        AggregationBuilder aggregation =null;
        if("10".equals(flag)){//按月
            start =  DateUtil.formatDate(DateUtil.addMonth(-6, DateUtil.formatCharDateYMD(month+"-01")), "yyyy-MM") + "-01";
            end =  DateUtil.getLastDate (month + "-01", "yyyy-MM-dd", "yyyy-MM-dd");
            aggregation = AggregationBuilders.dateHistogram("agg").field("event_date")
                    .interval(DateHistogramInterval.MONTH).format("yyyy-MM");

        }else{//按日
            //当前月份显示最近30天，否则显示当月的数据
            if(month.equals(DateUtil.toString(new Date(), "yyyy-MM"))){
                start = DateUtil.toString(DateUtil.addDate(-30,new Date()));
                end =  DateUtil.getLastDate (month+"-01", "yyyy-MM-dd", "yyyy-MM-dd");
            }else{
                start = month+"-01";
                end =  DateUtil.getLastDate (month+"-01", "yyyy-MM-dd", "yyyy-MM-dd");
            }
            aggregation = AggregationBuilders.dateHistogram("agg").field("event_date")
                    .interval(DateHistogramInterval.DAY).format("yyyy-MM-dd");
        }

        StringBuilder stringBuilder = new StringBuilder();
        stringBuilder.append("event_date>=" + start + " 00:00:00;");
        stringBuilder.append("event_date<=" + end + " 23:59:59;");
        stringBuilder.append("qc_step=1||qc_step=2;");
        AggregationBuilder terms = AggregationBuilders.terms("dataset").field("dataset").size(200);
        CardinalityBuilder childTerms = AggregationBuilders.cardinality("count").field("pack_id").precisionThreshold(40000);
        terms.subAggregation(childTerms);

        SearchRequestBuilder requestBuilder = transportClient.prepareSearch("json_archives_qc")
                .setTypes("qc_metadata_info");
        requestBuilder.setQuery(elasticSearchUtil.getQueryBuilder(stringBuilder.toString()));
        requestBuilder.addAggregation(aggregation.subAggregation(terms));
        requestBuilder.addSort("event_date", SortOrder.DESC);
        SearchResponse response = requestBuilder.execute().actionGet();
        Histogram agg = response.getAggregations().get("agg");

        for (Histogram.Bucket entry : agg.getBuckets()) {
            String keyAsString = entry.getKeyAsString();
            StringTerms longTerms = entry.getAggregations().get("dataset");
            double num = 0;
            for (Terms.Bucket item : longTerms.getBuckets()) {
                InternalCardinality extendedStats = item.getAggregations().get("count");
               num+=extendedStats.getValue();
            }
            map.put(keyAsString,num);
        }
        return map;
    }
}