质控准确率统计 sql语法改为es原生语法

pom.xml

@@ -5,7 +5,7 @@
    <modelVersion>4.0.0</modelVersion>
    <parent>
        <groupId>com.yihu.ehr</groupId>
        <artifactId>ehr-ms-parent</artifactId>
        <artifactId>ehr-cloud-parent</artifactId>
        <version>1.13.0</version>
    </parent>
@@ -28,17 +28,15 @@
        <dependency>
            <groupId>org.springframework.cloud</groupId>
            <artifactId>spring-cloud-starter-config</artifactId>
            <scope>${dependency.scope}</scope>
            
        </dependency>
        <dependency>
            <groupId>org.springframework.cloud</groupId>
            <artifactId>spring-cloud-starter-eureka</artifactId>
            <scope>${dependency.scope}</scope>
        </dependency>
        <dependency>
            <groupId>org.springframework.cloud</groupId>
            <artifactId>spring-cloud-starter-feign</artifactId>
            <scope>${dependency.scope}</scope>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
@@ -48,78 +46,77 @@
        <dependency>
            <groupId>com.timgroup</groupId>
            <artifactId>java-statsd-client</artifactId>
            <scope>${dependency.scope}</scope>
        </dependency>
        <dependency>
            <groupId>javax.servlet</groupId>
            <artifactId>javax.servlet-api</artifactId>
            <scope>${dependency.scope}</scope>
        </dependency>
        <dependency>
            <groupId>org.quartz-scheduler</groupId>
            <artifactId>quartz</artifactId>
            <scope>${dependency.scope}</scope>
        </dependency>
        <dependency>
            <groupId>com.yihu.ehr</groupId>
            <artifactId>commons-data-elasticsearch</artifactId>
            <scope>compile</scope>
            <exclusions>
                <exclusion>
                        <groupId>org.elasticsearch</groupId>
                        <artifactId>elasticsearch</artifactId>
                </exclusion>
            </exclusions>
        </dependency>
        <dependency>
            <groupId>org.elasticsearch</groupId>
            <artifactId>elasticsearch</artifactId>
            <version>2.4.6</version>
        </dependency>
        <dependency>
            <groupId>com.yihu.ehr</groupId>
            <artifactId>commons-data-fastdfs</artifactId>
            <scope>compile</scope>
        </dependency>
        <dependency>
            <groupId>com.yihu.ehr</groupId>
            <artifactId>commons-ehr-constants</artifactId>
            <scope>compile</scope>
        </dependency>
        <dependency>
            <groupId>com.yihu.ehr</groupId>
            <artifactId>commons-entity</artifactId>
            <scope>compile</scope>
        </dependency>
        <dependency>
            <groupId>com.yihu.ehr</groupId>
            <artifactId>commons-util</artifactId>
            <scope>compile</scope>
        </dependency>
        <dependency>
            <groupId>com.yihu.ehr</groupId>
            <artifactId>commons-data-mysql</artifactId>
            <scope>compile</scope>
        </dependency>
        <dependency>
            <groupId>com.yihu.ehr</groupId>
            <artifactId>commons-rest-model</artifactId>
            <scope>compile</scope>
        </dependency>
        <dependency>
            <groupId>com.yihu.ehr</groupId>
            <artifactId>commons-data-redis</artifactId>
            <scope>compile</scope>
        </dependency>
        <dependency>
            <groupId>com.yihu.ehr</groupId>
            <artifactId>commons-ui-swagger</artifactId>
            <scope>compile</scope>
        </dependency>
        <dependency>
            <groupId>com.yihu.ehr</groupId>
            <artifactId>commons-web</artifactId>
            <scope>compile</scope>
           
        </dependency>
        <dependency>
            <groupId>com.yihu.ehr</groupId>
            <artifactId>commons-profile-core</artifactId>
            <scope>compile</scope>
           
        </dependency>
        <dependency>
            <groupId>com.yihu.ehr</groupId>
            <artifactId>commons-redis-mq</artifactId>
            <scope>compile</scope>
           
        </dependency>
        <dependency>
            <groupId>com.yihu.ehr</groupId>
@@ -145,5 +142,9 @@
            <groupId>net.sourceforge.jexcelapi</groupId>
            <artifactId>jxl</artifactId>
        </dependency>
        <dependency>
            <groupId>commons-collections</groupId>
            <artifactId>commons-collections</artifactId>
        </dependency>
    </dependencies>
</project>

src/main/java/com/yihu/ehr/analyze/controller/dataQuality/TestDemo.java

@@ -13,6 +13,8 @@ import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.bucket.terms.TermsBuilder;
import org.elasticsearch.search.aggregations.metrics.cardinality.CardinalityBuilder;
import org.elasticsearch.search.aggregations.metrics.cardinality.InternalCardinality;
import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStats;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -144,23 +146,18 @@ public class TestDemo {
            builder.setSearchType(SearchType.DFS_QUERY_THEN_FETCH);
//                builder.setQuery(elasticSearchUtil.getQueryBuilder(filters.toString()));
            AggregationBuilder terms = AggregationBuilders.terms("upload_status").field("upload_status").size(200);
            CardinalityBuilder childTerms = AggregationBuilders.cardinality("COUNT(DISTINCT org_area)").field("distinct org_area").precisionThreshold(40000);
            CardinalityBuilder childTerms = AggregationBuilders.cardinality("COUNT(DISTINCT org_area)").field("org_area").precisionThreshold(40000);
            terms.subAggregation(childTerms);
            builder.addAggregation(terms);
            builder.setSize(0);
            builder.setFrom(0);
            Map<String,Object> sourceMap = new HashMap<>();
            List<String> include = new ArrayList<>();
            include.add("COUNT");
            sourceMap.put("includes",include);
            sourceMap.put("excludes",new ArrayList<>());
            builder.setSource(sourceMap);
            SearchResponse response = builder.get();
            StringTerms longTerms = response.getAggregations().get("upload_status");
            for (Terms.Bucket item : longTerms.getBuckets()) {
                InternalCardinality extendedStats = item.getAggregations().get("COUNT(DISTINCT org_area)");
                Map<String, Object> temp = new HashMap<>();
                temp.put("upload_status", item.getKeyAsString());
                temp.put("count11", item.getDocCount());
                temp.put("count11", extendedStats);
                resultList.add(temp);
            }
            logger.info("平台就诊人数 去重复：" + (System.currentTimeMillis() - starttime) + "ms");

src/main/java/com/yihu/ehr/analyze/service/dataQuality/DataQualityHomeService.java

@@ -14,6 +14,7 @@ import com.yihu.ehr.util.rest.Envelop;
import org.apache.commons.lang.StringUtils;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
@@ -22,6 +23,12 @@ import org.elasticsearch.search.aggregations.AggregationBuilder;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.metrics.cardinality.CardinalityBuilder;
import org.elasticsearch.search.aggregations.metrics.cardinality.InternalCardinality;
import org.elasticsearch.search.aggregations.metrics.sum.InternalSum;
import org.elasticsearch.search.aggregations.metrics.sum.SumBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -213,7 +220,7 @@ public class DataQualityHomeService extends BaseJpaService {
        // 3. 去重准确数【】
        totalCorrect = getErrorDataSetData("receive_date",start, end, null);
        //4. 数据集总量
        int dataSetsMun = getDataSetsMap(start, end, null);
        double dataSetsMun = getDataSetsMap(start, end, null);
        totalMap.put("orgCode", cloud);//机构code
        totalMap.put("orgName", cloudName);//机构名称
@@ -282,6 +289,7 @@ public class DataQualityHomeService extends BaseJpaService {
     * @param end   就诊时间- 截止
     */
    public Map<String, Object> getCorrectStatistics(String start, String end) throws Exception {
        long startlong = System.currentTimeMillis();
        Map<String, Object> totalMap = new HashMap<>();
        String dateStr = DateUtil.toString(new Date());
        if (StringUtils.isBlank(start)) {
@@ -292,15 +300,21 @@ public class DataQualityHomeService extends BaseJpaService {
        }
        double totalCorrect = 0;//总准确数
        Map<String, Object> dataMap = new HashMap<>();
        long starttime1 = System.currentTimeMillis();
        //1. 去重准确数
        totalCorrect = getErrorDataSetData("receive_date",start, end, null);
        long endTime1 = System.currentTimeMillis();
        logger.info("去重准确数耗时："+ (endTime1-starttime1));
        //2. 数据集总量
        int dataSetsMun = getDataSetsMap(start, end, null);
        double dataSetsMun = getDataSetsMap(start, end, null);
        long endTime2= System.currentTimeMillis();
        logger.info("总准确数据集数耗时："+ (endTime2-endTime1));
        totalMap.put("totalCorrect", totalCorrect);//准确数
        totalMap.put("totalDataSet", dataSetsMun);//数据集总行数
        //3  计算准确性
        totalMap.put("correctRate", calRate(totalCorrect, dataSetsMun));//数据集准确率
        long endlong = System.currentTimeMillis();
        logger.info("总耗时："+ (endlong-startlong));
        return totalMap;
    }
@@ -350,7 +364,8 @@ public class DataQualityHomeService extends BaseJpaService {
     * @return
     */
    public double getErrorDataSetData(String dateField,String start, String end, String orgCode) {
        Map<String, Object> map = new HashMap<String, Object>();
        long starttime1 = System.currentTimeMillis();
        List< Map<String, Object>> resultList = new ArrayList<>();
        double num1 = 0;
        try {
            String dateStr = DateUtil.toString(new Date());
@@ -360,29 +375,45 @@ public class DataQualityHomeService extends BaseJpaService {
            if (StringUtils.isBlank(end)) {
                end = dateStr;
            }
            List<String> fields = new ArrayList<String>();
            fields.add("dataset");
            fields.add("count");
            String sql1 = "";
            // 原生es
            TransportClient transportClient = elasticSearchPool.getClient();
            StringBuilder stringBuilder = new StringBuilder();
            stringBuilder.append("receive_date>=" + start + " 00:00:00;");
            stringBuilder.append("receive_date<=" + end + " 23:59:59;");
            stringBuilder.append("qc_step=1||qc_step=2;");
            if (StringUtils.isNotEmpty(orgCode)) {
                sql1 = "SELECT dataset,count(DISTINCT event_no) as count from json_archives_qc/qc_metadata_info where org_code='" + orgCode + "' " +
                        dateField + ">='" + start + " 00:00:00' and "+ dateField + "<='" + end + " 23:59:59' and (qc_step=1 or qc_step=2) group by dataset";
            } else {
                sql1 = "SELECT dataset,count(DISTINCT event_no) as count from json_archives_qc/qc_metadata_info where " +
                        dateField +">='" + start + " 00:00:00' and "+dateField+"<='" + end + " 23:59:59' and (qc_step=1 or qc_step=2) group by dataset";
                stringBuilder.append("org_code=" + orgCode);
            }
            List<Map<String, Object>> list1 = elasticSearchUtil.findBySql(fields, sql1);
            if (list1 != null && list1.size() > 0) {
                for (Map<String, Object> map1 : list1) {
                    num1 += (double) map1.get("count");
                }
            SearchRequestBuilder builder = transportClient.prepareSearch("json_archives_qc");
            builder.setTypes("qc_metadata_info");
            builder.setSearchType(SearchType.DFS_QUERY_THEN_FETCH);
            builder.setQuery(elasticSearchUtil.getQueryBuilder(stringBuilder.toString()));
            AggregationBuilder terms = AggregationBuilders.terms("dataset").field("dataset").size(200);
            CardinalityBuilder childTerms = AggregationBuilders.cardinality("count").field("pack_id").precisionThreshold(40000);
            terms.subAggregation(childTerms);
            builder.addAggregation(terms);
            builder.setSize(0);
            builder.setFrom(0);
            SearchResponse response = builder.get();
            StringTerms longTerms = response.getAggregations().get("dataset");
            for (Terms.Bucket item : longTerms.getBuckets()) {
                InternalCardinality extendedStats = item.getAggregations().get("count");
//                Map<String, Object> temp = new HashMap<>();
//                temp.put("dataset", item.getKeyAsString());
//                temp.put("count", extendedStats.getValue());
//                resultList.add(temp);
                num1 += extendedStats.getValue();
            }
        } catch (Exception e) {
            e.printStackTrace();
            num1 = 0;
        }
        return num1;
            return num1;
    }
@@ -394,9 +425,10 @@ public class DataQualityHomeService extends BaseJpaService {
     * @param orgCode
     * @return
     */
    public int getDataSetsMap(String start, String end, String orgCode) throws IOException {
    public double getDataSetsMap(String start, String end, String orgCode) throws IOException {
        //  数据集总量
        int totalNum = 0;
        double totalNum = 0;
        // 默认时间
        String dateStr = DateUtil.toString(new Date());
        if (StringUtils.isBlank(start)) {
            start = dateStr;
@@ -404,18 +436,26 @@ public class DataQualityHomeService extends BaseJpaService {
        if (StringUtils.isBlank(end)) {
            end = dateStr;
        }
        StringBuilder stringBuilder1 = new StringBuilder();
        stringBuilder1.append("receive_date>=").append(start).append(" 00:00:00").append(";");
        stringBuilder1.append("receive_date<=").append(end).append(" 23:59:59").append(";");
        if (StringUtils.isNotBlank(orgCode)) {
            stringBuilder1.append("org_code=" + orgCode);
        }
        List<Map<String, Object>> dataSets = elasticSearchUtil.list("json_archives_qc", "qc_dataset_detail", stringBuilder1.toString());
        for (Map<String, Object> dataSet : dataSets) {
            for (Map.Entry<String, Object> entry : dataSet.entrySet()) {
                totalNum += (Integer) dataSet.get("count");
            }
        }
        TransportClient transportClient = elasticSearchPool.getClient();
        //  条件设置
        StringBuilder stringBuilder = new StringBuilder();
        stringBuilder.append("receive_date>=" + start + " 00:00:00;");
        stringBuilder.append("receive_date<=" + end + " 23:59:59;");
        if (StringUtils.isNotEmpty(orgCode)) {
            stringBuilder.append("org_code=" + orgCode);
        }
        SearchRequestBuilder builder = transportClient.prepareSearch("json_archives_qc");
        builder.setTypes("qc_dataset_detail");
        builder.setSearchType(SearchType.DFS_QUERY_THEN_FETCH);
        builder.setQuery(elasticSearchUtil.getQueryBuilder(stringBuilder.toString()));
        SumBuilder terms = AggregationBuilders.sum("sumCount").field("count");
        builder.addAggregation(terms);
        builder.setSize(0);
        builder.setFrom(0);
        SearchResponse response = builder.get();
        InternalSum longTerms = response.getAggregations().get("sumCount");
        totalNum = longTerms.getValue();
        return totalNum;
    }
@@ -648,7 +688,7 @@ public class DataQualityHomeService extends BaseJpaService {
        List<Double> data = new ArrayList<>();
        List<String> name = new ArrayList<>();
        if("10".equals(flag)){//按月
            for(int i=0;i<6;i++){
            for(int i=5;i>=0;i--){
                String start =  DateUtil.formatDate(DateUtil.addMonth(0-i, DateUtil.formatCharDateYMD(month+"-01")), "yyyy-MM");
                String end =  DateUtil.getLastDate (start+"-01", "yyyy-MM-dd", "yyyy-MM-dd");
                double rate = getRate(dataType, start+"-01", end);
@@ -723,7 +763,7 @@ public class DataQualityHomeService extends BaseJpaService {
            //3. 去重准确数【】
            totalCorrect = getErrorDataSetData("receive_date",start, end, null);
            //4. 数据集总量
            int dataSetsMun = getDataSetsMap(start, end, null);
            double dataSetsMun = getDataSetsMap(start, end, null);
            if(dataSetsMun==0){
                return 0.0;
            }else{
@@ -813,9 +853,9 @@ public class DataQualityHomeService extends BaseJpaService {
            String keyAsString = entry.getKeyAsString();
            long docCount = entry.getDocCount();
            map.put(keyAsString,docCount);
            System.out.println("key [{" + keyAsString + "}]");
            System.out.println("date [{" + key + "}]");
            System.out.println("doc_count [{" + docCount + "}]");
            logger.info("key [{" + keyAsString + "}]");
            logger.info("date [{" + key + "}]");
            logger.info("doc_count [{" + docCount + "}]");
        }
        return map;
    }