Merge branch 'dev' of liuwenbin/patient-co-management into dev

patient-co/patient-co-statistics-es/src/main/java/com/yihu/wlyy/statistics/etl/extract/db/Data2Save.java

@@ -1,14 +1,19 @@
package com.yihu.wlyy.statistics.etl.extract.db;
import com.yihu.wlyy.entity.dimension.WlyyDimensionQuota;
import com.yihu.wlyy.entity.doctor.team.admin.AdminTeam;
import com.yihu.wlyy.entity.job.QuartzJobConfig;
import com.yihu.wlyy.statistics.dao.WlyyDimensionQuotaDao;
import com.yihu.wlyy.statistics.vo.DataModel;
import com.yihu.wlyy.statistics.vo.DictModel;
import com.yihu.wlyy.statistics.vo.SaveModel;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.BeanPropertyRowMapper;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component;
import java.lang.reflect.InvocationTargetException;
import java.util.*;
/**
@@ -18,23 +23,34 @@ import java.util.*;
public class Data2Save {
    @Autowired
    private JdbcTemplate jdbcTemplate;
    @Autowired
    private WlyyDimensionQuotaDao dimensionQuotaDao;
    public List<SaveModel> data2save(List<DataModel> dataModels, QuartzJobConfig quartzJobConfig, Date quotaDate, String timeLevel) {
    public List<SaveModel> data2save(List<DataModel> dataModels, QuartzJobConfig quartzJobConfig, Date quotaDate, String timeLevel) throws Exception{
        //得到全部的指标
        List<WlyyDimensionQuota> dimensionQuotas = dimensionQuotaDao.findDimensionQuotasByQuotaCode(quartzJobConfig.getQuotaId());
        //初始化维度的数据
        Map<String, DataModel> dataModelMap = new HashMap<>();
        if (dataModels != null) {
            for (DataModel dataModel : dataModels) {
                dataModelMap.put(dataModel.getTeam(), dataModel);
                StringBuilder sb = new StringBuilder();
                sb.append(dataModel.getTeam());
                for(int i = 0; i < dimensionQuotas.size(); i++){
                    int slaveKeyNum = i+1;
                    String invokeKey = (String) DataModel.class.getMethod("getSlaveKey"+slaveKeyNum).invoke(dataModel);
                    sb.append("-"+invokeKey);
                }
                dataModelMap.put(sb.toString(), dataModel);
            }
        }
        List<SaveModel> savelist = new ArrayList<>();
        //得到全部团队
        List<AdminTeam> teams = findAllTeam();
        List<SaveModel> savelist = new ArrayList<>();
        //没维度
        for (AdminTeam adminTeam : teams) {
            DataModel dataModel = dataModelMap.get(adminTeam.getId().toString());
            SaveModel saveModel = new SaveModel();
            saveModel.setCity("350206");
@@ -50,20 +66,52 @@ public class Data2Save {
            saveModel.setTimeLevel(timeLevel);
            saveModel.setCreateTime(new Date());
            saveModel.setQuotaDate(quotaDate);
            if (dataModel != null) {
                saveModel.setResult1(dataModel.getResult1());
                saveModel.setResult2(dataModel.getResult1());
            } else {
                saveModel.setResult1(0.0);
                saveModel.setResult2(0.0);
            }
            saveModel.setResult1(0.0);
            saveModel.setResult2(0.0);
            savelist.add(saveModel);
        }
        //如果有维度就设置维度的数据
        for (int i = 0; i < dimensionQuotas.size(); i++) {
            String dictSql = dimensionQuotas.get(i).getDictSql();
            List<DictModel> dictModels = jdbcTemplate.query(dictSql, new BeanPropertyRowMapper(DictModel.class));
            int savleKeyNum = i + 1;
            savelist = setSlaveKey(savleKeyNum, savelist, dictModels,dataModelMap,dimensionQuotas.size());
        }
        return savelist;
    }
    private List<SaveModel> setSlaveKey(int savleKeyNum, List<SaveModel> savelist, List<DictModel> dictModels,Map<String, DataModel> dataModelMap,int dimensize) {
        List<SaveModel> newSavelist = new ArrayList<>();
        savelist.stream().forEach(one -> {
            dictModels.stream().forEach(dict->{
                SaveModel newSaveModel = new SaveModel();
                BeanUtils.copyProperties(one, newSaveModel);
                try {
                    SaveModel.class.getMethod("setSlaveKey" + savleKeyNum , String.class).invoke(newSaveModel,dict.getCode());
                    SaveModel.class.getMethod("setSlaveKey" + savleKeyNum + "Name", String.class).invoke(newSaveModel,dict.getName());
                    if(savleKeyNum==dimensize){
                        StringBuilder sb = new StringBuilder();
                        sb.append(newSaveModel.getTeam());
                        for(int i=1;i<=dimensize;i++){
                            String invokeKey = (String) SaveModel.class.getMethod("getSlaveKey"+i).invoke(newSaveModel);
                            sb.append("-"+invokeKey);
                        }
                        DataModel dataModel = dataModelMap.get(sb.toString());
                        if(dataModel!=null){
                            newSaveModel.setResult1(dataModel.getResult1());
                        }
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                }
                newSavelist.add(newSaveModel);
            });
        });
        return newSavelist;
    }
    /**
     * 得到全部团队

patient-co/patient-co-statistics-es/src/main/java/com/yihu/wlyy/statistics/etl/extract/db/ESExtract.java

@@ -1,10 +1,15 @@
package com.yihu.wlyy.statistics.etl.extract.db;
import com.yihu.wlyy.entity.dimension.WlyyDimensionQuota;
import com.yihu.wlyy.entity.job.QuartzJobConfig;
import com.yihu.wlyy.statistics.dao.WlyyDimensionQuotaDao;
import com.yihu.wlyy.statistics.util.ElasticsearchUtil;
import com.yihu.wlyy.statistics.vo.DataModel;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Scope;
import org.springframework.jdbc.core.BeanPropertyRowMapper;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component;
import java.util.*;
@@ -18,6 +23,14 @@ import java.util.*;
public class ESExtract<T> {
    @Autowired
    private ElasticsearchUtil elasticsearchUtil;
    @Value("${es.type}")
    private String esType;
    @Value("${es.index}")
    private String esIndex;
    @Autowired
    private WlyyDimensionQuotaDao dimensionQuotaDao;
    @Autowired
    private JdbcTemplate jdbcTemplate;
    /**
     * ES抽取
@@ -26,17 +39,32 @@ public class ESExtract<T> {
     * @return
     */
    public List<DataModel> extract(QuartzJobConfig wlyyJobConfigVO) {
        //扩展维度
        //扩展维度
        String sql= initSql(wlyyJobConfigVO.getSql(),wlyyJobConfigVO.getStartTime(),wlyyJobConfigVO.getEndTime());
        String re = "";
        //得到该指标的维度
        List<WlyyDimensionQuota> dimensionQuotas = dimensionQuotaDao.findDimensionQuotasByQuotaCode(wlyyJobConfigVO.getQuotaId());
        if (dimensionQuotas != null && dimensionQuotas.size() > 0) {
            for(int i=1;i<=dimensionQuotas.size();i++){
                sql+=",slaveKey"+i;
                re+=","+dimensionQuotas.get(i-1).getKey()+" AS slaveKey"+i;
//                dictSql = one.getDictSql();
//                List<Map<String, Object>> temp = jdbcTemplate.queryForList(dictSql);
        List<DataModel> dataModels = elasticsearchUtil.excuteDataModel(sql);
            }
        }
        sql = sql.replace("[re]", re);
//        sql="SELECT adminTeamCode AS team ,COUNT(*) AS result1  ,isRead AS slaveKey1   FROM health_edu_article_patient_test3 where userType=2 AND createTime < '2018-01-04T17:00:00+0800'  AND createTime >= '2018-01-03T17:00:00+0800' group by team,slaveKey1";
        List<DataModel> dataModels = elasticsearchUtil.excute(sql,DataModel.class);
        return dataModels;
    }
    //初始化时间
    public String initSql(String sql, String startTime, String endTime) {
        return sql.replace("[startTime]", startTime).replace("[endTime]", endTime);
        return sql.replace("[table]","health_edu_article_patient_test3").replace("[startTime]", startTime).replace("[endTime]", endTime);
    }
}

patient-co/patient-co-statistics-es/src/main/java/com/yihu/wlyy/statistics/job/business/EsToEsQuotaJob.java

@@ -17,6 +17,7 @@ import com.yihu.wlyy.statistics.etl.save.es.ElasticFactory;
import com.yihu.wlyy.statistics.util.DateUtil;
import com.yihu.wlyy.statistics.util.SpringUtil;
import com.yihu.wlyy.statistics.vo.DataModel;
import com.yihu.wlyy.statistics.vo.ErrModel;
import com.yihu.wlyy.statistics.vo.FilterModel;
import com.yihu.wlyy.statistics.vo.SaveModel;
import io.searchbox.client.JestClient;
@@ -152,6 +153,17 @@ public class EsToEsQuotaJob implements Job {
            deleteData(quotaDate, wlyyJobCongId, timeLevel);
            // 1..抽取数据 如果是累加就是 List<DataModel>  如果是相除 Map<String,List<DataModel>>
            List<DataModel> dataModels = extract();
//            //扩展 有维度的统计
//
            //兼容旧代码，封装
//            FilterModel filterModel = new FilterModel(dataModels,new ArrayList<ErrModel>());
//            //得到该指标的维度
//            List<WlyyDimensionQuota> dimensionQuotas = dimensionQuotaDao.findDimensionQuotasByQuotaCode(wlyyJobCongId);
//            //2.1.从维度的key转换
//            if (dimensionQuotas != null && dimensionQuotas.size() > 0) {
//                filterModel = convert(filterModel, dimensionQuotas);
//            }
            // 2 DataModel 转SaveModel即可
            List<SaveModel> saveModels = data2Save.data2save(dataModels,quartzJobConfig,quotaDate,timeLevel);
            // 3.保存数据

patient-co/patient-co-statistics-es/src/main/java/com/yihu/wlyy/statistics/util/ElasticsearchUtil.java

@@ -3,9 +3,11 @@ package com.yihu.wlyy.statistics.util;
import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.expr.SQLQueryExpr;
import com.alibaba.druid.sql.parser.SQLExprParser;
import com.yihu.wlyy.entity.patient.Patient;
import com.yihu.wlyy.statistics.etl.save.es.ElasticFactory;
import com.yihu.wlyy.statistics.vo.DataModel;
import com.yihu.wlyy.statistics.vo.SaveModel;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.Client;
import org.nlpcn.es4sql.domain.Select;
@@ -20,10 +22,14 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.List;
import java.sql.Timestamp;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
/**
 * Created by chenweida on 2017/7/17.
@@ -374,6 +380,112 @@ public class ElasticsearchUtil {
        return saveModels;
    }
    public<T> List<T> excute(String sql, Class<T> clazz) {
        List saveModels = new ArrayList<>();
        SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXX");
        SimpleDateFormat dateFormat1 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        try {
            //解决 group by之后默认是200的问题
            if (sql.toLowerCase().contains("group by")) {
                sql = sql + " limit 0,2000";
            }
            SQLExprParser parser = new ElasticSqlExprParser(sql);
            SQLExpr expr = parser.expr();
            SQLQueryExpr queryExpr = (SQLQueryExpr) expr;
            Select select = null;
            select = new SqlParser().parseSelect(queryExpr);
            //通过抽象语法树，封装成自定义的Select，包含了select、from、where group、limit等
            AggregationQueryAction action = null;
            DefaultQueryAction queryAction = null;
            SqlElasticSearchRequestBuilder requestBuilder = null;
            if (select.isAgg) {
                //包含计算的的排序分组的
                action = new AggregationQueryAction(elasticFactory.getTransportClient(), select);
                requestBuilder = action.explain();
            } else {
                //封装成自己的Select对象
                queryAction = new DefaultQueryAction(elasticFactory.getTransportClient(), select);
                requestBuilder = queryAction.explain();
            }
            SearchResponse response = (SearchResponse) requestBuilder.get();
            Object queryResult = null;
            if (sql.toUpperCase().indexOf("GROUP") != -1 || sql.toUpperCase().indexOf("SUM") != -1 || select.isAgg) {
                queryResult = response.getAggregations();
            } else {
                queryResult = response.getHits();
            }
            ObjectResult temp = new ObjectResultsExtractor(true, true, true).extractResults(queryResult, true);
            List<String> heads = temp.getHeaders();
            temp.getLines().forEach(one -> {
                Object saveModel = null;
                try {
                    saveModel = clazz.newInstance();
                } catch (Exception e) {
                    logger.error(e.getMessage());
                }
                for (int i = 0; i < one.size(); i++) {
                    try {
                        String key = null;
                        Object value = one.get(i);
                        if (heads.get(i).startsWith("_")) {
                            continue;
                        }
                        key = "set" + UpFirstStr(heads.get(i));
                        if (heads.get(i).contains("quotaDate") || heads.get(i).contains("createTime") || heads.get(i).contains("date_histogram")) {
                            if (heads.get(i).contains("date_histogram")) {
                                key = "setQuotaDate";
                            }
                            try {
                                //yyyy-MM-dd'T'HH:mm:ssXX
                                value = dateFormat.parse(String.valueOf(one.get(i)));
                            } catch (Exception e) {
                                //yyyy-MM-dd HH:mm:ss
                                try {
                                    value = dateFormat1.parse(String.valueOf(one.get(i)));
                                }catch (Exception e1){
                                    Timestamp ts = new Timestamp(Long.parseLong(String.valueOf(one.get(i))));
                                    try {
                                        Date date = new Date();
                                        date = ts;
                                        value =date;
                                    } catch (Exception e2) {
                                        value = String.valueOf(one.get(i));
                                    }
                                }
                            }
//                            value = DateUtil.strToDate(String.valueOf(value).replace("T00:00:00+0800", " 00:00:00"), "yyyy-MM-dd HH:mm:ss");
                        }
                        if (value instanceof String) {
                            clazz.getMethod(key, String.class).invoke(saveModel, value);
                        } else if (value instanceof Integer) {
                            clazz.getMethod(key, Integer.class).invoke(saveModel, value);
                        } else if (value instanceof Double) {
                            clazz.getMethod(key, Double.class).invoke(saveModel, value);
                        } else if (value instanceof java.util.Date) {
                            clazz.getMethod(key, java.util.Date.class).invoke(saveModel, value);
                        } else if (value instanceof java.util.List) {
                            clazz.getMethod(key, java.util.List.class).invoke(saveModel, value);
                        }
                    } catch (Exception e) {
                        logger.warn(e.getMessage());
                    }
                }
                saveModels.add(saveModel);
            });
        } catch (Exception e) {
            logger.error(e.getMessage());
        }
        return saveModels;
    }
    /**
     * 执行sql查询es
@@ -416,7 +528,8 @@ public class ElasticsearchUtil {
            }
            ObjectResult temp = new ObjectResultsExtractor(true, true, true).extractResults(queryResult, true);
            List<String> heads = temp.getHeaders();
            temp.getLines().stream().forEach(one -> {
            for(List<Object> one:temp.getLines()){
//            temp.getLines().stream().forEach(one -> {
                try {
                    DataModel dataModel = new DataModel();
                    for (int i = 0; i < one.size(); i++) {
@@ -448,8 +561,8 @@ public class ElasticsearchUtil {
                    saveModels.add(dataModel);
                } catch (Exception e) {
                    e.printStackTrace();
                }
            });
                }}
//            });
        } catch (Exception e) {
            e.printStackTrace();
        }

patient-co/patient-co-statistics-es/src/main/java/com/yihu/wlyy/statistics/vo/DataModel.java

@@ -28,6 +28,7 @@ public class DataModel {
    private Double num = 1.0;//分数 如果是累加的计算 默认是1 如果是分数从数据库拿
    private String prescriptionCode;//处方code
    private String healthProblem;//诊断标签
    private Integer isRead;//是否已读
    private Double result1 = 0.0; //从ES统计的时候的数目
@@ -209,4 +210,11 @@ public class DataModel {
        this.result1 = result1;
    }
    public Integer getIsRead() {
        return isRead;
    }
    public void setIsRead(Integer isRead) {
        this.isRead = isRead;
    }
}