统计代码重构

patient-co/patient-co-statistics-es/pom.xml

@@ -442,8 +442,9 @@
            <version>2.4.1.0</version>
        </dependency>
        <dependency>
            <groupId>com.yihu</groupId>
            <artifactId>common-entity</artifactId>
            <groupId>org.elasticsearch.plugin</groupId>
            <artifactId>delete-by-query</artifactId>
            <version>2.4.1</version>
        </dependency>
        <dependency>
            <groupId>com.alibaba</groupId>

patient-co/patient-co-statistics-es/src/main/java/com/yihu/wlyy/statistics/etl/extract/db/Data2Save.java

@@ -0,0 +1,96 @@
package com.yihu.wlyy.statistics.etl.extract.db;
import com.yihu.wlyy.entity.doctor.team.admin.AdminTeam;
import com.yihu.wlyy.entity.job.QuartzJobConfig;
import com.yihu.wlyy.statistics.vo.DataModel;
import com.yihu.wlyy.statistics.vo.SaveModel;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.BeanPropertyRowMapper;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component;
import java.util.*;
/**
 * Created by chenweida on 2017/12/28.
 */
@Component
public class Data2Save {
    @Autowired
    private JdbcTemplate jdbcTemplate;
    public List<SaveModel> data2save(List<DataModel> dataModels, QuartzJobConfig quartzJobConfig, Date quotaDate, String timeLevel) {
        //初始化维度的数据
        Map<String, DataModel> dataModelMap = new HashMap<>();
        if (dataModels != null) {
            for (DataModel dataModel : dataModels) {
                dataModelMap.put(dataModel.getTeam(), dataModel);
            }
        }
        List<SaveModel> savelist = new ArrayList<>();
        //得到全部团队
        List<AdminTeam> teams = findAllTeam();
        for (AdminTeam adminTeam : teams) {
            DataModel dataModel = dataModelMap.get(adminTeam.getId().toString());
            SaveModel saveModel = new SaveModel();
            saveModel.setCity("350206");
            saveModel.setCityName("厦门市");
            saveModel.setTown(adminTeam.getTownCode());
            saveModel.setTownName(adminTeam.getTownName());
            saveModel.setHospital(adminTeam.getOrgCode());
            saveModel.setHospitalName(adminTeam.getOrgName());
            saveModel.setTeam(adminTeam.getId().toString());
            saveModel.setTeamName(adminTeam.getName());
            saveModel.setQuotaCode(quartzJobConfig.getQuotaId());
            saveModel.setAreaLevel(SaveModel.teamLevel);
            saveModel.setTimeLevel(timeLevel);
            saveModel.setCreateTime(new Date());
            saveModel.setQuotaDate(quotaDate);
            if (dataModel != null) {
                saveModel.setResult1(dataModel.getResult1());
                saveModel.setResult2(dataModel.getResult1());
            } else {
                saveModel.setResult1(0.0);
                saveModel.setResult2(0.0);
            }
            savelist.add(saveModel);
        }
        return savelist;
    }
    /**
     * 得到全部团队
     *
     * @return
     */
    private List<AdminTeam> findAllTeam() {
        //  Map<String, AdminTeam> adminTeamMap = new HashMap<>();
        String sql = "SELECT " +
                "  wat.id id, " +
                "  wat.name name, " +
                "  h.`code` org_code, " +
                "  h.`name` org_name, " +
                "  h.town   town_code, " +
                "  h.town_name  town_name " +
                "FROM " +
                "  dm_hospital h, " +
                "  wlyy_admin_team wat " +
                "WHERE " +
                "  h.`code` = wat.org_code " +
                "AND wat.available = 1";
        List<AdminTeam> adminTeams = jdbcTemplate.query(sql, new BeanPropertyRowMapper(AdminTeam.class));
//        for (AdminTeam adminTeam : adminTeams) {
//            adminTeamMap.put(adminTeam.getId().toString(), adminTeam);
//        }
        return adminTeams;
    }
}

patient-co/patient-co-statistics-es/src/main/java/com/yihu/wlyy/statistics/job/business/CurrentMysqlToEsQuotaJob.java

@@ -10,6 +10,7 @@ import com.yihu.wlyy.statistics.etl.cache.Cache;
import com.yihu.wlyy.statistics.etl.compute.ComputeHelper;
import com.yihu.wlyy.statistics.etl.convert.ConvertHelper;
import com.yihu.wlyy.statistics.etl.extract.ExtractHelper;
import com.yihu.wlyy.statistics.etl.extract.db.Data2Save;
import com.yihu.wlyy.statistics.etl.filter.FilterHelper;
import com.yihu.wlyy.statistics.etl.save.SaveHelper;
import com.yihu.wlyy.statistics.etl.save.es.ElasticFactory;
@@ -77,6 +78,8 @@ public class CurrentMysqlToEsQuotaJob implements Job {
    private QuartzJobConfigDao quartzJobConfigDao;
    @Autowired
    private StringRedisTemplate redisTemplate;
    @Autowired
    private Data2Save data2Save;
    public void execute(JobExecutionContext context)
@@ -135,31 +138,11 @@ public class CurrentMysqlToEsQuotaJob implements Job {
        List<QuartzJobConfig> list = quartzJobConfigDao.findByIds();
        list.stream().forEach(one -> {
            try {
                logger.info("========================quotaCode:" + one.getId() + "," + DateUtil.dateToStr(quotaDate, "yyyy-MM-dd") + ",timeLevel:" + timeLevel + " start========================");
                QuartzJobLog tjQuotaLog = new QuartzJobLog();
                tjQuotaLog.setJobId(one.getId());
                tjQuotaLog.setJobStartTime(new Date());
                //1..抽取数据 如果是累加就是 List<DataModel>  如果是相除 Map<String,List<DataModel>>
                List<DataModel> dataModels = extract(one);
                //2..根据规则过滤数据
                FilterModel filterModel = filter(dataModels);
                //得到改指标的维度
                List<WlyyDimensionQuota> dimensionQuotas = dimensionQuotaDao.findDimensionQuotasByQuotaCode(one.getId());
                //2.1.从维度的key转换
                if (dimensionQuotas != null && dimensionQuotas.size() > 0) {
                    filterModel = convert(filterModel, dimensionQuotas);
                if (StringUtils.isEmpty(one.getExtractType()) || "1".equals(one.getExtractType())){
                    mysql(one);
                }else if("2".equals(one.getExtractType())){
                    es(one);
                }
                //3.统计数据
                List<SaveModel> sms = compute(filterModel.getData(), dimensionQuotas, timeLevel, one);
                //4.更新数据
                Boolean success = updateData(sms, quotaDate, one.getId(), timeLevel, dimensionQuotas.size());
                tjQuotaLog.setJobType(success ? "1" : "0");
                tjQuotaLog.setJobEndTime(new Date());
                tjQuotaLog.setJobContent(JSONArray.fromObject(filterModel.getErrorModels()).toString());
                saveLog(tjQuotaLog);
                logger.info("========================quotaCode:" + one.getId() + "," + DateUtil.dateToStr(quotaDate, "yyyy-MM-dd") + " end========================");
            } catch (Exception e) {
                e.printStackTrace();
            }
@@ -169,15 +152,89 @@ public class CurrentMysqlToEsQuotaJob implements Job {
    }
    private void es(QuartzJobConfig one) throws Exception{
        this.endTime = DateUtil.getStringDate("yyyy-MM-dd'T'HH:mm:ssZZZ");
        //初始化统计年份
        this.year = getNowYearByDate();
        //初始化开始时间
        if ("2".equals(timeLevel)) {
            //按年度到达量
            startTime = this.year + "-06-30T17:00:00+0800";
        } else {
            //增量
            this.startTime = new LocalDate(new DateTime().minusDays(1)).toString("yyyy-MM-dd") + "T17:00:00+0800"; //2017-06-01 17:00:00
        }
        this.quotaDate = DateUtil.strToDate(endTime, "yyyy-MM-dd");
        one.setStartTime(startTime);
        one.setEndTime(endTime);
        logger.info("========================quotaCode:" + one.getQuotaId() + "," + DateUtil.dateToStr(quotaDate, "yyyy-MM-dd") + ",timeLevel:" + timeLevel + " start========================");
        QuartzJobLog tjQuotaLog = new QuartzJobLog();
        tjQuotaLog.setJobId( one.getQuotaId());
        tjQuotaLog.setJobStartTime(new Date());
        // 1..抽取数据 如果是累加就是 List<DataModel>  如果是相除 Map<String,List<DataModel>>
        List<DataModel> dataModels = extract2Es(one);
        // 2 DataModel 转SaveModel即可
        List<SaveModel> saveModels = data2Save.data2save(dataModels,one,quotaDate,timeLevel);
        // 3.更新数据
        Boolean success = updateData(saveModels, quotaDate, one.getId(), timeLevel);
        tjQuotaLog.setJobType(success ? "1" : "0");
        tjQuotaLog.setJobEndTime(new Date());
        //tjQuotaLog.setJobContent(JSONArray.fromObject(filterModel.getErrorModels()).toString());
        saveLog(tjQuotaLog);
        logger.info("========================quotaCode:" +  one.getQuotaId() + "," + DateUtil.dateToStr(quotaDate, "yyyy-MM-dd") + " end========================");
    }
    private void mysql(QuartzJobConfig one) {
        logger.info("========================quotaCode:" + one.getId() + "," + DateUtil.dateToStr(quotaDate, "yyyy-MM-dd") + ",timeLevel:" + timeLevel + " start========================");
        QuartzJobLog tjQuotaLog = new QuartzJobLog();
        tjQuotaLog.setJobId(one.getId());
        tjQuotaLog.setJobStartTime(new Date());
        //1..抽取数据 如果是累加就是 List<DataModel>  如果是相除 Map<String,List<DataModel>>
        List<DataModel> dataModels = extract2Es(one);
        //2..根据规则过滤数据
        FilterModel filterModel = filter(dataModels);
        //得到改指标的维度
        List<WlyyDimensionQuota> dimensionQuotas = dimensionQuotaDao.findDimensionQuotasByQuotaCode(one.getId());
        //2.1.从维度的key转换
        if (dimensionQuotas != null && dimensionQuotas.size() > 0) {
            filterModel = convert(filterModel, dimensionQuotas);
        }
        //3.统计数据
        List<SaveModel> sms = compute(filterModel.getData(), dimensionQuotas, timeLevel, one);
        //4.更新数据
        Boolean success = updateData(sms, quotaDate, one.getId(), timeLevel);
        tjQuotaLog.setJobType(success ? "1" : "0");
        tjQuotaLog.setJobEndTime(new Date());
        tjQuotaLog.setJobContent(JSONArray.fromObject(filterModel.getErrorModels()).toString());
        saveLog(tjQuotaLog);
        logger.info("========================quotaCode:" + one.getId() + "," + DateUtil.dateToStr(quotaDate, "yyyy-MM-dd") + " end========================");
    }
    private List<DataModel> extract2Es(QuartzJobConfig one) {
        try {
            List<DataModel> dataModels = SpringUtil.getBean(ExtractHelper.class).extractData(one, startTime, endTime, year, timeLevel);
            logger.info("quotaCode:" + one.getQuotaId() + ",size:" + dataModels.size());
            return dataModels;
        } catch (Exception e) {
            logger.error("extract error:" + e.getMessage());
        }
        return null;
    }
    /**
     * 删除 某个指标某一天的某个timelevel的数据
     *
     * @param quotaDate
     * @param quotaCode
     * @param timeLevel
     * @param size
     */
    private boolean updateData(List<SaveModel> sms, Date quotaDate, String quotaCode, String timeLevel, int size) {
    private boolean updateData(List<SaveModel> sms, Date quotaDate, String quotaCode, String timeLevel) {
        JestClient jestClient = null;
        try {
            jestClient = elasticFactory.getJestClient();
@@ -345,4 +402,9 @@ public class CurrentMysqlToEsQuotaJob implements Job {
            return (1900 + today.getYear() - 1) + "";
        }
    }
    public static void main(String[] args) {
        SimpleDateFormat s=new SimpleDateFormat("");
        System.out.println(s.format(new Date()));
    }
}

patient-co/patient-co-statistics-es/src/main/java/com/yihu/wlyy/statistics/job/business/EsToEsQuotaJob.java

@@ -10,6 +10,7 @@ import com.yihu.wlyy.statistics.dao.WlyyDimensionQuotaDao;
import com.yihu.wlyy.statistics.etl.compute.ComputeHelper;
import com.yihu.wlyy.statistics.etl.convert.ConvertHelper;
import com.yihu.wlyy.statistics.etl.extract.ExtractHelper;
import com.yihu.wlyy.statistics.etl.extract.db.Data2Save;
import com.yihu.wlyy.statistics.etl.filter.FilterHelper;
import com.yihu.wlyy.statistics.etl.save.SaveHelper;
import com.yihu.wlyy.statistics.etl.save.es.ElasticFactory;
@@ -74,6 +75,8 @@ public class EsToEsQuotaJob implements Job {
    private String esType;
    @Value("${es.index}")
    private String esIndex;
    @Autowired
    private Data2Save data2Save;
    public void execute(JobExecutionContext context)
            throws JobExecutionException {
@@ -150,7 +153,7 @@ public class EsToEsQuotaJob implements Job {
            // 1..抽取数据 如果是累加就是 List<DataModel>  如果是相除 Map<String,List<DataModel>>
            List<DataModel> dataModels = extract();
            // 2 DataModel 转SaveModel即可
            List<SaveModel> saveModels = data2save(dataModels);
            List<SaveModel> saveModels = data2Save.data2save(dataModels,quartzJobConfig,quotaDate,timeLevel);
            // 3.保存数据
            Boolean success = saveDate(saveModels);
@@ -164,49 +167,7 @@ public class EsToEsQuotaJob implements Job {
        }
    }
    private List<SaveModel> data2save(List<DataModel> dataModels) {
        //初始化维度的数据
        Map<String,DataModel> dataModelMap=new HashMap<>();
        if(dataModels!=null){
            for(DataModel dataModel:dataModels){
                dataModelMap.put(dataModel.getTeam(),dataModel);
            }
        }
        List<SaveModel> savelist = new ArrayList<>();
        //得到全部团队
        List<AdminTeam>  teams = findAllTeam();
        for(AdminTeam adminTeam:teams){
            DataModel dataModel= dataModelMap.get(adminTeam.getId().toString());
            SaveModel saveModel = new SaveModel();
            saveModel.setCity("350206");
            saveModel.setCityName("厦门市");
            saveModel.setTown(adminTeam.getTownCode());
            saveModel.setTownName(adminTeam.getTownName());
            saveModel.setHospital(adminTeam.getOrgCode());
            saveModel.setHospitalName(adminTeam.getOrgName());
            saveModel.setTeam(adminTeam.getId().toString());
            saveModel.setTeamName(adminTeam.getName());
            saveModel.setQuotaCode(quartzJobConfig.getQuotaId());
            saveModel.setAreaLevel(SaveModel.teamLevel);
            saveModel.setTimeLevel(timeLevel);
            saveModel.setCreateTime(new Date());
            saveModel.setQuotaDate(quotaDate);
            if(dataModel!=null){
                saveModel.setResult1(dataModel.getResult1());
                saveModel.setResult2(dataModel.getResult1());
            }else{
                saveModel.setResult1(0.0);
                saveModel.setResult2(0.0);
            }
            savelist.add(saveModel);
        }
        return savelist;
    }
    /**
     * 删除 某个指标某一天的某个timelevel的数据
@@ -348,31 +309,4 @@ public class EsToEsQuotaJob implements Job {
        }
    }
    /**
     * 得到全部团队
     *
     * @return
     */
    private List<AdminTeam> findAllTeam() {
      //  Map<String, AdminTeam> adminTeamMap = new HashMap<>();
        String sql = "SELECT " +
                "  wat.id id, " +
                "  wat.name name, " +
                "  h.`code` org_code, " +
                "  h.`name` org_name, " +
                "  h.town   town_code, " +
                "  h.town_name  town_name " +
                "FROM " +
                "  dm_hospital h, " +
                "  wlyy_admin_team wat " +
                "WHERE " +
                "  h.`code` = wat.org_code " +
                "AND wat.available = 1";
        List<AdminTeam> adminTeams = jdbcTemplate.query(sql, new BeanPropertyRowMapper(AdminTeam.class));
//        for (AdminTeam adminTeam : adminTeams) {
//            adminTeamMap.put(adminTeam.getId().toString(), adminTeam);
//        }
        return adminTeams;
    }
}