代码修改

svr/svr-quota/pom.xml

@@ -61,6 +61,16 @@
            <groupId>org.elasticsearch</groupId>
            <artifactId>elasticsearch</artifactId>
        </dependency>
        <dependency>
            <groupId>org.nlpcn</groupId>
            <artifactId>elasticsearch-sql</artifactId>
            <version>2.4.1.0</version>
        </dependency>
        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>druid</artifactId>
            <version>1.0.15</version>
        </dependency>
        <dependency>
            <groupId>org.quartz-scheduler</groupId>
            <artifactId>quartz</artifactId>

svr/svr-quota/src/main/java/com/yihu/jw/quota/etl/Contant.java

@@ -1,9 +1,5 @@
package com.yihu.jw.quota.etl;
import com.yihu.jw.quota.model.jpa.dimension.TjQuotaDimensionSlave;
import com.yihu.jw.quota.vo.QuotaDimensionSlaveVO;
import org.springframework.beans.BeanUtils;
import java.util.ArrayList;
import java.util.List;
@@ -30,6 +26,7 @@ public class Contant {
        public static final String computeKey1 = "oneKey";
        public static final String computeKey2 = "senondKey";
    }
    /**
     * 运算常量
     */
@@ -43,8 +40,8 @@ public class Contant {
     */
    public static class main_dimension {
        public static final String time_day = "1";//时间维度 日
        public static final String time_week = "2";//时间维度  月
        public static final String time_month = "3";//时间维度 周
        public static final String time_month = "2";//时间维度 周
        public static final String time_week = "3";//时间维度  月
        public static final String time_year = "4";//时间维度 年
        public static final String area_province = "5";//行政区划 省
        public static final String area_city = "6";//行政区划 市
@@ -53,16 +50,21 @@ public class Contant {
        public static final String area_team = "9";//行政区划  团队
    }
    public static class quota{
        public static final String dataLeval_all="1";
        public static final String dataLeval_oneDay="2";
    }
    /**
     * areaLevel 具体的值
     */
    public static class main_dimension_areaLevel {
        public static final String area_province = "1";//行政区划 省
        public static final String area_city = "2";//行政区划 市
        public static final String area_town = "3";//行政区划 区县
        public static final String area_org = "4";//行政区划 机构
        public static final String area_team = "5";//行政区划  团队
        public static final String area_province = "5";//行政区划 省
        public static final String area_city = "6";//行政区划 市
        public static final String area_town = "7";//行政区划 区县
        public static final String area_org = "8";//行政区划 机构
        public static final String area_team = "9";//行政区划  团队
        public static String getAreaLevelByMainDimension(String key) {
            switch (key) {
@@ -108,6 +110,7 @@ public class Contant {
        public static final String one = "one";
        public static final String two = "two";
    }
    public static class save_status {
        public static final String success = "1";
        public static final String fail = "0";
@@ -122,53 +125,23 @@ public class Contant {
    }
    public static class save {
        public static final String mysql = "1";
        public static final String redis = "2";
        public static final String es = "3";
        public static final String es = "1";
        public static final String mysql = "2";
    }
    public static class convert{
        public static String level_age_1="1";
        public static String level_age_2="2";
        public static String level_age_3="3";
        public static String level_age_4="4";
        public static String level_age_5="5";
        public static String level_age_6="6";
        public static String level_age_1_name="0~6";
        public static String level_age_2_name="7~18";
        public static String level_age_3_name="19~30";
        public static String level_age_4_name="31~50";
        public static String level_age_5_name="51~65";
        public static String level_age_6_name=">65";
    public static class convert {
        public static String level_age_1 = "1";
        public static String level_age_2 = "2";
        public static String level_age_3 = "3";
        public static String level_age_4 = "4";
        public static String level_age_5 = "5";
        public static String level_age_6 = "6";
        public static String level_age_1_name = "0~6";
        public static String level_age_2_name = "7~18";
        public static String level_age_3_name = "19~30";
        public static String level_age_4_name = "31~50";
        public static String level_age_5_name = "51~65";
        public static String level_age_6_name = ">65";
    }
    /**
     * 数据设置
     * @param quotaDimensionSlaves
     * @param key
     * @return
     */
    public static List<QuotaDimensionSlaveVO> init(List<TjQuotaDimensionSlave> quotaDimensionSlaves, String key) {
        List<QuotaDimensionSlaveVO> returnData = new ArrayList<>();
        if(Contant.slave_dimension_key.one.equals(key)){
            quotaDimensionSlaves.stream().forEach(one->{
                QuotaDimensionSlaveVO q=new QuotaDimensionSlaveVO();
                BeanUtils.copyProperties(one,q);
                q.setDictSql(one.getGetOneDictSql());
                q.setConvertClazz(one.getGetOneConvertClazz());
                returnData.add(q);
            });
        }else if(Contant.slave_dimension_key.two.equals(key)){
            quotaDimensionSlaves.stream().forEach(one->{
                QuotaDimensionSlaveVO q=new QuotaDimensionSlaveVO();
                BeanUtils.copyProperties(one,q);
                q.setDictSql(one.getGetTwoDictSql());
                q.setConvertClazz(one.getGetTwoConvertClazz());
                returnData.add(q);
            });
            return returnData;
        }
        return returnData;
    }
}

svr/svr-quota/src/main/java/com/yihu/jw/quota/etl/compute/ComputeHelper.java

@@ -1,113 +0,0 @@
package com.yihu.jw.quota.etl.compute;
import com.mchange.lang.IntegerUtils;
import com.yihu.jw.quota.etl.Contant;
import com.yihu.jw.quota.etl.compute.main.AreaCompute;
import com.yihu.jw.quota.etl.compute.slave.SlaveCompute;
import com.yihu.jw.quota.model.jpa.dimension.TjQuotaDimensionMain;
import com.yihu.jw.quota.model.jpa.dimension.TjQuotaDimensionSlave;
import com.yihu.jw.quota.service.dimension.TjDimensionMainService;
import com.yihu.jw.quota.service.dimension.TjDimensionSlaveService;
import com.yihu.jw.quota.util.SpringUtil;
import com.yihu.jw.quota.vo.*;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
 * Created by chenweida on 2017/6/1.
 */
@Component
@Scope("prototype")
public class ComputeHelper {
    @Autowired
    private TjDimensionMainService dimensionMainService;
    @Autowired
    private TjDimensionSlaveService dimensionSlaveService;
    public List<SaveModel> compute(Object dataModels, QuotaVO quotaVO, String timeLevel, String saasid, String startTime) {
        //返回的list
        List<SaveModel> saveModels = new ArrayList<>();
        //只得到地址维度的所有主维度
        List<TjQuotaDimensionMain> quotaDimensionMains = dimensionMainService.findTjQuotaDimensionMainByQuotaIncudeAddress(quotaVO.getCode());
        //得到从维度
        List<TjQuotaDimensionSlave> quotaDimensionSlaves = dimensionSlaveService.findTjQuotaDimensionSlaveByQuotaCode(quotaVO.getCode());
        if (Contant.compute.add.equals(quotaVO.getComputeType())) {
            //判断是否是累加计算 如果强转成list类型
            List<DataModel> dmList = (List<DataModel>) dataModels;
            ;
            Map<String, MainDimensionModel> one = getLastData(quotaVO, quotaDimensionMains,Contant. init(quotaDimensionSlaves, Contant.slave_dimension_key.one), dmList);
            for (Map.Entry<String, MainDimensionModel> tjone : one.entrySet()) {
                SaveModel saveModel = new SaveModel();
                BeanUtils.copyProperties(tjone.getValue(), saveModel);
                saveModel.setTimeLevel(timeLevel);
                saveModel.setSaasId(saasid);
                saveModel.setQuotaDate(startTime);
                saveModel.setResult(tjone.getValue().getSlaveDimensionModels().size() );
                saveModels.add(saveModel);
            }
        } else if (Contant.compute.division.equals(quotaVO.getComputeType())) {
            //判断是否是除法计算 如果强转成Map类型 key在Contant中
            Map<String, List<DataModel>> dmMap = (Map<String, List<DataModel>>) dataModels;
            List<DataModel> oneList = dmMap.get(Contant.extract.computeKey1);
            List<DataModel> twoList = dmMap.get(Contant.extract.computeKey2);
            //计算出分子
            Map<String, MainDimensionModel> one = getLastData(quotaVO, quotaDimensionMains, Contant. init(quotaDimensionSlaves, Contant.slave_dimension_key.one), oneList);
            //计算出分母
            Map<String, MainDimensionModel> tow = getLastData(quotaVO, quotaDimensionMains, Contant. init(quotaDimensionSlaves, Contant.slave_dimension_key.two), twoList);
            for (Map.Entry<String, MainDimensionModel> tjone : one.entrySet()) {
                SaveModel saveModel = new SaveModel();
                BeanUtils.copyProperties(tjone.getValue(), saveModel);
                saveModel.setTimeLevel(timeLevel);
                saveModel.setSaasId(saasid);
                saveModel.setQuotaDate(startTime);
                //如果是2位数的计算 多保存分子和分母
                Integer oneResult=tjone.getValue().getSlaveDimensionModels().size();
                Integer twoResult=tow.get(tjone.getKey()).getSlaveDimensionModels().size();
                //为了保持数据的格式统一是integer 所以这边默认保留2位乘以100
                DecimalFormat df=new DecimalFormat("0.00");
                String tempResult=df.format((double)oneResult/twoResult);
                Integer intResult= new Double((Double.valueOf(tempResult)*100)).intValue();
                saveModel.setResult(intResult);
                saveModel.setOne(oneResult);
                saveModel.setTwo(twoResult);
                saveModels.add(saveModel);
            }
        }
        return saveModels;
    }
    private Map<String, MainDimensionModel> getLastData(QuotaVO quotaVO, List<TjQuotaDimensionMain> quotaDimensionMains, List<QuotaDimensionSlaveVO> quotaDimensionSlaves, List<DataModel> dmList) {
        Map<String, MainDimensionModel> mainData = new HashMap<>();
        //先统计主维度的数据-----地址维度
        quotaDimensionMains.stream().forEach(one -> {
            mainData.putAll(SpringUtil.getBean(AreaCompute.class).compute(
                    quotaVO,
                    dmList,//数据
                    one.getDitcSql(), //字典的sql语句
                    Contant.main_dimension_areaLevel.getAreaLevelByMainDimension(one.getType()) //维度转换
            ));
        });
        //再统计从维度的数据
        return SpringUtil.getBean(SlaveCompute.class).compute(
                mainData,//数据
                quotaDimensionSlaves //字典的sql语句
        );
    }
}

svr/svr-quota/src/main/java/com/yihu/jw/quota/etl/compute/main/AreaCompute.java

@@ -1,135 +0,0 @@
package com.yihu.jw.quota.etl.compute.main;
import com.yihu.jw.quota.etl.Contant;
import com.yihu.jw.quota.model.jpa.compute.TjCompute;
import com.yihu.jw.quota.vo.*;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Scope;
import org.springframework.jdbc.core.BeanPropertyRowMapper;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
 * Created by chenweida on 2017/6/1.
 */
@Component
@Scope("prototype")
public class AreaCompute {
    @Autowired
    private JdbcTemplate jdbcTemplate;
    /**
     *
     * @param dmList 需要统计的元数据
     * @param sql 查询字典的sql语句
     * @param areaLevel 可能是 1 省 2 市  3区  4机构 5团队
     * @return
     */
    public Map<String, MainDimensionModel> compute(QuotaVO quotaVO,List<DataModel> dmList, String sql, String areaLevel) {
        if(StringUtils.isEmpty(sql)){
            return new HashMap<>(); //没有字典返回null
        }
        //得到字典
        List<DictModel> quotaDataSources = jdbcTemplate.query(sql, new BeanPropertyRowMapper(DictModel.class));
        //把字典转成Map
        Map<String,String> dictModelMap=new HashMap<>();
        quotaDataSources.stream().forEach(one->{
            dictModelMap.put(one.getCode(),one.getName());
        });
        Map<String, MainDimensionModel> datas = new HashMap<>();
        //设置全部key到map中
        quotaDataSources.stream().forEach(one -> {
            datas.put(one.getCode(),null);
        });
        //只是用来是设置为空的时候填充上级
       final MainDimensionModel mainDimensionModelTemp=new MainDimensionModel();
        //数据根据key分组
        dmList.stream().forEach(one->{
            MainDimensionModel mainDimensionModel= datas.get(one.getKey(areaLevel));
            if(mainDimensionModel==null){
                mainDimensionModel=new MainDimensionModel();
                mainDimensionModel.setAreaLevel(areaLevel);
                mainDimensionModel.setProvince(one.getProvince());
                mainDimensionModel.setCity(one.getCity());
                mainDimensionModel.setTown(one.getTown());
                mainDimensionModel.setHospital(one.getHospital());
                mainDimensionModel.setTeam(one.getTeam());
                mainDimensionModel.setQuotaCode(quotaVO.getCode());
                //设置名称
                switch (areaLevel){
                    case Contant.main_dimension_areaLevel.area_province:{mainDimensionModel.setProvinceName(dictModelMap.get(one.getProvince()));break;}
                    case Contant.main_dimension_areaLevel.area_city:{mainDimensionModel.setCityName(dictModelMap.get(one.getCity()));break;}
                    case Contant.main_dimension_areaLevel.area_town:{mainDimensionModel.setTownName(dictModelMap.get(one.getTown()));break;}
                    case Contant.main_dimension_areaLevel.area_org:{mainDimensionModel.setHospitalName(dictModelMap.get(one.getHospital()));break;}
                    case Contant.main_dimension_areaLevel.area_team:{mainDimensionModel.setTeamName(dictModelMap.get(one.getTeam()));break;}
                }
                BeanUtils.copyProperties(mainDimensionModel,mainDimensionModelTemp);
            }
            //设置从维度
            mainDimensionModel.getSlaveDimensionModels().add(new SlaveDimensionModel(
                    one.getSlaveKey1(),
                    one.getSlaveKey2(),
                    one.getSlaveKey3(),
                    one.getSlaveKey4()
            ));
            datas.put(one.getKey(areaLevel),mainDimensionModel);
        });
        //如果是空的设置初始值
        initNull(quotaVO, areaLevel, quotaDataSources, datas, mainDimensionModelTemp);
        return datas;
    }
    private void initNull(QuotaVO quotaVO, String areaLevel, List<DictModel> quotaDataSources, Map<String, MainDimensionModel> datas, MainDimensionModel mainDimensionModelTemp) {
        quotaDataSources.stream().forEach(one -> {
            MainDimensionModel mainDimensionModel= datas.get(one.getCode());
            if(mainDimensionModel==null){
                mainDimensionModel=new MainDimensionModel();
                mainDimensionModel.setAreaLevel(areaLevel);
                mainDimensionModel.setQuotaCode(quotaVO.getCode());
                //设置名称
                switch (areaLevel){
                    case Contant.main_dimension_areaLevel.area_province:{
                        mainDimensionModel.setProvince(one.getCode());
                        mainDimensionModel.setProvinceName(one.getName());break;
                    }
                    case Contant.main_dimension_areaLevel.area_city:{
                        mainDimensionModel.setProvince(mainDimensionModelTemp.getProvince());
                        mainDimensionModel.setCity(one.getCode());
                        mainDimensionModel.setCityName(one.getName());break;
                    }
                    case Contant.main_dimension_areaLevel.area_town:{
                        mainDimensionModel.setProvince(mainDimensionModelTemp.getProvince());
                        mainDimensionModel.setCity(mainDimensionModelTemp.getCity());
                        mainDimensionModel.setTown(one.getCode());
                        mainDimensionModel.setTownName(one.getName());break;
                    }
                    case Contant.main_dimension_areaLevel.area_org:{
                        mainDimensionModel.setProvince(mainDimensionModelTemp.getProvince());
                        mainDimensionModel.setCity(mainDimensionModelTemp.getCity());
                        mainDimensionModel.setTown(mainDimensionModelTemp.getTown());
                        mainDimensionModel.setHospital(one.getCode());
                        mainDimensionModel.setHospitalName(one.getName());break;
                    }
                    case Contant.main_dimension_areaLevel.area_team:{
                        mainDimensionModel.setProvince(mainDimensionModelTemp.getProvince());
                        mainDimensionModel.setCity(mainDimensionModelTemp.getCity());
                        mainDimensionModel.setTown(mainDimensionModelTemp.getTown());
                        mainDimensionModel.setHospital(mainDimensionModelTemp.getHospital());
                        mainDimensionModel.setTeam(one.getCode());
                        mainDimensionModel.setTeamName(one.getName());break;
                    }
                }
                datas.put(one.getCode(),mainDimensionModel);
            }
        });
    }
}

svr/svr-quota/src/main/java/com/yihu/jw/quota/etl/compute/slave/SlaveCompute.java

@@ -1,98 +0,0 @@
package com.yihu.jw.quota.etl.compute.slave;
import com.yihu.jw.quota.model.jpa.dimension.TjQuotaDimensionSlave;
import com.yihu.jw.quota.vo.DictModel;
import com.yihu.jw.quota.vo.MainDimensionModel;
import com.yihu.jw.quota.vo.QuotaDimensionSlaveVO;
import com.yihu.jw.quota.vo.SlaveDimensionModel;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Scope;
import org.springframework.jdbc.core.BeanPropertyRowMapper;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
 * Created by chenweida on 2017/6/1.
 * 从维度计算的类
 */
@Component
@Scope("prototype")
public class SlaveCompute {
    private Logger logger= LoggerFactory.getLogger(SlaveCompute.class);
    @Autowired
    private JdbcTemplate jdbcTemplate;
    public Map<String, MainDimensionModel> compute(Map<String, MainDimensionModel> mainData,  List<QuotaDimensionSlaveVO> quotaDimensionSlaves) {
        try {
            //找出所有得到字典放到里面List里面去
            List<Map<String, String>> dictList = new ArrayList<>();
            for (int i = 0; i < quotaDimensionSlaves.size(); i++) {
                Map<String, String> dictMap = new HashMap<>();
                QuotaDimensionSlaveVO tjQuotaDimensionSlave= quotaDimensionSlaves.get(i);
                String dictSql=tjQuotaDimensionSlave.getDictSql();
                if(StringUtils.isEmpty(dictSql)){
                    logger.warn("slaveDict sql is null ,quotaID:"+tjQuotaDimensionSlave.getQuotaCode()+",quotaSlaveId:"+tjQuotaDimensionSlave.getId());
                }
                List<DictModel> quotaDataSources = jdbcTemplate.query(dictSql, new BeanPropertyRowMapper(DictModel.class));
                quotaDataSources.stream().forEach(one -> {
                    dictMap.put(one.getCode(), one.getName());
                });
                dictList.add(dictMap);
            }
            //递归遍历从维度
            for (int i = 0; i < quotaDimensionSlaves.size(); i++) {
                Map<String, MainDimensionModel> temp = new HashMap<>();
                //得到维度对应的字典
                Map<String, String> dict = dictList.get(i);
                //遍历元数据
                for (Map.Entry<String, MainDimensionModel> oneMainDimensionModel : mainData.entrySet()) {
                    //遍历字典
                    for (Map.Entry<String, String> oneDict : dict.entrySet()) {
                        List<SlaveDimensionModel> slaveDimensionModels = oneMainDimensionModel.getValue().getSlaveDimensionModels();
                        //生成新的key
                        String key = new StringBuffer(oneMainDimensionModel.getKey() + "-" + oneDict.getKey()).toString();
                        List<SlaveDimensionModel> slaveDimensionModelTemps = new ArrayList<>();
                        MainDimensionModel mainDimensionModelTemp = new MainDimensionModel();
                        BeanUtils.copyProperties(oneMainDimensionModel.getValue(), mainDimensionModelTemp);
                        for (int j = 0; j < slaveDimensionModels.size(); j++) {
                            SlaveDimensionModel oneSlaveDimensionModel = slaveDimensionModels.get(j);
                            SlaveDimensionModel slaveDimensionModelTemp = new SlaveDimensionModel();
                            BeanUtils.copyProperties(oneSlaveDimensionModel, slaveDimensionModelTemp);
                            //通过反射动态调用设置
                            String oneSlaveDimensionModelKey = (String) SlaveDimensionModel.class.getMethod(new StringBuffer("getSlaveKey" + (i+1)).toString()).invoke(oneSlaveDimensionModel);
                            if (oneDict.getKey().equals(oneSlaveDimensionModelKey)) {
                                slaveDimensionModelTemps.add(slaveDimensionModelTemp);
                            }
                        }
                        MainDimensionModel.class.getMethod(new StringBuffer("setSlaveKey"+ (i+1)).toString(),String.class).invoke(mainDimensionModelTemp,oneDict.getKey());
                        MainDimensionModel.class.getMethod(new StringBuffer("setSlaveKey"+ (i+1)+"Name").toString(),String.class).invoke(mainDimensionModelTemp,oneDict.getValue());
                        mainDimensionModelTemp.setSlaveDimensionModels(slaveDimensionModelTemps);
                        temp.put(key, mainDimensionModelTemp);
                    }
                }
                mainData = temp;
            }
            return mainData;
        }catch (Exception e){
            e.printStackTrace();
        }
        return null;
    }
}

svr/svr-quota/src/main/java/com/yihu/jw/quota/etl/dimenconvert/Convert.java

@@ -1,12 +0,0 @@
package com.yihu.jw.quota.etl.dimenconvert;
import com.yihu.jw.quota.vo.DataModel;
import java.util.List;
/**
 * Created by chenweida on 2017/6/5.
 */
public interface Convert {
    public List<DataModel> convert(List<DataModel> oneList, String slaveLevel);
}

svr/svr-quota/src/main/java/com/yihu/jw/quota/etl/dimenconvert/ConvertHelper.java

@@ -1,73 +0,0 @@
package com.yihu.jw.quota.etl.dimenconvert;
import com.yihu.jw.quota.etl.Contant;
import com.yihu.jw.quota.model.jpa.dimension.TjQuotaDimensionSlave;
import com.yihu.jw.quota.service.dimension.TjDimensionSlaveService;
import com.yihu.jw.quota.util.SpringUtil;
import com.yihu.jw.quota.vo.DataModel;
import com.yihu.jw.quota.vo.FilterModel;
import com.yihu.jw.quota.vo.QuotaDimensionSlaveVO;
import com.yihu.jw.quota.vo.QuotaVO;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.propertyeditors.ClassArrayEditor;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;
import java.lang.reflect.Method;
import java.util.List;
import java.util.Map;
/**
 * Created by chenweida on 2017/6/5.
 * 维度的key值转换器
 */
@Component
@Scope("prototype")
public class ConvertHelper {
    @Autowired
    private TjDimensionSlaveService dimensionSlaveService;
    public FilterModel convert(FilterModel filterModel, QuotaVO quotaVO) throws Exception {
        //得到从维度
        List<TjQuotaDimensionSlave> tj = dimensionSlaveService.findTjQuotaDimensionSlaveByQuotaCode(quotaVO.getCode());
        for (int i = 0; i < tj.size(); i++) {
            TjQuotaDimensionSlave temp = tj.get(i);
            if (!StringUtils.isEmpty(temp.getConvertClazz())) {
                if (Contant.compute.add.equals(quotaVO.getComputeType())) {
                    //判断是否是累加计算 如果强转成list类型
                    List<DataModel> oneList = (List<DataModel>) filterModel.getData();
                    List<QuotaDimensionSlaveVO> oneQuotaDimensionSlaveVOList = Contant.init(tj, Contant.slave_dimension_key.one);
                    String clazz=oneQuotaDimensionSlaveVOList.get(i).getConvertClazz();
                    if(!StringUtils.isEmpty(clazz)){
                        //反射出对象并且调用convert方法去转换对应的slavekey
                        Object obj=Class.forName(clazz).newInstance();
                        Method method=obj.getClass().getMethod("convert",List.class,String.class);
                        method.invoke(obj,oneList,String.valueOf(i+1));
                    }
                } else if (Contant.compute.division.equals(quotaVO.getComputeType())) {
                    //判断是否是除法计算 如果强转成Map类型 key在Contant中
                    Map<String, List<DataModel>> dmMap = (Map<String, List<DataModel>>) filterModel.getData();
                    List<DataModel> oneList = dmMap.get(Contant.extract.computeKey1);
                    List<DataModel> twoList = dmMap.get(Contant.extract.computeKey2);
                    List<QuotaDimensionSlaveVO> oneQuotaDimensionSlaveVOList = Contant.init(tj, Contant.slave_dimension_key.one);
                    String clazzOne=oneQuotaDimensionSlaveVOList.get(i).getConvertClazz();
                    if(!StringUtils.isEmpty(clazzOne)){
                        Object obj=Class.forName(clazzOne);
                        Convert.class.getMethod("convert",List.class,String.class).invoke(obj,oneList,String.valueOf(i+1));
                    }
                    dmMap.put(Contant.extract.computeKey1,oneList);
                    List<QuotaDimensionSlaveVO> twoQuotaDimensionSlaveVOList = Contant.init(tj, Contant.slave_dimension_key.two);
                    String clazzTwo=twoQuotaDimensionSlaveVOList.get(i).getConvertClazz();
                    if(!StringUtils.isEmpty(clazzTwo)){
                        Object obj=Class.forName(clazzTwo);
                        Convert.class.getMethod("convert",List.class,String.class).invoke(obj,twoList,String.valueOf(i+1));
                        dmMap.put(Contant.extract.computeKey2,twoList);
                    }
                }
            }
        }
        return filterModel;
    }
}

svr/svr-quota/src/main/java/com/yihu/jw/quota/etl/dimenconvert/wlyy/AgeConvert.java

@@ -1,51 +0,0 @@
package com.yihu.jw.quota.etl.dimenconvert.wlyy;
import com.yihu.jw.quota.etl.Contant;
import com.yihu.jw.quota.etl.dimenconvert.Convert;
import com.yihu.jw.quota.util.IdCardUtil;
import com.yihu.jw.quota.vo.DataModel;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
import java.util.List;
/**
 * Created by chenweida on 2017/6/5.
 */
public class AgeConvert implements Convert {
    /**
     * @param oneList    数据
     * @param slaveLevel 从1开始
     * @return
     */
    public List<DataModel> convert(List oneList, String slaveLevel) {
        oneList.stream().forEach(one -> {
            try {
                Object value = DataModel.class.getMethod("getSlaveKey" + slaveLevel).invoke(one);
                Integer age= IdCardUtil.getAgeForIdcard(String.valueOf(value));
                String key = getAgeCode(age);
                DataModel.class.getMethod("setSlaveKey" + slaveLevel, String.class).invoke(one, key);
            } catch (Exception e) {
                e.printStackTrace();
            }
        });
        return oneList;
    }
    public String getAgeCode(Integer age) {
        if (age <= 6) {
            return Contant.convert.level_age_1;
        } else if (age >= 7 && age <= 18) {
            return Contant.convert.level_age_2;
        } else if (age >= 19 && age <= 30) {
            return Contant.convert.level_age_3;
        } else if (age >= 31 && age <= 50) {
            return Contant.convert.level_age_4;
        } else if (age >= 51 && age <= 65) {
            return Contant.convert.level_age_5;
        } else {
            return Contant.convert.level_age_6;
        }
    }
}

svr/svr-quota/src/main/java/com/yihu/jw/quota/etl/extract/ExtractHelper.java

@@ -1,28 +1,25 @@
package com.yihu.jw.quota.etl.extract;
import com.yihu.jw.quota.etl.Contant;
import com.yihu.jw.quota.etl.extract.db.DBExtract;
import com.yihu.jw.quota.etl.model.DbConfig;
import com.yihu.jw.quota.etl.extract.es.EsExtract;
import com.yihu.jw.quota.etl.model.EsConfig;
import com.yihu.jw.quota.model.jpa.dimension.TjQuotaDimensionMain;
import com.yihu.jw.quota.model.jpa.dimension.TjQuotaDimensionSlave;
import com.yihu.jw.quota.model.jpa.source.TjDataSource;
import com.yihu.jw.quota.model.jpa.source.TjQuotaDataSource;
import com.yihu.jw.quota.service.dimension.TjDimensionMainService;
import com.yihu.jw.quota.service.dimension.TjDimensionSlaveService;
import com.yihu.jw.quota.service.source.TjDataSourceService;
import com.yihu.jw.quota.util.SpringUtil;
import com.yihu.jw.quota.vo.DataModel;
import com.yihu.jw.quota.vo.QuotaVO;
import com.yihu.jw.quota.vo.SaveModel;
import net.sf.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.jdbc.DataSourceBuilder;
import org.springframework.context.annotation.Scope;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;
import javax.sql.DataSource;
import java.util.HashMap;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
 * Created by chenweida on 2017/6/1.
@@ -30,10 +27,12 @@ import java.util.Map;
@Component
@Scope("prototype")
public class ExtractHelper {
    private String startTime;
    private String endTime;
    @Autowired
    private TjDataSourceService dataSourceService;
    @Autowired
    private TjDimensionMainService dimensionMainService;
    @Autowired
    private TjDimensionSlaveService dimensionSlaveService;
    private Logger logger = LoggerFactory.getLogger(ExtractHelper.class);
@@ -44,11 +43,8 @@ public class ExtractHelper {
     * @return
     * @throws Exception
     */
    public Object extractData(QuotaVO quotaVO, String startTime, String endTime) throws Exception {
    public List<SaveModel> extractData(QuotaVO quotaVO, String startTime, String endTime,String timeLevel,String saasid) throws Exception {
        try {
            this.startTime = startTime;
            this.endTime = endTime;
            //得到该指标的数据来源
            TjQuotaDataSource quotaDataSource = dataSourceService.findSourceByQuotaCode(quotaVO.getCode());
            //如果为空说明数据错误
@@ -56,44 +52,19 @@ public class ExtractHelper {
                throw new Exception("QuotaDataSource data error");
            }
            //判断数据源是什么类型,根据类型和数据库相关的配置信息抽取数据
            if (TjDataSource.type_mysql.equals(quotaDataSource.getType())) {
                //获取数据库配置
                DbConfig dbConfig = (DbConfig) JSONObject.toBean(JSONObject.fromObject(quotaDataSource.getConfigJson()), DbConfig.class);
                //通过配置得到数据库链接
                JdbcTemplate jdbcTemplate = initJdbcTemplate(dbConfig);
                //设置时间
                String sql = initSql(dbConfig.getSql());
                String sqlCount = initSql(dbConfig.getSqlCount());
                //抽取数据
                List<DataModel> ds1 = SpringUtil.getBean(DBExtract.class).extractByPage(
                        DataModel.class,
                        sql,
                        sqlCount,
                        true,
                        jdbcTemplate);
                if (StringUtils.isEmpty(dbConfig.getSecondSql())) {
                    return ds1;
                } else {
                    //设置时间
                    String secondSql = initSql(dbConfig.getSecondSql());
                    String secondSqlCount = initSql(dbConfig.getSecondSqlCount());
                    //抽取数据
                    List<DataModel> ds2 = SpringUtil.getBean(DBExtract.class).extractByPage(DataModel.class,
                            secondSql,
                            secondSqlCount,
                            true,
                            jdbcTemplate);
                    Map<String, List<DataModel>> params = new HashMap<String, List<DataModel>>();
                    params.put(Contant.extract.computeKey1, ds1);
                    params.put(Contant.extract.computeKey2, ds2);
                    return params;
                }
            } else if (TjDataSource.type_redis.equals(quotaDataSource.getType())) {
            if (TjDataSource.type_es.equals(quotaDataSource.getType())) {
                JSONObject obj = new JSONObject().fromObject(quotaDataSource.getConfigJson());
                EsConfig esConfig= (EsConfig) JSONObject.toBean(obj,EsConfig.class);
                //得到主维度
                List<TjQuotaDimensionMain> tjQuotaDimensionMains = dimensionMainService.findTjQuotaDimensionMainByQuotaIncudeAddress(quotaDataSource.getQuotaCode());
                //得到细维度
                List<TjQuotaDimensionSlave> tjQuotaDimensionSlaves = dimensionSlaveService.findTjQuotaDimensionSlaveByQuotaCode(quotaDataSource.getQuotaCode());
                //查询ES数据
                return  SpringUtil.getBean(EsExtract.class).extract(tjQuotaDimensionMains,tjQuotaDimensionSlaves,startTime,endTime,timeLevel,saasid,quotaVO,esConfig);
            }
        } catch (Exception e) {
            e.printStackTrace();
            logger.error("extract error:" + e.getMessage());
            logger.error("quotaVOr:" + quotaVO.toString());
@@ -101,39 +72,6 @@ public class ExtractHelper {
        return null;
    }
    /**
     * 初始化sql
     * @param sql 替换时间
     * @return
     */
    private String initSql(String sql) {
        try {
            if (sql.contains(Contant.endTime)) {
                sql = sql.replace(Contant.endTime, endTime);
            }
            if (sql.contains(Contant.startTime)) {
                sql = sql.replace(Contant.startTime, startTime);
            }
            logger.info("sql init :"+sql);
        }catch (Exception e){
        }
        return sql;
    }
    /**
     * 初始化数据库的链接
     *
     * @param dbConfig
     * @return
     */
    private JdbcTemplate initJdbcTemplate(DbConfig dbConfig) {
        //初始化数据库链接
        DataSource ds = DataSourceBuilder.create().
                url(dbConfig.getUrl()).
                username(dbConfig.getUsername()).
                driverClassName(dbConfig.getDriver()).
                password(dbConfig.getPassword()).build();
        return new JdbcTemplate(ds);
    }
}

svr/svr-quota/src/main/java/com/yihu/jw/quota/etl/extract/db/DBExtract.java

@@ -1,70 +0,0 @@
package com.yihu.jw.quota.etl.extract.db;
import com.yihu.jw.quota.util.SpringUtil;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.Scope;
import org.springframework.jdbc.core.BeanPropertyRowMapper;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;
import javax.sql.DataSource;
import java.util.List;
/**
 * Created by Administrator on 2016.10.16.
 * 数据库抽取
 */
@Component
@Scope("prototype")
public class DBExtract<T> {
    /**
     * 不分页抽取
     * @param clazz
     * @param sql
     * @return
     */
    public List<T> extract(Class<T> clazz,String sql,JdbcTemplate jdbcTemplate){
        List<T> returnList= jdbcTemplate.query(sql,new BeanPropertyRowMapper(clazz));
        return returnList;
    }
    /**
     * 分页抽取
     * @param clazz
     * @param sql
     * @param pageSize 每页显示多少 默认10000
     * @param isMultithreading 是否多线程抽取 默认否
     * @return
     */
    public List<T> extractByPage(Class<T> clazz,String sql,String countSql,int pageSize,Boolean isMultithreading,JdbcTemplate jdbcTemplate)throws  Exception{
        return SpringUtil.getBean(DBPageExtract.class).extractByPage(clazz,sql,countSql,pageSize,isMultithreading,jdbcTemplate);
    }
    /**
     * 分页抽取
     * @param clazz
     * @param sql
     * @param isMultithreading 是否多线程抽取 默认否
     * @return
     */
    public List<T> extractByPage(Class<T> clazz,String sql,String countSql,Boolean isMultithreading,JdbcTemplate jdbcTemplate)throws  Exception{
        if(StringUtils.isEmpty(countSql)){
            return extract(clazz,sql,jdbcTemplate);
        }
        int pageSize=10000;
        return SpringUtil.getBean(DBPageExtract.class).extractByPage(clazz,sql,countSql,pageSize,isMultithreading,jdbcTemplate);
    }
    /**
     * 分页抽取
     * @param clazz
     * @param sql
     * @return
     */
    public List<Object> extractByPage(Class<T> clazz,String sql,JdbcTemplate jdbcTemplate)throws  Exception{
        int pageSize=10000;
        Boolean isMultithreading=false;
        String countSql="";
        return SpringUtil.getBean(DBPageExtract.class).extractByPage(clazz,sql,countSql,pageSize,isMultithreading,jdbcTemplate);
    }
}

svr/svr-quota/src/main/java/com/yihu/jw/quota/etl/extract/db/DBPageExtract.java

@@ -1,162 +0,0 @@
package com.yihu.jw.quota.etl.extract.db;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.Scope;
import org.springframework.jdbc.core.BeanPropertyRowMapper;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.scheduling.annotation.Async;
import org.springframework.scheduling.annotation.AsyncResult;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;
import javax.annotation.Resource;
import javax.sql.DataSource;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
/**
 * Created by Administrator on 2016.10.16.
 * 分页抽取器
 */
@Component
@Scope("prototype")
public class DBPageExtract<T> {
    private  List<T> returnList=new ArrayList<T>();
    public synchronized void addData( List<T> dataList) {
         returnList.addAll(dataList);
    }
    public List<T> extractByPage(Class<T> clazz, String sql, String countSql, int pageSize, Boolean isMultithreading,JdbcTemplate jdbcTemplate) throws Exception{
        if(!StringUtils.isEmpty(countSql)){
            isMultithreading=true;
        }
        if(isMultithreading){
            if(StringUtils.isEmpty(countSql)){
                throw new Exception("countSql is null");
            }
            return MultiThreadExtract(clazz, sql,countSql, pageSize,jdbcTemplate);
        }else{
            return noMultiThreadExtract(clazz, sql, pageSize,jdbcTemplate);
        }
    }
    /**
     * 不用多线程抽取
     * @param clazz
     * @param sql
     * @param pageSize
     * @return
     */
    private List<T> noMultiThreadExtract(Class<T> clazz, String sql, int pageSize,JdbcTemplate jdbcTemplate) {
        List<T> returnList=new ArrayList<T>();
        int start=0;
        int page=1;
        while (true){
            String finalSql=sql+" limit "+start+","+pageSize; //拼凑分页的语句
            List<T> listTemp= jdbcTemplate.query(finalSql,new BeanPropertyRowMapper(clazz));
            returnList.addAll(listTemp);//添加到list里面
            //判断是都是最后页面
            if(listTemp.size()!=pageSize){
                listTemp.clear();
                break;
            }else{
                start =page*pageSize;
                page++;
                listTemp.clear();
            }
        }
        return returnList;
    }
    /**
     * 多线程抽取数据
     * @param clazz
     * @param sql
     * @param pageSize
     * @return
     */
    private   List<T> MultiThreadExtract(Class<T> clazz, String sql,String countSql, int pageSize,JdbcTemplate jdbcTemplate)  {
        try{
            //得到数据的总数
            Integer dataCount=getCount(countSql,jdbcTemplate);
            //根据count 计算出 总共要执行几次
            Integer forCount=getForCount(dataCount,pageSize);
            forCount++;
            //綫程返回值的值
            List<AsyncResult<Boolean>> asyncResultPool=new ArrayList<AsyncResult<Boolean>>();
            for(int page=0;page<forCount;page++){
                //启动多线程采集数据
                AsyncResult<Boolean> future= multiExtractData(sql,page*pageSize,pageSize,clazz,jdbcTemplate);
                asyncResultPool.add(future);
            }
            ///循环判断，等待获取结果信息
            while (true){
                //得到迭代器
                Iterator<AsyncResult<Boolean>> asyncResultIterator=asyncResultPool.iterator();
                //判断有没有下一个
                while (asyncResultIterator.hasNext()){
                    AsyncResult<Boolean> asyncResult= asyncResultIterator.next();
                    if(asyncResult.isDone()){
                        //如果做完了就移除迭代器
                        asyncResultIterator.remove();
                    }else{
                        Thread.sleep(500L);
                    }
                }
                //如果一直移除到没有下一个就跳出循环 说明数据已经采集完成
                if(!asyncResultIterator.hasNext()){
                    break;
                }
            }
            return returnList;
        }catch (Exception e){
            //如果多线程错误 就改用单线程
            return noMultiThreadExtract(clazz,sql,pageSize,jdbcTemplate);
        }
    }
    private Integer getForCount(Integer dataCount, int pageSize) {
        //根据所有的数据取余数
        int yushu=dataCount%pageSize;
        int page=dataCount/pageSize;
        if(yushu==0){
            //如果没有余数 返回总的页数
            return page;
        }else{
            //如果有余数 页数多加1
            page++;
            return page;
        }
    }
    private Integer getCount(String countSql,JdbcTemplate jdbcTemplate) {
        Integer countMap= jdbcTemplate.queryForObject(countSql,Integer.class);
        return countMap;
    }
    /**
     * 多线程采集数据
     * @param sql
     * @param start
     * @param pageSize
     * @param clazz
     * @return
     */
    @Async("dbExtractExecutor")
    private AsyncResult<Boolean> multiExtractData(String sql, int start, int pageSize, Class<T> clazz,JdbcTemplate jdbcTemplate) {
        String finalSql=sql+" limit "+start+","+pageSize; //拼凑分页的语句
        List<T> listTemp= jdbcTemplate.query(finalSql,new BeanPropertyRowMapper(clazz));
        addData(listTemp);
        return new AsyncResult<>(true);
    }
}

svr/svr-quota/src/main/java/com/yihu/jw/quota/etl/extract/es/EsExtract.java

@@ -0,0 +1,311 @@
package com.yihu.jw.quota.etl.extract.es;
import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.expr.SQLQueryExpr;
import com.alibaba.druid.sql.parser.ParserException;
import com.alibaba.druid.sql.parser.SQLExprParser;
import com.alibaba.druid.sql.parser.Token;
import com.yihu.jw.quota.etl.Contant;
import com.yihu.jw.quota.etl.model.EsConfig;
import com.yihu.jw.quota.etl.save.es.ElasticFactory;
import com.yihu.jw.quota.model.jpa.dimension.TjQuotaDimensionMain;
import com.yihu.jw.quota.model.jpa.dimension.TjQuotaDimensionSlave;
import com.yihu.jw.quota.vo.DictModel;
import com.yihu.jw.quota.vo.QuotaVO;
import com.yihu.jw.quota.vo.SaveModel;
import net.sf.json.JSONObject;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.collect.HppcMaps;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.InternalAggregations;
import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.bucket.terms.TermsBuilder;
import org.elasticsearch.search.aggregations.metrics.valuecount.InternalValueCount;
import org.nlpcn.es4sql.domain.Select;
import org.nlpcn.es4sql.parse.ElasticSqlExprParser;
import org.nlpcn.es4sql.parse.SqlParser;
import org.nlpcn.es4sql.query.AggregationQueryAction;
import org.nlpcn.es4sql.query.DefaultQueryAction;
import org.nlpcn.es4sql.query.SqlElasticSearchRequestBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Scope;
import org.springframework.jdbc.core.BeanPropertyRowMapper;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;
import java.util.*;
/**
 * Created by chenweida on 2017/6/6.
 */
@Component
@Scope("prototype")
public class EsExtract {
    private Logger logger = LoggerFactory.getLogger(EsExtract.class);
    @Autowired
    private ElasticFactory elasticFactory;
    @Autowired
    private JdbcTemplate jdbcTemplate;
    private String startTime;
    private String endTime;
    private String timeLevel;
    private String saasid;
    private QuotaVO quotaVO;
    private EsConfig esConfig;
    public List<SaveModel> extract(List<TjQuotaDimensionMain> qdm,//主维度
                                   List<TjQuotaDimensionSlave> qds,//细维度
                                   String startTime,//开始时间
                                   String endTime, //结束时间
                                   String timeLevel, //时间维度  1日,2 周, 3 月,4 年
                                   String saasid,//saasid
                                   QuotaVO quotaVO,//指标code
                                   EsConfig esConfig //es配置
    ) {
        this.startTime = startTime;
        this.endTime = endTime;
        this.timeLevel = timeLevel;
        this.saasid = saasid;
        this.quotaVO = quotaVO;
        this.esConfig = esConfig;
        //拼凑查询的sql
        Map<String, TjQuotaDimensionMain> sqls = getSql(qdm, qds);
        //根据sql查询ES
        return queryEsBySql(sqls, qds);
    }
    private Map<String, SaveModel> setAllSlaveData(Map<String, SaveModel> allData, List<DictModel> dictData) {
        try {
            Map<String, SaveModel> returnAllData = new HashMap<>();
            for (Map.Entry<String, SaveModel> one : allData.entrySet()) {
                for (int i = 0; i < dictData.size(); i++) {
                    DictModel dictOne = dictData.get(i);
                    //设置新key
                    StringBuffer newKey = new StringBuffer(one.getKey() + "-" + dictOne.getCode());
                    //设置新的value
                    SaveModel saveModelTemp = new SaveModel();
                    BeanUtils.copyProperties(one.getValue(), saveModelTemp);
                    StringBuffer keyMethodName = new StringBuffer("setSlaveKey" + (i + 1));
                    StringBuffer nameMethodName = new StringBuffer("setSlaveKey" + (i + 1) + "Name");
                    SaveModel.class.getMethod(keyMethodName.toString(), String.class).invoke(saveModelTemp, dictOne.getCode());
                    SaveModel.class.getMethod(nameMethodName.toString(), String.class).invoke(saveModelTemp, dictOne.getName());
                    returnAllData.put(newKey.toString(), saveModelTemp);
                }
            }
            return returnAllData;
        } catch (Exception e) {
        }
        return null;
    }
    /**
     * @param allData
     * @param dictData
     * @param dictType
     */
    private void setAllData(Map<String, SaveModel> allData, List<SaveModel> dictData, String dictType) {
        switch (dictType) {
            case Contant.main_dimension_areaLevel.area_province: {
                //设置省的全部的值
                dictData.stream().forEach(one -> {
                    //StringBuffer key = new StringBuffer(one.getProvince());
                    setOneData(allData, one.getProvince(), one, Contant.main_dimension_areaLevel.area_province);
                });
                break;
            }
            case Contant.main_dimension_areaLevel.area_city: {
                //设置市的全部的值
                dictData.stream().forEach(one -> {
                    //StringBuffer key = new StringBuffer(one.getProvince() + "-" + one.getCity());
                    setOneData(allData, one.getCity(), one, Contant.main_dimension_areaLevel.area_city);
                });
                break;
            }
            case Contant.main_dimension_areaLevel.area_town: {
                //设置区的全部的值
                dictData.stream().forEach(one -> {
                    //StringBuffer key = new StringBuffer(one.getProvince() + "-" + one.getCity() + "-" + one.getTown());
                    setOneData(allData, one.getTown(), one, Contant.main_dimension_areaLevel.area_town);
                });
                break;
            }
            case Contant.main_dimension_areaLevel.area_org: {
                //设置机构
                dictData.stream().forEach(one -> {
                    // StringBuffer key = new StringBuffer(one.getProvince() + "-" + one.getCity() + "-" + one.getTown() + "-" + one.getHospital());
                    setOneData(allData, one.getHospital(), one, Contant.main_dimension_areaLevel.area_org);
                });
                break;
            }
            case Contant.main_dimension_areaLevel.area_team: {
                //设置团队
                dictData.stream().forEach(one -> {
                    // StringBuffer key = new StringBuffer(one.getProvince() + "-" + one.getCity() + "-" + one.getTown() + "-" + one.getHospital() + "-" + one.getTeam());
                    setOneData(allData, one.getTeam(), one, Contant.main_dimension_areaLevel.area_team);
                });
                break;
            }
        }
    }
    private void setOneData(Map<String, SaveModel> allData, String key, SaveModel one, String areaLevel) {
        one.setAreaLevel(areaLevel);
        one.setResult(0);
        one.setCreateTime(new Date());
        one.setTimeLevel(timeLevel);
        one.setSaasId(saasid);
        allData.put(key, one);
    }
    private  List<SaveModel> queryEsBySql(Map<String, TjQuotaDimensionMain> sqls, List<TjQuotaDimensionSlave> tjQuotaDimensionSlaves) {
        List<SaveModel> returnList = new ArrayList<>();
        //初始化es链接
        esConfig = (EsConfig) JSONObject.toBean(JSONObject.fromObject(esConfig), EsConfig.class);
        //初始化链接
        Client client = elasticFactory.getClient(esConfig.getHost(), esConfig.getPort(), null);
        for (Map.Entry<String, TjQuotaDimensionMain> one : sqls.entrySet()) {
            logger.info("excute sql:" + one.getKey());
            try {
                SQLExprParser parser = new ElasticSqlExprParser(one.getKey());
                SQLExpr expr = parser.expr();
                if (parser.getLexer().token() != Token.EOF) {
                    throw new ParserException("illegal sql expr : " + one);
                }
                SQLQueryExpr queryExpr = (SQLQueryExpr) expr;
                //通过抽象语法树，封装成自定义的Select，包含了select、from、where group、limit等
                Select select = null;
                select = new SqlParser().parseSelect(queryExpr);
                AggregationQueryAction action = null;
                DefaultQueryAction queryAction = null;
                SqlElasticSearchRequestBuilder requestBuilder = null;
                if (select.isAgg) {
                    //包含计算的的排序分组的
                    action = new AggregationQueryAction(client, select);
                    requestBuilder = action.explain();
                } else {
                    //封装成自己的Select对象
                    queryAction = new DefaultQueryAction(client, select);
                    requestBuilder = queryAction.explain();
                }
                //之后就是对ES的操作
                SearchResponse response = (SearchResponse) requestBuilder.get();
                StringTerms stringTerms = (StringTerms) response.getAggregations().asList().get(0);
                Iterator<Terms.Bucket> gradeBucketIt = stringTerms.getBuckets().iterator();
                //里面存放的数据 例  350200-5-2-2    主维度  细维度1  细维度2  值
                Map<String,Integer> map = new HashMap<>();
                //递归解析json
                expainJson(gradeBucketIt, map, null);
                Map<String, SaveModel> allData = new HashMap<>();
                //初始化主细维度
                allData= initDimension(tjQuotaDimensionSlaves, one, allData);
                for(Map.Entry<String,Integer> oneMap:map.entrySet()){
                    String key=oneMap.getKey();
                    Integer value=oneMap.getValue();
                    SaveModel saveModel=allData.get(key);
                    if(saveModel!=null){
                        saveModel.setResult(value);
                        returnList.add(saveModel);
                    }
                }
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
        return returnList;
    }
    /**
     * 初始化主细维度
     */
    private  Map<String, SaveModel>  initDimension(List<TjQuotaDimensionSlave> tjQuotaDimensionSlaves, Map.Entry<String, TjQuotaDimensionMain> one, Map<String, SaveModel> allData) {
        TjQuotaDimensionMain quotaDimensionMain = one.getValue();
        //查询字典数据
        List<SaveModel> dictData = jdbcTemplate.query(quotaDimensionMain.getDictSql(), new BeanPropertyRowMapper(SaveModel.class));
        //设置到map里面
        setAllData(allData, dictData, quotaDimensionMain.getType());
        for (int i = 0; i < tjQuotaDimensionSlaves.size(); i++) {
           List<DictModel> dictDataSlave = jdbcTemplate.query(tjQuotaDimensionSlaves.get(i).getDictSql(), new BeanPropertyRowMapper(DictModel.class));
            allData = setAllSlaveData(allData, dictDataSlave);
        }
        return allData;
    }
    /**
     * 递归解析json
     *
     * @param gradeBucketIt
     * @param map
     * @param sb
     */
    private void expainJson(Iterator<Terms.Bucket> gradeBucketIt,Map<String,Integer>map, StringBuffer sb) {
        while (gradeBucketIt.hasNext()) {
            StringTerms.Bucket b = (StringTerms.Bucket) gradeBucketIt.next();
            if (b.getAggregations().asList().get(0) instanceof StringTerms) {
                StringTerms stringTermsCh = (StringTerms) b.getAggregations().asList().get(0);
                Iterator<Terms.Bucket> gradeBucketItCh = stringTermsCh.getBuckets().iterator();
                while (gradeBucketItCh.hasNext()) {
                    StringBuffer sbTemp = new StringBuffer((sb == null ? "" : (sb.toString() + "-")) + b.getKey());
                    expainJson(gradeBucketItCh, map, sbTemp);
                }
            } else {
                InternalValueCount count = (InternalValueCount) b.getAggregations().asList().get(0);
                map.put(new StringBuffer(sb.toString() + "-" + b.getKey()).toString() , (int)count.getValue());
            }
        }
    }
    /**
     * @param tjQuotaDimensionMains
     * @param tjQuotaDimensionSlaves
     * @return
     */
    private Map<String, TjQuotaDimensionMain> getSql(List<TjQuotaDimensionMain> tjQuotaDimensionMains, List<TjQuotaDimensionSlave> tjQuotaDimensionSlaves) {
        Map<String, TjQuotaDimensionMain> sqlS = new HashMap<>();
        for (int j = 0; j < tjQuotaDimensionMains.size(); j++) {
            TjQuotaDimensionMain one = tjQuotaDimensionMains.get(j);
            String tableName = esConfig.getIndex();
            if (StringUtils.isEmpty(one.getKey())) {
                continue;
            }
            StringBuffer allField = new StringBuffer(one.getKey() + ",");// 例如区  town,sex,age
            StringBuffer AllGroupBy = new StringBuffer(one.getKey() + ",");// 例如区  town,sex,age
            for (int i = 0; i < tjQuotaDimensionSlaves.size(); i++) {
                allField.append(tjQuotaDimensionSlaves.get(i).getKey());
                AllGroupBy.append(tjQuotaDimensionSlaves.get(i).getKey());
                if (i != (tjQuotaDimensionSlaves.size() - 1)) {
                    allField.append(",");
                    AllGroupBy.append(",");
                }
            }
            //拼凑where语句
            StringBuffer whereSql = new StringBuffer(" saasId= '" + saasid + "' and " + esConfig.getTimekey() + " < '" + endTime + "'");
            //是否是增量
            if (Contant.quota.dataLeval_oneDay.endsWith(quotaVO.getDataLevel())) {
                whereSql.append(" and " + esConfig.getTimekey() + " > '" + startTime + "'");
            }
            StringBuffer sql = new StringBuffer("select " + allField + " ,count(*) result from " + tableName + " where " + whereSql + " group by " + AllGroupBy);
            sqlS.put(sql.toString(), one);
        }
        return sqlS;
    }
}

svr/svr-quota/src/main/java/com/yihu/jw/quota/etl/filter/FilterHelper.java

@@ -1,49 +0,0 @@
package com.yihu.jw.quota.etl.filter;
import com.yihu.jw.quota.etl.Contant;
import com.yihu.jw.quota.etl.filter.filter.NotNullFilter;
import com.yihu.jw.quota.model.jpa.rule.TjQuotaRule;
import com.yihu.jw.quota.service.rule.TjCleanRuleService;
import com.yihu.jw.quota.util.SpringUtil;
import com.yihu.jw.quota.vo.FilterModel;
import com.yihu.jw.quota.vo.QuotaVO;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
import java.util.List;
/**
 * Created by chenweida on 2017/6/1.
 */
@Component
@Scope("prototype")
public class FilterHelper {
    @Autowired
    private TjCleanRuleService cleanRuleService;
    public FilterHelper() {
    }
    /**
     * 根据过滤规则过滤数据
     *
     * @param dataModels
     * @param quotaVO
     * @return
     */
    public FilterModel filter(Object dataModels, QuotaVO quotaVO) {
        List<TjQuotaRule> rules = cleanRuleService.findByQuotaCode(quotaVO.getCode());
        FilterModel f=new FilterModel(dataModels);
        rules.stream().forEach(rule -> {
             switch (rule.getType()){
                 //非空校验
                 case Contant.role.not_null: {
                     SpringUtil.getBean(NotNullFilter.class).filter(f,quotaVO);break;}
                 //如果有其他的校验规则在这边扩展
             }
        });
        return f;
    }
}

svr/svr-quota/src/main/java/com/yihu/jw/quota/etl/filter/filter/NotNullFilter.java

@@ -1,84 +0,0 @@
package com.yihu.jw.quota.etl.filter.filter;
import com.yihu.jw.quota.etl.Contant;
import com.yihu.jw.quota.model.jpa.dimension.TjDimensionSlave;
import com.yihu.jw.quota.service.dimension.TjDimensionSlaveService;
import com.yihu.jw.quota.vo.DataModel;
import com.yihu.jw.quota.vo.ErrModel;
import com.yihu.jw.quota.vo.FilterModel;
import com.yihu.jw.quota.vo.QuotaVO;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
/**
 * Created by chenweida on 2017/6/1.
 */
@Component
@Scope("prototype")
public class NotNullFilter {
    private Logger logger = LoggerFactory.getLogger(NotNullFilter.class);
    @Autowired
    private TjDimensionSlaveService dimensionSlaveService;
    /**
     * 不判断主维度的数据是否为空，只判断从维度额的数据是否为空
     *
     * @param filterModel
     * @param quotaVO
     * @return
     */
    public FilterModel filter(FilterModel filterModel, QuotaVO quotaVO) {
        try {
            //得到从维度
            List<TjDimensionSlave> tj = dimensionSlaveService.getDimensionSlaveByQuotaCode(quotaVO.getCode());
            if (Contant.compute.add.equals(quotaVO.getComputeType())) {
                //判断是否是累加计算 如果强转成list类型
                List<DataModel> dmList = (List<DataModel>) filterModel.getData();
                //过滤数据
                filterList(dmList, tj, filterModel.getErrorModels());
            } else if (Contant.compute.division.equals(quotaVO.getComputeType())) {
                //判断是否是除法计算 如果强转成Map类型 key在Contant中
                Map<String, List<DataModel>> dmMap = (Map<String, List<DataModel>>) filterModel.getData();
                List<DataModel> one = dmMap.get(Contant.extract.computeKey1);
                List<DataModel> two = dmMap.get(Contant.extract.computeKey2);
                filterList(one, tj, filterModel.getErrorModels());
                filterList(two, tj, filterModel.getErrorModels());
            }
            return filterModel;
        }catch (Exception e){
            e.printStackTrace();
            logger.error(e.getMessage());
        }
        return null;
    }
    private void filterList(List<DataModel> dmList, List<TjDimensionSlave> tj, List<ErrModel> errorModels) {
        Iterator<DataModel> iterator= dmList.iterator();
        while (iterator.hasNext()) {
            DataModel dataModel = iterator.next();
            for (int i = 1; i <= tj.size(); i++) {
                try {
                    //根据反射判断字段值是否为空
                    Object obj = DataModel.class.getMethod("getSlaveKey" + i).invoke(dataModel);
                    if (StringUtils.isEmpty(obj)) {
                        dmList.iterator().remove();
                        errorModels.add(new ErrModel(dataModel.getBusinessId(), new StringBuffer("slaveKey" + i + " is null").toString()));
                    }
                } catch (Exception e) {
                    logger.error(dataModel.toString() + "," + e.getMessage());
                    e.printStackTrace();
                }
            }
        }
    }
}

svr/svr-quota/src/main/java/com/yihu/jw/quota/etl/model/EsConfig.java

@@ -6,8 +6,10 @@ package com.yihu.jw.quota.etl.model;
public class EsConfig {
    private String host;//地址
    private Integer port;//端口
    private String index;// 索引 es相当与数据库
    private String type;// 类型 es 相当于表
    private String timekey;//根据sql去查询的key
    public String getHost() {
        return host;
@@ -34,4 +36,19 @@ public class EsConfig {
    }
    public String getTimekey() {
        return timekey;
    }
    public void setTimekey(String timekey) {
        this.timekey = timekey;
    }
    public Integer getPort() {
        return port;
    }
    public void setPort(Integer port) {
        this.port = port;
    }
}

svr/svr-quota/src/main/java/com/yihu/jw/quota/etl/save/SaveHelper.java

@@ -22,18 +22,15 @@ public class SaveHelper {
    @Autowired
    private TjDataSaveService datsSaveService;
    public Boolean save(List<SaveModel> sms, QuotaVO quotaVO) {
    public Boolean save(List<SaveModel> dataModels, QuotaVO quotaVO) {
        //查看指标保存的数据源
        TjQuotaDataSave quotaDataSave = datsSaveService.findByQuota(quotaVO.getCode());
        switch (quotaDataSave.getType()) {
            case Contant.save.mysql: {
                break;
            }
            case Contant.save.redis: {
                break;
                return null;
            }
            case Contant.save.es: {
                return SpringUtil.getBean(ElastricSearchSave.class).save(sms,quotaDataSave.getJsonConfig());
                return SpringUtil.getBean(ElastricSearchSave.class).save(dataModels,quotaDataSave.getConfigJson());
            }
        }
        return false;

svr/svr-quota/src/main/java/com/yihu/jw/quota/etl/save/es/ElasticFactory.java

@@ -4,8 +4,15 @@ import io.searchbox.client.JestClient;
import io.searchbox.client.JestClientFactory;
import io.searchbox.client.config.ClientConfig;
import io.searchbox.client.config.HttpClientConfig;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.util.concurrent.TimeUnit;
/**
@@ -14,11 +21,12 @@ import java.util.concurrent.TimeUnit;
@Component
public class ElasticFactory {
    /**
     * @param hostAddress "http://localhost:9200"
     * @param host "localhost"
     * @param port 9200
     * @return
     */
    public JestClient getClient(String hostAddress) {
        // Construct a new Jest client according to configuration via factory
    public JestClient getJestClient(String host, Integer port) {
        String hostAddress="http://"+host+":"+port;
        JestClientFactory factory = new JestClientFactory();
        factory.setHttpClientConfig(new HttpClientConfig
                .Builder(hostAddress)
@@ -28,4 +36,20 @@ public class ElasticFactory {
                .build());
        return factory.getObject();
    }
    public Client getClient(String host, Integer port,String clusterName) {
        try {
            Settings settings = Settings.settingsBuilder()
                    .put("client.transport.sniff", true)
                    .put("cluster.name", StringUtils.isEmpty(clusterName)?"jkzl":clusterName)
                    .build();
            Client client = TransportClient.builder().settings(settings).build()
                    .addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(host), port));
            return client;
        } catch (Exception e) {
            e.printStackTrace();
        }
        return null;
    }
}

svr/svr-quota/src/main/java/com/yihu/jw/quota/etl/save/es/ElastricSearchSave.java

@@ -15,6 +15,7 @@ import org.springframework.stereotype.Component;
import java.util.Date;
import java.util.List;
import java.util.Map;
/**
 * Created by chenweida on 2017/6/2.
@@ -29,16 +30,15 @@ public class ElastricSearchSave {
    private EsConfig esConfig;
    public Boolean save(List<SaveModel> sms, String jsonConfig) {
    public Boolean save(List<SaveModel> smss, String jsonConfig) {
        try {
            //初始化参数
            esConfig = (EsConfig) JSONObject.toBean(JSONObject.fromObject(jsonConfig), EsConfig.class);
            //得到链接
            JestClient jestClient = elasticFactory.getClient(esConfig.getHost());
            JestClient jestClient = elasticFactory.getJestClient(esConfig.getHost(),esConfig.getPort());
            Bulk.Builder bulk = new Bulk.Builder().defaultIndex(esConfig.getIndex()).defaultType(esConfig.getType());
            for (SaveModel obj : sms) {
                obj.setCreateTime(new Date());
            for (SaveModel obj : smss) {
                Index index = new Index.Builder(obj).build();
                bulk.addAction(index);
            }

svr/svr-quota/src/main/java/com/yihu/jw/quota/job/QuotaJob.java

@@ -2,19 +2,13 @@ package com.yihu.jw.quota.job;
import com.yihu.jw.quota.dao.jpa.TjQuotaLogDao;
import com.yihu.jw.quota.etl.Contant;
import com.yihu.jw.quota.etl.compute.ComputeHelper;
import com.yihu.jw.quota.etl.dimenconvert.ConvertHelper;
import com.yihu.jw.quota.etl.extract.ExtractHelper;
import com.yihu.jw.quota.etl.filter.FilterHelper;
import com.yihu.jw.quota.etl.save.SaveHelper;
import com.yihu.jw.quota.model.jpa.TjQuotaLog;
import com.yihu.jw.quota.model.jpa.compute.TjCompute;
import com.yihu.jw.quota.service.compute.TjComputeService;
import com.yihu.jw.quota.service.rule.TjCleanRuleService;
import com.yihu.jw.quota.service.source.TjDataSourceService;
import com.yihu.jw.quota.util.SpringUtil;
import com.yihu.jw.quota.vo.*;
import net.sf.json.JSONObject;
import com.yihu.jw.quota.vo.QuotaVO;
import com.yihu.jw.quota.vo.SaveModel;
import org.joda.time.DateTime;
import org.joda.time.LocalDate;
import org.quartz.*;
@@ -22,51 +16,42 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Scope;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.StringUtils;
import org.springframework.web.context.support.SpringBeanAutowiringSupport;
import java.text.SimpleDateFormat;
import java.time.format.DateTimeFormatter;
import java.util.Date;
import java.util.List;
import java.util.Map;
/**
 * Created by Administrator on 2016.10.10.
 * 每天的签约到达量统计
 * Created by chenweida on 2017/6/6.
 */
@Component
@Scope("prototype")
@DisallowConcurrentExecution//防止到了执行时间点前一任务还在执行中，但是这时有空闲的线程，那么马上又会执行，这样一来就会存在同一job被并行执行
public class QuotaJob implements Job {
    private Logger logger = LoggerFactory.getLogger(QuotaJob.class);
    @Autowired
    private JdbcTemplate jdbcTemplate;
    @Autowired
    private TjComputeService computeService;
    @Autowired
    private TjQuotaLogDao tjQuotaLogDao;
public class EsQuotaJob implements Job {
    private Logger logger = LoggerFactory.getLogger(EsQuotaJob.class);
    private String saasid;//saasid
    private QuotaVO quotaVO;//指标对象
    private String endTime;//结束时间
    private String startTime;//开始时间
    private String timeLevel;//时间
    @Autowired
    private TjQuotaLogDao tjQuotaLogDao;
    @Autowired
    private TjComputeService computeService;
    public void execute(JobExecutionContext context)
            throws JobExecutionException {
    @Override
    public void execute(JobExecutionContext context) throws JobExecutionException {
        try {
            //springz注入
            SpringBeanAutowiringSupport.processInjectionBasedOnCurrentContext(this);
            //初始化参数
            initParams(context);
            //统计指标
            computequota();
            //统计
            quota();
        } catch (Exception e) {
            //如果出錯立即重新執行
            JobExecutionException e2 = new JobExecutionException(e);
@@ -75,143 +60,89 @@ public class QuotaJob implements Job {
        }
    }
    /**
     * 初始化参数
     *
     * @param context
     */
    private void initParams(JobExecutionContext context) {
        JobDataMap map = context.getJobDetail().getJobDataMap();
        this.saasid = map.getString("saasid");
        this.endTime = map.getString("endTime");
        if (StringUtils.isEmpty(endTime)) {
            endTime = LocalDate.now().toString("yyyy-MM-dd"); //2017-06-01 默认今天
        }
        this.startTime = map.getString("startTime");
        if (StringUtils.isEmpty(startTime)) {
            startTime = new LocalDate(new DateTime().minusDays(1)).toString("yyyy-MM-dd"); //默认昨天
        }
        this.quotaVO = (QuotaVO) map.get("quota");
        if (quotaVO != null) {
            //得到算法
            TjCompute compute = computeService.findByQuotaCode(quotaVO.getCode());
            quotaVO.setComputeType(compute.getType());
        } else {
            logger.error("quotaVO is null");
            return;
        }
        this.timeLevel = (String) map.get("timeLevel");
        if (StringUtils.isEmpty(this.timeLevel)) {
            this.timeLevel = Contant.main_dimension_timeLevel.day;
        }
    }
    /**
     * 计算指标
     * 统计过程
     */
    private void computequota() {
    private void quota() {
        try {
            TjQuotaLog tjQuotaLog = new TjQuotaLog();
            tjQuotaLog.setQuotaCode(quotaVO.getCode());
            tjQuotaLog.setSaasId(saasid);
            tjQuotaLog.setStartTime(new Date());
            JobLogModel jobLogModel = new JobLogModel();
            //抽取数据 如果是累加就是 List<DataModel>  如果是相除 Map<String,List<DataModel>>
            Object dataModels = extract();
            //根据规则过滤数据
            FilterModel filterModel = filter(dataModels);
            jobLogModel.setErrorModels(filterModel.getErrorModels());
            //从维度的key转换
            filterModel = convert(filterModel);
            //统计数据
            List<SaveModel> sms = compute(filterModel.getData());
            List<SaveModel> dataModels = extract();
            //保存数据
            Boolean success = saveDate(sms);
            Boolean success = saveDate(dataModels);
            tjQuotaLog.setStatus(success ? Contant.save_status.success : Contant.save_status.fail);
            tjQuotaLog.setEndTime(new Date());
            tjQuotaLog.setContent(JSONObject.fromObject(jobLogModel).toString());
            saveLog(tjQuotaLog);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
    @Transactional
    private void saveLog(TjQuotaLog tjQuotaLog) {
        tjQuotaLogDao.save(tjQuotaLog);
    }
    private FilterModel convert(FilterModel dataModels) {
        try{
            return SpringUtil.getBean(ConvertHelper.class).convert(dataModels,quotaVO);
        }catch (Exception e){
            logger.error("convert error:" + e.getMessage());
        }
        return null;
    }
    /**
     * 保存数据
     * 抽取数据
     *
     * @param sms
     * @return
     */
    private Boolean saveDate(List<SaveModel> sms) {
    private List<SaveModel> extract() {
        try {
            return SpringUtil.getBean(SaveHelper.class).save(sms, quotaVO);
            return SpringUtil.getBean(ExtractHelper.class).extractData(quotaVO, startTime, endTime,timeLevel,saasid);
        } catch (Exception e) {
            logger.error("save error:" + e.getMessage());
            e.printStackTrace();
        }
        return false;
        return null;
    }
    /**
     * 根据计算规则统计数据
     * 初始化参数
     *
     * @param dataModels
     * @param context
     */
    private List<SaveModel> compute(Object dataModels) {
        try {
            return SpringUtil.getBean(ComputeHelper.class).compute(dataModels, quotaVO, timeLevel, saasid, startTime);
        } catch (Exception e) {
            logger.error("compute error:" + e.getMessage());
    private void initParams(JobExecutionContext context) {
        JobDataMap map = context.getJobDetail().getJobDataMap();
        this.saasid = map.getString("saasid");
        this.endTime = map.getString("endTime");
        if (StringUtils.isEmpty(endTime)) {
            endTime = LocalDate.now().toString("yyyy-MM-dd"); //2017-06-01 默认今天
        }
        return null;
    }
    /**
     * 过滤数据
     *
     * @param filterModel
     * @return
     */
    private FilterModel filter(Object filterModel) {
        try {
            return SpringUtil.getBean(FilterHelper.class).filter(filterModel,quotaVO);
        } catch (Exception e) {
            logger.error("filter error:" + e.getMessage());
        this.startTime = map.getString("startTime");
        if (StringUtils.isEmpty(startTime)) {
            startTime = new LocalDate(new DateTime().minusDays(1)).toString("yyyy-MM-dd"); //默认昨天
        }
        return null;
        this.quotaVO = (QuotaVO) map.get("quota");
        this.timeLevel = (String) map.get("timeLevel");
        if (StringUtils.isEmpty(this.timeLevel)) {
            this.timeLevel = Contant.main_dimension_timeLevel.day;
        }
    }
    @Transactional
    private void saveLog(TjQuotaLog tjQuotaLog) {
        tjQuotaLogDao.save(tjQuotaLog);
    }
    /**
     * 抽取数据
     * 保存数据
     *
     * @return
     * @param dataModels
     */
    private Object extract() {
    private Boolean saveDate(List<SaveModel> dataModels) {
        try {
            return SpringUtil.getBean(ExtractHelper.class).extractData(quotaVO, startTime, endTime);
            return SpringUtil.getBean(SaveHelper.class).save(dataModels, quotaVO);
        } catch (Exception e) {
            logger.error("extract error:" + e.getMessage());
            logger.error("save error:" + e.getMessage());
        }
        return null;
        return false;
    }
}

svr/svr-quota/src/main/java/com/yihu/jw/quota/model/jpa/TjQuota.java

@@ -16,10 +16,10 @@ public class TjQuota implements java.io.Serializable {
	private Integer id;
	private String code;
	private String compute; //关联表tj_compute code
	private String name;
	private String cron;//quartz时间表达式
	private String jobClazz; //类class
	private String dataLevel;//1 全量  2增量
	private Date createTime;
	private String createUser;
	private String createUserName;
@@ -170,14 +170,6 @@ public class TjQuota implements java.io.Serializable {
		this.remark = remark;
	}
	public String getCompute() {
		return compute;
	}
	public void setCompute(String compute) {
		this.compute = compute;
	}
	public String getCron() {
		return cron;
	}
@@ -185,4 +177,12 @@ public class TjQuota implements java.io.Serializable {
	public void setCron(String cron) {
		this.cron = cron;
	}
	public String getDataLevel() {
		return dataLevel;
	}
	public void setDataLevel(String dataLevel) {
		this.dataLevel = dataLevel;
	}
}

svr/svr-quota/src/main/java/com/yihu/jw/quota/model/jpa/dimension/TjQuotaDimensionMain.java

@@ -16,7 +16,8 @@ public class TjQuotaDimensionMain implements java.io.Serializable {
	private Integer id;
	private String quotaCode;
	private String mainCode;
	private String ditcSql;
	private String dictSql;
	private String key;
	private String type;
	// Constructors
@@ -55,13 +56,22 @@ public class TjQuotaDimensionMain implements java.io.Serializable {
		this.mainCode = mainCode;
	}
	public String getDitcSql() {
		return ditcSql;
	public String getDictSql() {
		return dictSql;
	}
	public void setDitcSql(String ditcSql) {
		this.ditcSql = ditcSql;
	public void setDictSql(String dictSql) {
		this.dictSql = dictSql;
	}
	public String getKey() {
		return key;
	}
	public void setKey(String key) {
		this.key = key;
	}
	@Transient
	public String getType() {
		return type;

svr/svr-quota/src/main/java/com/yihu/jw/quota/model/jpa/dimension/TjQuotaDimensionSlave.java

@@ -23,13 +23,9 @@ public class TjQuotaDimensionSlave implements java.io.Serializable {
    private String quotaCode;
    private String slaveCode;
    private String dictSql;//字典的sql
    private String convertClazz;//转换的类
    private String key;
    private String type;
    private String getOneDictSql;//得到分子的
    private String getTwoDictSql;//得到分母的
    private String getOneConvertClazz;//得到分子的
    private String getTwoConvertClazz;//得到分母的
    // Constructors
@@ -95,67 +91,11 @@ public class TjQuotaDimensionSlave implements java.io.Serializable {
        this.type = type;
    }
    public String getConvertClazz() {
        return convertClazz;
    public String getKey() {
        return key;
    }
    public void setConvertClazz(String convertClazz) {
        this.convertClazz = convertClazz;
    }
    @Transient
    public String getGetOneDictSql() {
        try {
            return JSONObject.fromObject(dictSql).getString(Contant.slave_dimension_key.one);
        } catch (Exception e) {
            logger.warn("OneDictSql error");
        }
        return null;
    }
    public void setGetOneDictSql(String getOneDictSql) {
        this.getOneDictSql = getOneDictSql;
    }
    @Transient
    public String getGetTwoDictSql() {
        try {
            return JSONObject.fromObject(dictSql).getString(Contant.slave_dimension_key.two);
        } catch (Exception e) {
            logger.warn("GetTwoDictSql error");
        }
        return null;
    }
    public void setGetTwoDictSql(String getTwoDictSql) {
        this.getTwoDictSql = getTwoDictSql;
    }
    @Transient
    public String getGetOneConvertClazz() {
        try {
            return JSONObject.fromObject(convertClazz).getString(Contant.slave_dimension_key.one);
        } catch (Exception e) {
            logger.warn("GetOneConvertClazz error");
        }
        return null;
    }
    public void setGetOneConvertClazz(String getOneConvertClazz) {
        this.getOneConvertClazz = getOneConvertClazz;
    }
    @Transient
    public String getGetTwoConvertClazz() {
        try {
            return JSONObject.fromObject(convertClazz).getString(Contant.slave_dimension_key.two);
        } catch (Exception e) {
            logger.warn("TwoConvertClazz error");
        }
        return null;
    }
    public void setGetTwoConvertClazz(String getTwoConvertClazz) {
        this.getTwoConvertClazz = getTwoConvertClazz;
    public void setKey(String key) {
        this.key = key;
    }
}

svr/svr-quota/src/main/java/com/yihu/jw/quota/model/jpa/save/TjQuotaDataSave.java

@@ -16,7 +16,7 @@ public class TjQuotaDataSave implements java.io.Serializable {
	private Integer id;
	private String quotaCode;
	private String saveCode;
	private String jsonConfig;
	private String configJson;
	private String type;//1:mysql数据库 2:redis 3ES
	// Constructors
@@ -25,12 +25,6 @@ public class TjQuotaDataSave implements java.io.Serializable {
	public TjQuotaDataSave() {
	}
	/** full constructor */
	public TjQuotaDataSave(String quotaCode, String saveCode, String jsonConfig) {
		this.quotaCode = quotaCode;
		this.saveCode = saveCode;
		this.jsonConfig = jsonConfig;
	}
	// Property accessors
	@Id
@@ -62,14 +56,15 @@ public class TjQuotaDataSave implements java.io.Serializable {
		this.saveCode = saveCode;
	}
	@Column(name = "json_config", length = 1500)
	public String getJsonConfig() {
		return this.jsonConfig;
	@Column(name = "config_json", length = 1500)
	public String getConfigJson() {
		return configJson;
	}
	public void setJsonConfig(String jsonConfig) {
		this.jsonConfig = jsonConfig;
	public void setConfigJson(String configJson) {
		this.configJson = configJson;
	}
	@Transient
	public String getType() {
		return type;

svr/svr-quota/src/main/java/com/yihu/jw/quota/model/jpa/source/TjDataSource.java

@@ -11,7 +11,7 @@ import static javax.persistence.GenerationType.IDENTITY;
@Entity
@Table(name = "tj_data_source")
public class TjDataSource implements java.io.Serializable {
	public static String type_mysql="1";
	public static String type_es="1";
	public static String type_redis="2";
	// Fields

svr/svr-quota/src/main/java/com/yihu/jw/quota/vo/DataModel.java

@@ -1,126 +0,0 @@
package com.yihu.jw.quota.vo;
import com.yihu.jw.quota.etl.Contant;
/**
 * Created by chenweida on 2017/6/1.
 */
public class DataModel {
    private String businessId;//业务id
    private String province;//省级代码 350000
    private String city;//城市代码 350200
    private String town;//区代码 350206
    private String hospital;//机构code
    private String team;//团队的code
    private String slaveKey1;//从维度  1级维度
    private String slaveKey2;//从维度  2级维度
    private String slaveKey3;//从维度  3级维度
    private String slaveKey4;//从维度  4级维度
    public String getBusinessId() {
        return businessId;
    }
    public void setBusinessId(String businessId) {
        this.businessId = businessId;
    }
    public String getTeam() {
        return team;
    }
    public void setTeam(String team) {
        this.team = team;
    }
    public String getHospital() {
        return hospital;
    }
    public void setHospital(String hospital) {
        this.hospital = hospital;
    }
    public String getTown() {
        return town;
    }
    public void setTown(String town) {
        this.town = town;
    }
    public String getCity() {
        return city;
    }
    public void setCity(String city) {
        this.city = city;
    }
    public String getSlaveKey1() {
        return slaveKey1;
    }
    public void setSlaveKey1(String slaveKey1) {
        this.slaveKey1 = slaveKey1;
    }
    public String getSlaveKey2() {
        return slaveKey2;
    }
    public void setSlaveKey2(String slaveKey2) {
        this.slaveKey2 = slaveKey2;
    }
    public String getSlaveKey3() {
        return slaveKey3;
    }
    public void setSlaveKey3(String slaveKey3) {
        this.slaveKey3 = slaveKey3;
    }
    public String getSlaveKey4() {
        return slaveKey4;
    }
    public void setSlaveKey4(String slaveKey4) {
        this.slaveKey4 = slaveKey4;
    }
    public String getProvince() {
        return province;
    }
    public void setProvince(String province) {
        this.province = province;
    }
    public String getKey(String leveltype){
        switch (leveltype){
            case Contant.main_dimension_areaLevel.area_province:{return getProvince();}
            case Contant.main_dimension_areaLevel.area_city:{return getCity();}
            case Contant.main_dimension_areaLevel.area_town:{return getTown();}
            case Contant.main_dimension_areaLevel.area_org:{return getHospital();}
            case Contant.main_dimension_areaLevel.area_team:{return getTeam();}
        }
        return "";
    }
    @Override
    public String toString() {
        return "DataModel{" +
                "businessId='" + businessId + '\'' +
                ", province='" + province + '\'' +
                ", city='" + city + '\'' +
                ", town='" + town + '\'' +
                ", hospital='" + hospital + '\'' +
                ", team='" + team + '\'' +
                ", slaveKey1='" + slaveKey1 + '\'' +
                ", slaveKey2='" + slaveKey2 + '\'' +
                ", slaveKey3='" + slaveKey3 + '\'' +
                ", slaveKey4='" + slaveKey4 + '\'' +
                '}';
    }
}

svr/svr-quota/src/main/java/com/yihu/jw/quota/vo/FilterModel.java

@@ -1,42 +0,0 @@
package com.yihu.jw.quota.vo;
import java.util.ArrayList;
import java.util.List;
/**
 * Created by chenweida on 2017/6/1.
 */
public class FilterModel {
    private Object data;//可能是list 也可能是map
    private List<ErrModel> ErrorModels =new ArrayList<>();//
    private List<ErrModel> slaveKeyChange=new ArrayList<>();
    public FilterModel(Object data) {
        this.data = data;
    }
    public FilterModel() {
    }
    public Object getData() {
        return data;
    }
    public void setData(Object data) {
        this.data = data;
    }
    public List<ErrModel> getErrorModels() {
        return ErrorModels;
    }
    public void setErrorModels(List<ErrModel> ErrorModels) {
        this.ErrorModels = ErrorModels;
    }
    public List<ErrModel> getSlaveKeyChange() {
        return slaveKeyChange;
    }
    public void setSlaveKeyChange(List<ErrModel> slaveKeyChange) {
        this.slaveKeyChange = slaveKeyChange;
    }
}

svr/svr-quota/src/main/java/com/yihu/jw/quota/vo/MainDimensionModel.java

@@ -1,202 +0,0 @@
package com.yihu.jw.quota.vo;
import java.util.ArrayList;
import java.util.List;
/**
 * Created by chenweida on 2017/6/1.
 */
public class MainDimensionModel {
    private String quotaCode;//指标code
    private String province;//省级代码 350000
    private String provinceName;//省名字
    private String city;//城市代码 350200
    private String cityName;//
    private String town;//区代码 350206
    private String townName;//
    private String hospital;//机构code
    private String hospitalName;//
    private String team;//团队的code
    private String teamName;//
    private String areaLevel;// 1 省 2 市 3 区县 4 机构 5团队
    //用来计算从维度的字段--------------
    private String slaveKey1;//从维度  1级维度
    private String slaveKey1Name;
    private String slaveKey2;//从维度  2级维度
    private String slaveKey2Name;
    private String slaveKey3;//从维度  3级维度
    private String slaveKey3Name;
    private String slaveKey4;//从维度  4级维度
    private String slaveKey4Name;
    //用来计算从维度的字段--------------
    List<SlaveDimensionModel> slaveDimensionModels=new ArrayList<>();
    public String getAreaLevel() {
        return areaLevel;
    }
    public void setAreaLevel(String areaLevel) {
        this.areaLevel = areaLevel;
    }
    public String getQuotaCode() {
        return quotaCode;
    }
    public void setQuotaCode(String quotaCode) {
        this.quotaCode = quotaCode;
    }
    public String getProvince() {
        return province;
    }
    public void setProvince(String province) {
        this.province = province;
    }
    public String getProvinceName() {
        return provinceName;
    }
    public void setProvinceName(String provinceName) {
        this.provinceName = provinceName;
    }
    public String getCity() {
        return city;
    }
    public void setCity(String city) {
        this.city = city;
    }
    public String getCityName() {
        return cityName;
    }
    public void setCityName(String cityName) {
        this.cityName = cityName;
    }
    public String getTown() {
        return town;
    }
    public void setTown(String town) {
        this.town = town;
    }
    public String getTownName() {
        return townName;
    }
    public void setTownName(String townName) {
        this.townName = townName;
    }
    public String getHospital() {
        return hospital;
    }
    public void setHospital(String hospital) {
        this.hospital = hospital;
    }
    public String getHospitalName() {
        return hospitalName;
    }
    public void setHospitalName(String hospitalName) {
        this.hospitalName = hospitalName;
    }
    public String getTeam() {
        return team;
    }
    public void setTeam(String team) {
        this.team = team;
    }
    public String getTeamName() {
        return teamName;
    }
    public void setTeamName(String teamName) {
        this.teamName = teamName;
    }
    public List<SlaveDimensionModel> getSlaveDimensionModels() {
        return slaveDimensionModels;
    }
    public void setSlaveDimensionModels(List<SlaveDimensionModel> slaveDimensionModels) {
        this.slaveDimensionModels = slaveDimensionModels;
    }
    public String getSlaveKey1() {
        return slaveKey1;
    }
    public void setSlaveKey1(String slaveKey1) {
        this.slaveKey1 = slaveKey1;
    }
    public String getSlaveKey1Name() {
        return slaveKey1Name;
    }
    public void setSlaveKey1Name(String slaveKey1Name) {
        this.slaveKey1Name = slaveKey1Name;
    }
    public String getSlaveKey2() {
        return slaveKey2;
    }
    public void setSlaveKey2(String slaveKey2) {
        this.slaveKey2 = slaveKey2;
    }
    public String getSlaveKey2Name() {
        return slaveKey2Name;
    }
    public void setSlaveKey2Name(String slaveKey2Name) {
        this.slaveKey2Name = slaveKey2Name;
    }
    public String getSlaveKey3() {
        return slaveKey3;
    }
    public void setSlaveKey3(String slaveKey3) {
        this.slaveKey3 = slaveKey3;
    }
    public String getSlaveKey3Name() {
        return slaveKey3Name;
    }
    public void setSlaveKey3Name(String slaveKey3Name) {
        this.slaveKey3Name = slaveKey3Name;
    }
    public String getSlaveKey4() {
        return slaveKey4;
    }
    public void setSlaveKey4(String slaveKey4) {
        this.slaveKey4 = slaveKey4;
    }
    public String getSlaveKey4Name() {
        return slaveKey4Name;
    }
    public void setSlaveKey4Name(String slaveKey4Name) {
        this.slaveKey4Name = slaveKey4Name;
    }
}

svr/svr-quota/src/main/java/com/yihu/jw/quota/vo/QuotaDimensionSlaveVO.java

@@ -1,62 +0,0 @@
package com.yihu.jw.quota.vo;
/**
 * Created by chenweida on 2017/6/5.
 */
public class QuotaDimensionSlaveVO {
    private Integer id;
    private String quotaCode;
    private String slaveCode;
    private String dictSql;//字典的sql
    private String convertClazz;//转换的类
    private String type;
    public Integer getId() {
        return id;
    }
    public void setId(Integer id) {
        this.id = id;
    }
    public String getQuotaCode() {
        return quotaCode;
    }
    public void setQuotaCode(String quotaCode) {
        this.quotaCode = quotaCode;
    }
    public String getSlaveCode() {
        return slaveCode;
    }
    public void setSlaveCode(String slaveCode) {
        this.slaveCode = slaveCode;
    }
    public String getDictSql() {
        return dictSql;
    }
    public void setDictSql(String dictSql) {
        this.dictSql = dictSql;
    }
    public String getConvertClazz() {
        return convertClazz;
    }
    public void setConvertClazz(String convertClazz) {
        this.convertClazz = convertClazz;
    }
    public String getType() {
        return type;
    }
    public void setType(String type) {
        this.type = type;
    }
}

svr/svr-quota/src/main/java/com/yihu/jw/quota/vo/QuotaVo.java

@@ -10,9 +10,9 @@ public class QuotaVO implements Serializable{
    private Integer id;
    private String code;
    private String compute;
    private String computeType;//计算类型
    private String name;
    private String cron;//quartz时间表达式
    private String dataLevel;//1 全量  2增量
    private String jobClazz;
    private Date createTime;
    private String createUser;
@@ -120,13 +120,6 @@ public class QuotaVO implements Serializable{
        this.remark = remark;
    }
    public String getComputeType() {
        return computeType;
    }
    public void setComputeType(String computeType) {
        this.computeType = computeType;
    }
    public String getCompute() {
        return compute;
@@ -142,7 +135,6 @@ public class QuotaVO implements Serializable{
                "id=" + id +
                ", code='" + code + '\'' +
                ", compute='" + compute + '\'' +
                ", computeType='" + computeType + '\'' +
                ", name='" + name + '\'' +
                ", jobClazz='" + jobClazz + '\'' +
                ", createTime=" + createTime +
@@ -163,4 +155,12 @@ public class QuotaVO implements Serializable{
    public void setCron(String cron) {
        this.cron = cron;
    }
    public String getDataLevel() {
        return dataLevel;
    }
    public void setDataLevel(String dataLevel) {
        this.dataLevel = dataLevel;
    }
}

svr/svr-quota/src/main/java/com/yihu/jw/quota/vo/SaveModel.java

@@ -65,8 +65,6 @@ public class SaveModel {
    @CreatedDate
    private Date createTime;//创建时间
    private Integer one;
    private Integer two;
    public String getQuotaDate() {
@@ -277,19 +275,4 @@ public class SaveModel {
        this.id = id;
    }
    public Integer getOne() {
        return one;
    }
    public void setOne(Integer one) {
        this.one = one;
    }
    public Integer getTwo() {
        return two;
    }
    public void setTwo(Integer two) {
        this.two = two;
    }
}

svr/svr-quota/src/main/java/com/yihu/jw/quota/vo/SlaveDimensionModel.java

@@ -1,89 +0,0 @@
package com.yihu.jw.quota.vo;
/**
 * Created by chenweida on 2017/6/1.
 */
public class SlaveDimensionModel {
    private String slaveKey1;//从维度  1级维度
    private String slaveKey1Name;
    private String slaveKey2;//从维度  2级维度
    private String slaveKey2Name;
    private String slaveKey3;//从维度  3级维度
    private String slaveKey3Name;
    private String slaveKey4;//从维度  4级维度
    private String slaveKey4Name;
    public SlaveDimensionModel() {
    }
    public SlaveDimensionModel(String slaveKey1, String slaveKey2, String slaveKey3, String slaveKey4) {
        this.slaveKey1 = slaveKey1;
        this.slaveKey2 = slaveKey2;
        this.slaveKey3 = slaveKey3;
        this.slaveKey4 = slaveKey4;
    }
    public String getSlaveKey1() {
        return slaveKey1;
    }
    public void setSlaveKey1(String slaveKey1) {
        this.slaveKey1 = slaveKey1;
    }
    public String getSlaveKey1Name() {
        return slaveKey1Name;
    }
    public void setSlaveKey1Name(String slaveKey1Name) {
        this.slaveKey1Name = slaveKey1Name;
    }
    public String getSlaveKey2() {
        return slaveKey2;
    }
    public void setSlaveKey2(String slaveKey2) {
        this.slaveKey2 = slaveKey2;
    }
    public String getSlaveKey2Name() {
        return slaveKey2Name;
    }
    public void setSlaveKey2Name(String slaveKey2Name) {
        this.slaveKey2Name = slaveKey2Name;
    }
    public String getSlaveKey3() {
        return slaveKey3;
    }
    public void setSlaveKey3(String slaveKey3) {
        this.slaveKey3 = slaveKey3;
    }
    public String getSlaveKey3Name() {
        return slaveKey3Name;
    }
    public void setSlaveKey3Name(String slaveKey3Name) {
        this.slaveKey3Name = slaveKey3Name;
    }
    public String getSlaveKey4() {
        return slaveKey4;
    }
    public void setSlaveKey4(String slaveKey4) {
        this.slaveKey4 = slaveKey4;
    }
    public String getSlaveKey4Name() {
        return slaveKey4Name;
    }
    public void setSlaveKey4Name(String slaveKey4Name) {
        this.slaveKey4Name = slaveKey4Name;
    }
}