统计代码提交

svr-lib-parent-pom/pom.xml

@@ -68,9 +68,11 @@
        <version.quartz>2.3.0</version.quartz>
        <version.logback>1.2.3</version.logback>
        <version.spring-elasticsearch>2.1.3.RELEASE</version.spring-elasticsearch>
        <version.jest>2.4.0</version.jest>
        <version.json>20160212</version.json>
        <version.net-json>2.4</version.net-json>
        <version.springside>4.2.3-GA</version.springside>
        <version.common.lang3>3.1</version.common.lang3>
    </properties>
    <!--dependencyManagement作用子配置不写版本默认继承父配置-->
    <dependencyManagement>
@@ -446,14 +448,22 @@
            </dependency>
            <!--json end -->
            <!-- springside-core start-->
            <!-- es 客户端 start-->
            <dependency>
                <groupId>org.springside</groupId>
                <artifactId>springside-core</artifactId>
                <version>${version.springside}</version>
                <groupId>io.searchbox</groupId>
                <artifactId>jest</artifactId>
                <version>${version.jest}</version>
            </dependency>
            <!-- es 客户端 end-->
            <dependency>
                <groupId>org.apache.commons</groupId>
                <artifactId>commons-lang3</artifactId>
                <version>${version.common.lang3}</version>
            </dependency>
            <!-- springside-core end-->
        </dependencies>
    </dependencyManagement>
    <build>

svr/svr-base/pom.xml

@@ -16,15 +16,15 @@
    <dependencies>
        <dependency>
            <groupId>com.yihu.jw</groupId>
            <artifactId>common-swagger</artifactId>
            <artifactId>common-util</artifactId>
        </dependency>
        <dependency>
            <groupId>com.yihu.jw</groupId>
            <artifactId>common-util</artifactId>
            <artifactId>common-rest-model</artifactId>
        </dependency>
        <dependency>
            <groupId>com.yihu.jw</groupId>
            <artifactId>common-rest-model</artifactId>
            <artifactId>common-swagger</artifactId>
        </dependency>
        <dependency>
            <groupId>com.yihu.jw</groupId>
@@ -112,9 +112,5 @@
            <groupId>org.json</groupId>
            <artifactId>json</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springside</groupId>
            <artifactId>springside-core</artifactId>
        </dependency>
    </dependencies>
</project>

svr/svr-base/src/main/java/com/yihu/jw/wx/service/WxAccessTokenService.java

@@ -13,8 +13,8 @@ import org.json.JSONObject;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import org.springside.modules.utils.Clock;
import java.util.Date;
import java.util.List;
import java.util.UUID;
@@ -74,7 +74,7 @@ public class WxAccessTokenService extends BaseJpaService<WxAccessToken, WxAccess
                newaccessToken.setAccessToken(token);
                newaccessToken.setExpiresIn(Long.parseLong(expires_in));
                newaccessToken.setAddTimestamp(System.currentTimeMillis());
                newaccessToken.setCzrq(new Clock.DefaultClock().getCurrentDate());
                newaccessToken.setCzrq(new Date());
                newaccessToken.setCode(UUID.randomUUID().toString().replace("-",""));
                newaccessToken.setWechatCode(wechatCode);
                wxAccessTokenDao.save(newaccessToken);

svr/svr-quota/pom.xml

@@ -15,7 +15,6 @@
    <version>1.0.0</version>
    <dependencies>
        <dependency>
            <groupId>com.yihu.jw</groupId>
            <artifactId>common-swagger</artifactId>
@@ -101,5 +100,11 @@
            <artifactId>jna</artifactId>
            <version>4.2.2</version>
        </dependency>
        <dependency>
            <groupId>io.searchbox</groupId>
            <artifactId>jest</artifactId>
        </dependency>
    </dependencies>
</project>

svr/svr-quota/src/main/java/com/yihu/jw/quota/etl/Contant.java

@@ -98,6 +98,11 @@ public class Contant {
        public static final String age = "2";//年龄段
    }
    public static class save_status {
        public static final String success = "1";
        public static final String fail = "0";
    }
    /**
     * 数据过滤用到的参量
     */

svr/svr-quota/src/main/java/com/yihu/jw/quota/etl/model/EsConfig.java

@@ -6,8 +6,6 @@ package com.yihu.jw.quota.etl.model;
public class EsConfig {
    private String host;//地址
    private String port;//端口
    private String clusterName;//集群的名字
    private String index;// 索引 es相当与数据库
    private String type;// 类型 es 相当于表
@@ -19,22 +17,6 @@ public class EsConfig {
        this.host = host;
    }
    public String getPort() {
        return port;
    }
    public void setPort(String port) {
        this.port = port;
    }
    public String getClusterName() {
        return clusterName;
    }
    public void setClusterName(String clusterName) {
        this.clusterName = clusterName;
    }
    public String getIndex() {
        return index;
    }
@@ -50,4 +32,6 @@ public class EsConfig {
    public void setType(String type) {
        this.type = type;
    }
}

svr/svr-quota/src/main/java/com/yihu/jw/quota/etl/save/SaveHelper.java

@@ -22,7 +22,7 @@ public class SaveHelper {
    @Autowired
    private TjDataSaveService datsSaveService;
    public void save(List<SaveModel> sms, QuotaVO quotaVO) {
    public Boolean save(List<SaveModel> sms, QuotaVO quotaVO) {
        //查看指标保存的数据源
        TjQuotaDataSave quotaDataSave = datsSaveService.findByQuota(quotaVO.getCode());
        switch (quotaDataSave.getType()) {
@@ -33,9 +33,9 @@ public class SaveHelper {
                break;
            }
            case Contant.save.es: {
                SpringUtil.getBean(ElastricSearchSave.class).save(sms,quotaDataSave.getJsonConfig());
                break;
                return SpringUtil.getBean(ElastricSearchSave.class).save(sms,quotaDataSave.getJsonConfig());
            }
        }
        return false;
    }
}

svr/svr-quota/src/main/java/com/yihu/jw/quota/etl/save/es/ElasticFactory.java

@@ -0,0 +1,30 @@
package com.yihu.jw.quota.etl.save.es;
import io.searchbox.client.JestClient;
import io.searchbox.client.JestClientFactory;
import io.searchbox.client.config.ClientConfig;
import io.searchbox.client.config.HttpClientConfig;
import org.springframework.stereotype.Component;
import java.util.concurrent.TimeUnit;
/**
 * Created by chenweida on 2017/6/5.
 */
@Component
public class ElasticFactory {
    /**
     * @param hostAddress "http://localhost:9200"
     * @return
     */
    public JestClient getClient(String hostAddress) {
        // Construct a new Jest client according to configuration via factory
        JestClientFactory factory = new JestClientFactory();
        factory.setHttpClientConfig(new HttpClientConfig
                .Builder(hostAddress)
                .multiThreaded(true)
                .discoveryEnabled(true)
                .build());
        return factory.getObject();
    }
}

svr/svr-quota/src/main/java/com/yihu/jw/quota/etl/save/es/ElastricSearchSave.java

@@ -2,22 +2,18 @@ package com.yihu.jw.quota.etl.save.es;
import com.yihu.jw.quota.etl.model.EsConfig;
import com.yihu.jw.quota.vo.SaveModel;
import net.sf.json.JSONObject;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import io.searchbox.client.JestClient;
import io.searchbox.core.Bulk;
import io.searchbox.core.BulkResult;
import io.searchbox.core.Index;
import net.sf.json.JSONObject;;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Scope;
import org.springframework.data.elasticsearch.core.ElasticsearchTemplate;
import org.springframework.data.elasticsearch.core.query.IndexQuery;
import org.springframework.data.elasticsearch.core.query.IndexQueryBuilder;
import org.springframework.stereotype.Component;
import java.net.InetAddress;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
 * Created by chenweida on 2017/6/2.
@@ -26,47 +22,30 @@ import java.util.Map;
@Scope("prototype")
public class ElastricSearchSave {
    private ElasticsearchTemplate elasticsearchTemplate;
    private EsConfig esConfig;
    public void save(List<SaveModel> sms, String jsonConfig) {
        //初始化链接
        initTemplate(jsonConfig);
        //判断索引是否存在
//        if (!elasticsearchTemplate.indexExists(esConfig.getIndex())) {
//            //不存在就新增缩影
//            Map<String, String> setting = new HashMap<>();
//            setting.put("number_of_replicas", "2");// 设置备份
//            setting.put("number_of_shards", "6");//设置分片
//            elasticsearchTemplate.createIndex(esConfig.getIndex(), setting);
//        }
//        //判断type是否存在
//        if(!elasticsearchTemplate.typeExists(esConfig.getIndex(),esConfig.getType())){
//
//        }
    private Logger logger = LoggerFactory.getLogger(ElastricSearchSave.class);
    @Autowired
    private ElasticFactory elasticFactory;
        List<IndexQuery> queries = new ArrayList<IndexQuery>();
        for (SaveModel saveModel : sms) {
            IndexQuery indexQuery = new IndexQueryBuilder().withObject(saveModel).build();
            queries.add(indexQuery);
        }
        elasticsearchTemplate.bulkIndex(queries);
    }
    private EsConfig esConfig;
    private void initTemplate(String jsonConfig) {
    public Boolean save(List<SaveModel> sms, String jsonConfig) {
        try {
            //初始化参数
            esConfig = (EsConfig) JSONObject.toBean(JSONObject.fromObject(jsonConfig), EsConfig.class);
            Settings esSettings = Settings.settingsBuilder()
                    .put("cluster.name", esConfig.getClusterName())
                    .build();
            Client client = TransportClient.builder()
                    .settings(esSettings)
                    .build()
                    .addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(esConfig.getHost()), Integer.valueOf(esConfig.getPort())));
            elasticsearchTemplate = new ElasticsearchTemplate(client);
            //得到链接
            JestClient jestClient = elasticFactory.getClient(esConfig.getHost());
            Bulk.Builder bulk = new Bulk.Builder().defaultIndex(esConfig.getIndex()).defaultType(esConfig.getType());
            for (SaveModel obj : sms) {
                Index index = new Index.Builder(obj).build();
                bulk.addAction(index);
            }
            BulkResult br = jestClient.execute(bulk.build());
            return br.isSucceeded();
        } catch (Exception e) {
            e.printStackTrace();
            logger.error(" save error ：" + e.getMessage());
        }
        return null;
    }
}

svr/svr-quota/src/main/java/com/yihu/jw/quota/job/QuotaJob.java

@@ -99,13 +99,13 @@ public class QuotaJob implements Job {
            //得到算法
            TjCompute compute = computeService.findByQuotaCode(quotaVO.getCode());
            quotaVO.setComputeType(compute.getType());
        }else{
        } else {
            logger.error("quotaVO is null");
            return;
        }
        this.timeLevel= (String) map.get("timeLevel");
        if(StringUtils.isEmpty(this.timeLevel)){
            this.timeLevel=Contant.main_dimension_timeLevel.day;
        this.timeLevel = (String) map.get("timeLevel");
        if (StringUtils.isEmpty(this.timeLevel)) {
            this.timeLevel = Contant.main_dimension_timeLevel.day;
        }
    }
@@ -114,11 +114,11 @@ public class QuotaJob implements Job {
     */
    private void computequota() {
        try {
            TjQuotaLog tjQuotaLog=new TjQuotaLog();
            TjQuotaLog tjQuotaLog = new TjQuotaLog();
            tjQuotaLog.setQuotaCode(quotaVO.getCode());
            tjQuotaLog.setSaasId(saasid);
            tjQuotaLog.setStartTime(new Date());
            JobLogModel jobLogModel=new JobLogModel();
            JobLogModel jobLogModel = new JobLogModel();
            //抽取数据 如果是累加就是 List<DataModel>  如果是相除 Map<String,List<DataModel>>
            Object dataModels = extract();
            //根据规则过滤数据
@@ -127,7 +127,8 @@ public class QuotaJob implements Job {
            //统计数据
            List<SaveModel> sms = compute(filterModel.getData());
            //保存数据
            saveDate(sms);
            Boolean success = saveDate(sms);
            tjQuotaLog.setStatus(success ? Contant.save_status.success : Contant.save_status.fail);
            tjQuotaLog.setEndTime(new Date());
            tjQuotaLog.setContent(JSONObject.fromObject(jobLogModel).toString());
        } catch (Exception e) {
@@ -137,14 +138,16 @@ public class QuotaJob implements Job {
    /**
     * 保存数据
     *
     * @param sms
     */
    private void saveDate(List<SaveModel> sms) {
    private Boolean saveDate(List<SaveModel> sms) {
        try {
           SpringUtil.getBean(SaveHelper.class).save(sms, quotaVO);
            return SpringUtil.getBean(SaveHelper.class).save(sms, quotaVO);
        } catch (Exception e) {
            logger.error("save error:"+e.getMessage());
            logger.error("save error:" + e.getMessage());
        }
        return false;
    }
    /**
@@ -154,9 +157,9 @@ public class QuotaJob implements Job {
     */
    private List<SaveModel> compute(Object dataModels) {
        try {
            return SpringUtil.getBean(ComputeHelper.class).compute(dataModels, quotaVO,timeLevel,saasid,startTime);
            return SpringUtil.getBean(ComputeHelper.class).compute(dataModels, quotaVO, timeLevel, saasid, startTime);
        } catch (Exception e) {
            logger.error("compute error:"+e.getMessage());
            logger.error("compute error:" + e.getMessage());
        }
        return null;
    }
@@ -171,7 +174,7 @@ public class QuotaJob implements Job {
        try {
            return SpringUtil.getBean(FilterHelper.class).filter(dataModels, quotaVO);
        } catch (Exception e) {
            logger.error("filter error:"+e.getMessage());
            logger.error("filter error:" + e.getMessage());
        }
        return null;
    }
@@ -185,7 +188,7 @@ public class QuotaJob implements Job {
        try {
            return SpringUtil.getBean(ExtractHelper.class).extractData(quotaVO, startTime, endTime);
        } catch (Exception e) {
            logger.error("extract error:"+e.getMessage());
            logger.error("extract error:" + e.getMessage());
        }
        return null;
    }