采集终端mongo数据至中心统一添加create_time规范

hos-arbiter/src/main/java/com/yihu/hos/arbiter/controllers/ArbiterServerController.java

@@ -3,12 +3,14 @@ package com.yihu.hos.arbiter.controllers;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.yihu.hos.arbiter.models.ArbiterServer;
import com.yihu.hos.arbiter.services.ArbiterServerService;
import com.yihu.hos.core.datatype.DateUtil;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.io.IOException;
import java.util.Date;
/**
 * @author Airhead
@@ -26,6 +28,7 @@ public class ArbiterServerController {
    public void register(@RequestBody String body) {
        try {
            ArbiterServer arbiterServer = objectMapper.readValue(body, ArbiterServer.class);
            arbiterServer.setCreateTime(DateUtil.getSysDateTime());
            arbiterServerService.save(arbiterServer);
        } catch (IOException e) {
            e.printStackTrace();

hos-arbiter/src/main/java/com/yihu/hos/arbiter/controllers/BrokerServerController.java

@@ -4,6 +4,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.yihu.hos.arbiter.models.BrokerServer;
import com.yihu.hos.arbiter.services.BrokerServerService;
import com.yihu.hos.core.datatype.DateUtil;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
@@ -26,7 +27,7 @@ public class BrokerServerController {
        try {
            ObjectMapper objectMapper = new ObjectMapper();
            BrokerServer server = objectMapper.readValue(brokerServer, BrokerServer.class);
            server.setUpdateTime(new Date());
            server.setCreateTime(DateUtil.getSysDateTime());
            brokerServerService.save(server);
        } catch (IOException e) {
            e.printStackTrace();
@@ -55,7 +56,6 @@ public class BrokerServerController {
        try {
            ObjectMapper objectMapper = new ObjectMapper();
            BrokerServer server = objectMapper.readValue(brokerServer, BrokerServer.class);
            server.setUpdateTime(new Date());
            brokerServerService.delete(server);
        } catch (IOException e) {
            e.printStackTrace();

hos-arbiter/src/main/java/com/yihu/hos/arbiter/controllers/EndpointController.java

@@ -3,12 +3,14 @@ package com.yihu.hos.arbiter.controllers;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.yihu.hos.arbiter.models.Endpoint;
import com.yihu.hos.arbiter.services.EndpointService;
import com.yihu.hos.core.datatype.DateUtil;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RestController;
import java.io.IOException;
import java.util.Date;
/**
 * @created Airhead 2016/7/27.
@@ -24,6 +26,7 @@ public class EndpointController {
        try {
            ObjectMapper objectMapper = new ObjectMapper();
            Endpoint endpoint = objectMapper.readValue(service, Endpoint.class);
            endpoint.setCreateTime(DateUtil.getSysDateTime());
            endpointService.save(endpoint);
        } catch (IOException e) {
            e.printStackTrace();

hos-arbiter/src/main/java/com/yihu/hos/arbiter/models/ArbiterServer.java

@@ -1,9 +1,12 @@
package com.yihu.hos.arbiter.models;
import com.yihu.hos.core.datatype.DateUtil;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.index.Indexed;
import org.springframework.data.mongodb.core.mapping.Document;
import java.util.Date;
/**
 * @author Airhead
 * @since 2016/12/9.
@@ -15,8 +18,11 @@ public class ArbiterServer {
    @Indexed
    private String tenant;
    private String url;
    private Date createTime;
    private Date updateTime;
    public ArbiterServer() {
        this.createTime = DateUtil.getSysDateTime();
    }
    public String getId() {
@@ -42,4 +48,20 @@ public class ArbiterServer {
    public void setUrl(String url) {
        this.url = url;
    }
    public Date getCreateTime() {
        return createTime;
    }
    public void setCreateTime(Date createTime) {
        this.createTime = createTime;
    }
    public Date getUpdateTime() {
        return updateTime;
    }
    public void setUpdateTime(Date updateTime) {
        this.updateTime = updateTime;
    }
}

hos-arbiter/src/main/java/com/yihu/hos/arbiter/models/BrokerServer.java

@@ -1,5 +1,6 @@
package com.yihu.hos.arbiter.models;
import com.yihu.hos.core.datatype.DateUtil;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.index.Indexed;
import org.springframework.data.mongodb.core.mapping.Document;
@@ -22,8 +23,13 @@ public class BrokerServer {
    private boolean registered;
    @Indexed(name = "updateTime_1", expireAfterSeconds = 30)
    private Date updateTime;
    private Date createTime;
    private ArrayList<ServiceFlow> serviceFlows;
    public BrokerServer() {
        this.createTime = DateUtil.getSysDateTime();
    }
    public boolean isFlowOn(String routeCode) {
        if (serviceFlows == null) {
            return false;
@@ -62,6 +68,14 @@ public class BrokerServer {
        this.port = port;
    }
    public Date getCreateTime() {
        return createTime;
    }
    public void setCreateTime(Date createTime) {
        this.createTime = createTime;
    }
    public Date getUpdateTime() {
        return updateTime;
    }

hos-arbiter/src/main/java/com/yihu/hos/arbiter/models/Endpoint.java

@@ -1,5 +1,6 @@
package com.yihu.hos.arbiter.models;
import com.yihu.hos.core.datatype.DateUtil;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.index.Indexed;
import org.springframework.data.mongodb.core.mapping.Document;
@@ -22,6 +23,11 @@ public class Endpoint {
    private String healthCheckURL;
    private Integer metricsType;
    private String metricsURL;
    private Date createTime;
    public Endpoint() {
        this.createTime = DateUtil.getSysDateTime();
    }
    public String getId() {
        return id;
@@ -47,6 +53,14 @@ public class Endpoint {
        this.endpoint = endpoint;
    }
    public Date getCreateTime() {
        return createTime;
    }
    public void setCreateTime(Date createTime) {
        this.createTime = createTime;
    }
    public Date getUpdateTime() {
        return updateTime;
    }

hos-broker/src/main/java/com/yihu/hos/broker/common/camelrouter/CenterMongoRouter.java

@@ -1,72 +0,0 @@
package com.yihu.hos.broker.common.camelrouter;
import org.apache.camel.Exchange;
import org.apache.camel.builder.RouteBuilder;
import org.bson.types.ObjectId;
import org.json.JSONArray;
import org.springframework.stereotype.Component;
import java.util.HashMap;
import java.util.Map;
@Component
public class CenterMongoRouter extends RouteBuilder {
    static public final String DESTINATION_NAME = "business-log";
    @Override
    public void configure() throws Exception {
//        from("quartz://myGroup/myTimerName?cron=0 0 0 1 /1 * ?")
//            .setBody().constant("{ \"flowType\": \"class\" }")
//            .to("mongodb:mongo?database=runtime&collection=serviceFlow&operation=findOneByQuery")
//            .split(simple("${body}"))
//            .process(new Processor() {
//                @Override
//                public void process(Exchange exchange) throws Exception {
//                    addMongoInfo(exchange, "runtime", "serviceFlow");
//                }
//            }).to("bean:centerMongoService?method=log")
//
//            .setBody().constant("{ \"tenant\": \"yichang\" }")
//            .to("mongodb:mongo?database=runtime&collection=arbiterServer&operation=findOneByQuery")
//            .split(simple("${body}"))
//            .process(new Processor() {
//                @Override
//                public void process(Exchange exchange) throws Exception {
//                    addMongoInfo(exchange, "runtime", "arbiterServer");
//                }
//            }).to("bean:centerMongoService?method=log");
//
//            .setBody().constant("{ \"flowType\": \"class\" }")
//            .to("mongodb:mongo?database=runtime&collection=brokerServer&operation=findOneByQuery")
//            .split(simple("${body}"))
//            .process(new Processor() {
//                @Override
//                public void process(Exchange exchange) throws Exception {
//                    addMongoInfo(exchange, "runtime", "serviceFlow");
//                }
//            }).to("bean:centerMongoService?method=log")
//
//            .setBody().constant("{ \"flowType\": \"class\" }")
//            .to("mongodb:mongo?database=runtime&collection=serviceFlow&operation=findOneByQuery")
//            .split(simple("${body}"))
//            .process(new Processor() {
//                @Override
//                public void process(Exchange exchange) throws Exception {
//                    addMongoInfo(exchange, "runtime", "serviceFlow");
//                }
//            }).to("bean:centerMongoService?method=log");
    }
    public void addMongoInfo(Exchange exchange, String database, String collection) {
        Map record = exchange.getIn().getBody(Map.class);
        Map info = new HashMap();
        info.put("database", database);
        info.put("collection", collection);
        ObjectId objectId = (ObjectId) record.get("_id");
        record.put("_id", objectId.toString());
        JSONArray jsonArray = new JSONArray();
        jsonArray.put(info);
        jsonArray.put(record);
        exchange.getIn().setBody(jsonArray);
    }
}

hos-broker/src/main/java/com/yihu/hos/broker/common/camelrouter/CrawlerMongoRouter.java

@@ -0,0 +1,40 @@
package com.yihu.hos.broker.common.camelrouter;
import com.yihu.hos.broker.common.processor.CrawlerMongoProcessor;
import org.apache.camel.builder.RouteBuilder;
import org.springframework.stereotype.Component;
@Component
public class CrawlerMongoRouter extends RouteBuilder {
    @Override
    public void configure() throws Exception {
        from("quartz://myGroup/myTimerName?cron=0 0/1 * * * ? ")
            .process(new CrawlerMongoProcessor())
                .to("stream:out");
//            .process(new CrawlerMongoProcessor())
//            .to("mongodb:mongo?database=runtime&collection=arbiterServer&operation=findOneByQuery")
//            .split(simple("${body}"))
//            .process(new AddMongoProcessor("runtime", "arbiterServer"))
//            .to("bean:centerMongoService?method=save")
//            .process(new CrawlerMongoProcessor())
//            .to("mongodb:mongo?database=runtime&collection=brokerServer&operation=findOneByQuery")
//            .split(simple("${body}"))
//            .process(new AddMongoProcessor("runtime", "brokerServer"))
//            .to("bean:centerMongoService?method=save")
//            .process(new CrawlerMongoProcessor())
//            .to("mongodb:mongo?database=runtime&collection=endpoint&operation=findOneByQuery")
//            .split(simple("${body}"))
//            .process(new AddMongoProcessor("runtime", "endpoint"))
//            .to("bean:centerMongoService?method=save")
//            .process(new CrawlerMongoProcessor())
//            .to("mongodb:mongo?database=log&collection=server&operation=findOneByQuery")
//            .split(simple("${body}"))
//            .process(new AddMongoProcessor("log", "server"))
//            .to("bean:centerMongoService?method=save")
//            .process(new CrawlerMongoProcessor())
//            .to("mongodb:mongo?database=log&collection=service&operation=findOneByQuery")
//            .split(simple("${body}"))
//            .process(new AddMongoProcessor("log", "service"))
//            .to("bean:centerMongoService?method=save");
    }
}

hos-broker/src/main/java/com/yihu/hos/broker/common/processor/AddMongoProcessor.java

@@ -0,0 +1,41 @@
package com.yihu.hos.broker.common.processor;
import com.mongodb.BasicDBObject;
import com.mongodb.QueryOperators;
import com.yihu.hos.core.datatype.DateUtil;
import org.apache.camel.Exchange;
import org.apache.camel.Message;
import org.apache.camel.Processor;
import org.bson.types.ObjectId;
import org.json.JSONArray;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
/**
 * Created by Zdm on 2016/7/13.
 */
public class AddMongoProcessor implements Processor {
    private String database;
    private String collection;
    public AddMongoProcessor(String database, String collection) {
        this.database = database;
        this.collection = collection;
    }
    @Override
    public void process(Exchange exchange) throws Exception {
        Map record = exchange.getIn().getBody(Map.class);
        Map info = new HashMap();
        info.put("database", database);
        info.put("collection", collection);
        ObjectId objectId = (ObjectId) record.get("_id");
        record.put("_id", objectId.toString());
        JSONArray jsonArray = new JSONArray();
        jsonArray.put(info);
        jsonArray.put(record);
        exchange.getIn().setBody(jsonArray);
    }
}

hos-broker/src/main/java/com/yihu/hos/broker/common/processor/CrawlerMongoProcessor.java

@@ -0,0 +1,31 @@
package com.yihu.hos.broker.common.processor;
import com.mongodb.BasicDBObject;
import com.mongodb.QueryOperators;
import com.yihu.hos.core.datatype.DateUtil;
import org.apache.camel.Exchange;
import org.apache.camel.Message;
import org.apache.camel.Processor;
import java.util.Date;
/**
 * Created by Zdm on 2016/7/13.
 */
public class CrawlerMongoProcessor implements Processor {
    @Override
    public void process(Exchange exchange) throws Exception {
        Message out = exchange.getOut();
        Date now = new Date();
        Date beforeDate = new Date(now.getTime() - 60000);
        String beginTime = DateUtil.toString(beforeDate, DateUtil.DEFAULT_YMDHMSDATE_FORMAT);
        String endTime = DateUtil.toString(now, DateUtil.DEFAULT_YMDHMSDATE_FORMAT);
        BasicDBObject queryObject = new BasicDBObject().append(QueryOperators.AND,
                new BasicDBObject[]{
                        new BasicDBObject().append("create_time",
                                new BasicDBObject().append(QueryOperators.GTE, beginTime)),
                        new BasicDBObject().append("create_time",
                                new BasicDBObject().append(QueryOperators.LT, endTime))});
        out.setBody(queryObject.toJson());
    }
}

hos-broker/src/main/java/com/yihu/hos/broker/models/BusinessLog.java

@@ -1,5 +1,6 @@
package com.yihu.hos.broker.models;
import com.yihu.hos.core.datatype.DateUtil;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.index.Indexed;
import org.springframework.data.mongodb.core.mapping.Document;
@@ -28,6 +29,12 @@ public class BusinessLog {
    private String body;
    private Integer bodyLength;
    private String fireTimeSource;
    private Date updateTime;
    private Date createTime;
    public BusinessLog() {
        this.createTime = DateUtil.getSysDateTime();
    }
    //@Indexed(name = "fireTime_1", expireAfterSeconds = 30)
    @Indexed
@@ -144,4 +151,20 @@ public class BusinessLog {
    public void setOrder(String order) {
        this.order = order;
    }
    public Date getUpdateTime() {
        return updateTime;
    }
    public void setUpdateTime(Date updateTime) {
        this.updateTime = updateTime;
    }
    public Date getCreateTime() {
        return createTime;
    }
    public void setCreateTime(Date createTime) {
        this.createTime = createTime;
    }
}

hos-broker/src/main/java/com/yihu/hos/broker/services/BusinessLogService.java

@@ -27,6 +27,7 @@ public class BusinessLogService {
            String fireTimeSource = businessLog.getFireTimeSource();
            businessLog.setFireTime(DateUtil.toTimestamp(fireTimeSource));
            businessLog.setBodyLength(businessLog.getBody().length());
            businessLog.setCreateTime(DateUtil.getSysDateTime());
            mongoOperations.save(businessLog);
        } catch (IOException e) {
            e.printStackTrace();

hos-broker/src/main/java/com/yihu/hos/broker/services/CenterMongoService.java

@@ -15,7 +15,7 @@ public class CenterMongoService {
    @Value("${hos.rest.url}")
    private String url;
    public void log(String msg) {
    public void save(String msg) {
        Map<String, String> params = new HashMap<>();
        params.put("msg", msg);

hos-broker/src/main/java/com/yihu/hos/broker/services/ServerMonitorService.java

@@ -13,9 +13,6 @@ import org.springframework.stereotype.Service;
import java.util.Date;
import java.util.List;
/**
 * Created by chenweida on 2016/1/27.
 */
@Service("ServerMonitorService")
public class ServerMonitorService {
    public static final String BEAN_ID = "ServerMonitorService";
@@ -37,7 +34,7 @@ public class ServerMonitorService {
            DBCollection terminal = mongo.getDB(MonitorConstant.MONITOR_DATABASE).getCollection(MonitorConstant.SERVER);
            result.put("tenant", tenant);
            result.put("create_date", DateUtil.getCurrentString(DateUtil.DEFAULT_YMDHMSDATE_FORMAT));
            result.put("create_time", new Date());
            result.put("create_time", DateUtil.getSysDateTime());
            result.put("host", host);
            //cpu

hos-broker/src/main/java/com/yihu/hos/broker/services/ServiceMonitorService.java

@@ -60,7 +60,7 @@ public class ServiceMonitorService {
            bandwidth(beginTime, endTime, codeList);
            qps(beginTime, endTime, codeList);
//            delay(beginTime, endTime, codeList);
            delay(beginTime, endTime, codeList);
            usage(beginTime, endTime, codeList);
        } catch (Exception e) {
            e.printStackTrace();
@@ -270,7 +270,6 @@ public class ServiceMonitorService {
        Map<String, BigDecimal> delayMapF = new HashMap<>();
        for (DBObject dbObject : flowOutput.results()) {
            Integer count = Integer.parseInt(StringUtil.toString(dbObject.get("count")));
            BasicDBObject id = (BasicDBObject) dbObject.get("_id");
            String code = StringUtil.toString(id.get("routeId"));
            String begin = StringUtil.toString(dbObject.get("beginTime"));
@@ -384,7 +383,7 @@ public class ServiceMonitorService {
        document.put("name", name);
        document.put("type", type);
        document.put("value", value);
        document.put("createTime", createTime);
        document.put("create_time", createTime);
        DBCollection terminal = mongo.getDB(MonitorConstant.MONITOR_DATABASE).getCollection(MonitorConstant.SERVICE);
        terminal.save(document);
    }

src/main/java/com/yihu/hos/monitor/service/ServiceMonitorService.java

@@ -75,9 +75,9 @@ public class ServiceMonitorService {
        DBCollection metrics = mongoOperations.getCollection(tableName);
        BasicDBObject queryObject = new BasicDBObject().append(QueryOperators.AND,
                new BasicDBObject[]{
                        new BasicDBObject().append("createTime",
                        new BasicDBObject().append("create_time",
                                new BasicDBObject().append(QueryOperators.GTE, beginTime)),
                        new BasicDBObject().append("createTime",
                        new BasicDBObject().append("create_time",
                                new BasicDBObject().append(QueryOperators.LT, endTime)),
                        new BasicDBObject("name", code),
                        new BasicDBObject("tenant", tenantSession.getTenant())});