bug提交

patient-co-statistics-es/doc/es/统计新增索引.txt

@@ -0,0 +1,100 @@
创建索引
POST  http://172.19.103.68:9200/wlyy_quota_test
查询索引
GET http://172.19.103.68:9200/wlyy_quota_test/
给索引加mapping
POST http://172.19.103.68:9200/wlyy_quota_test/wlyy_quota_test/_mapping
{
    "wlyy_quota_test": {
            "properties": {
                "quotaCode": {
                    "type": "string",
                    "index": "not_analyzed"
                },
                "city": {
                    "type": "string",
                    "index": "not_analyzed"
                },
                "cityName": {
                    "type": "string",
                    "index": "not_analyzed"
                },
                "town": {
                    "type": "string",
                    "index": "not_analyzed"
                },
                "townName": {
                    "type": "string",
                    "index": "not_analyzed"
                },
                "hospital": {
                    "type": "string",
                    "index": "not_analyzed"
                },
                "hospitalName": {
                    "type": "string",
                    "index": "not_analyzed"
                },
                "team": {
                    "type": "string",
                    "index": "not_analyzed"
                },
                "teamName": {
                    "type": "string",
                    "index": "not_analyzed"
                },
                "slaveKey1": {
                    "type": "string",
                    "index": "not_analyzed"
                },
                "slaveKey1Name": {
                    "type": "string",
                    "index": "not_analyzed"
                },
                "slaveKey2": {
                    "type": "string",
                    "index": "not_analyzed"
                },
                "slaveKey2Name": {
                    "type": "string",
                    "index": "not_analyzed"
                },
                "slaveKey3": {
                    "type": "string",
                    "index": "not_analyzed"
                },
                "slaveKey3Name": {
                    "type": "string",
                    "index": "not_analyzed"
                },
                "slaveKey4": {
                    "type": "string",
                    "index": "not_analyzed"
                },
                "slaveKey4Name": {
                    "type": "string",
                    "index": "not_analyzed"
                },
                "areaLevel": {
                    "type": "string",
                    "index": "not_analyzed"
                }, 
               "timeLevel": {
                    "type": "string",
                    "index": "not_analyzed"
                },
                "result1": {
                    "type": "integer"
                },
                "result2": {
                    "type": "integer"
                },
                "quotaDate": {
                    "type": "date"
                },
                "createTime": {
                    "type": "date"
                }
            }
        }
  }

patient-co-statistics-es/pom.xml

@@ -419,6 +419,16 @@
            <version>2.4</version>
        </dependency>
        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>druid</artifactId>
            <version>1.0.15</version>
        </dependency>
        <dependency>
            <groupId>org.nlpcn</groupId>
            <artifactId>elasticsearch-sql</artifactId>
            <version>2.4.1.0</version>
        </dependency>
    </dependencies>
    <build>

patient-co-statistics-es/src/main/java/com/yihu/wlyy/statistics/controller/QueryController.java

@@ -0,0 +1,96 @@
package com.yihu.wlyy.statistics.controller;
import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.expr.SQLQueryExpr;
import com.alibaba.druid.sql.parser.ParserException;
import com.alibaba.druid.sql.parser.SQLExprParser;
import com.alibaba.druid.sql.parser.Token;
import com.yihu.wlyy.statistics.etl.save.es.ElasticFactory;
import com.yihu.wlyy.statistics.model.job.ElasticsearchService;
import com.yihu.wlyy.statistics.util.DateUtil;
import com.yihu.wlyy.statistics.vo.SaveModel;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
import org.elasticsearch.action.search.SearchResponse;
import org.nlpcn.es4sql.jdbc.ObjectResult;
import org.nlpcn.es4sql.jdbc.ObjectResultsExtractor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.util.StringUtils;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.nlpcn.es4sql.domain.Select;
import org.nlpcn.es4sql.parse.ElasticSqlExprParser;
import org.nlpcn.es4sql.parse.SqlParser;
import org.nlpcn.es4sql.query.AggregationQueryAction;
import org.nlpcn.es4sql.query.DefaultQueryAction;
import org.nlpcn.es4sql.query.SqlElasticSearchRequestBuilder;
import java.util.Date;
import java.util.List;
/**
 * Created by chenweida on 2017/7/1.
 */
@RestController
@RequestMapping("/quotaQuery")
@Api(description = "指标查询")
public class QueryController {
    @Autowired
    private ElasticsearchService elasticsearchService;
    @Autowired
    private ElasticFactory elasticFactory;
    @ApiOperation(value = "查询某几个指标某个层级的列表")
    @RequestMapping(value = "/getQuotasList", method = RequestMethod.GET)
    public String getQuotasList(
            @ApiParam(name = "ids", value = "指标id,多个逗号分割", required = true) @RequestParam(value = "ids", required = true) String ids,
            @ApiParam(name = "code", value = "指标的code", required = true) @RequestParam(value = "code", required = true) String code,
            @ApiParam(name = "arealevel", value = "指标的level(1 省 2 市 3 区县 4 机构 5团队)", required = true) @RequestParam(value = "arealevel", required = true) String arealevel,
            @ApiParam(name = "timeLevel", value = " 1增量的指标  2到达量的指标", required = true) @RequestParam(value = "timeLevel", required = true) String timeLevel,
            @ApiParam(name = "date", value = "时间(yyyy-MM-dd),不传默认查今天", required = false) @RequestParam(value = "date", required = false) String date) {
        //如果日期是空初始化是今天
        if (StringUtils.isEmpty(date)) {
            date = DateUtil.dateToStrShort(new Date());
        }
        return null;
    }
    @ApiOperation(value = "查询某个指标某个层级的折线图")
    @RequestMapping(value = "/getQuotasLine", method = RequestMethod.GET)
    public String getQuotasLine(
            @ApiParam(name = "id", value = "指标id", required = true) @RequestParam(value = "id", required = true) String id,
            @ApiParam(name = "code", value = "指标的code", required = true) @RequestParam(value = "code", required = true) String code,
            @ApiParam(name = "arealevel", value = "指标的level(1 省 2 市 3 区县 4 机构 5团队)", required = true) @RequestParam(value = "arealevel", required = true) String arealevel,
            @ApiParam(name = "timeLevel", value = " 1增量的指标  2到达量的指标", required = true) @RequestParam(value = "timeLevel", required = true) String timeLevel,
            @ApiParam(name = "startDate", value = "时间(yyyy-MM-dd)", required = false) @RequestParam(value = "startDate", required = true) String startDate,
            @ApiParam(name = "endDate", value = "时间(yyyy-MM-dd),不传默认查今天", required = false) @RequestParam(value = "endDate", required = false) String endDate) {
        //如果日期是空初始化是今天
        if (StringUtils.isEmpty(endDate)) {
            endDate = DateUtil.dateToStrShort(new Date());
        }
        return null;
    }
    @ApiOperation(value = "查询某几个指标的到达量")
    @RequestMapping(value = "/getQuotas", method = RequestMethod.GET)
    public String getQuotas(
            @ApiParam(name = "ids", value = "指标id,多个逗号分割", required = true) @RequestParam(value = "ids", required = true) String ids,
            @ApiParam(name = "code", value = "指标的code", required = true) @RequestParam(value = "code", required = true) String code,
            @ApiParam(name = "arealevel", value = "指标的level(1 省 2 市 3 区县 4 机构 5团队)", required = true) @RequestParam(value = "arealevel", required = true) String arealevel,
            @ApiParam(name = "date", value = "时间(yyyy-MM-dd),不传默认查今天", required = false) @RequestParam(value = "date", required = false) String date) {
        return null;
    }
    @RequestMapping(value = "/test", method = RequestMethod.GET)
    public List<SaveModel> getQuotas()throws Exception {
        String sql="SELECT city,cityName,sum(result1) result1 FROM wlyy_quota_test group by town,townName";
        return elasticsearchService.excute(sql);
    }
}

patient-co-statistics-es/src/main/java/com/yihu/wlyy/statistics/etl/save/es/ElasticFactory.java

@@ -3,8 +3,16 @@ package com.yihu.wlyy.statistics.etl.save.es;
import io.searchbox.client.JestClient;
import io.searchbox.client.JestClientFactory;
import io.searchbox.client.config.HttpClientConfig;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;
import java.net.InetAddress;
import java.net.UnknownHostException;
/**
 * Created by chenweida on 2017/6/5.
@@ -15,12 +23,19 @@ public class ElasticFactory {
    @Value("${es.host}")
    private String esHost;
    @Value("${es.port}")
    private String port;
    @Value("${es.tPort}")
    private String tPort;
    @Value("${es.clusterName}")
    private String clusterName;
//-----------------------------------jestClient----------------------------------------
    /**
     * @param "http://localhost:9200"
     * @return
     */
    public JestClient getClient() {
    public JestClient getJestClient() {
        if (factory == null) {
            //初始化链接
            init();
@@ -35,10 +50,35 @@ public class ElasticFactory {
        // Construct a new Jest client according to configuration via factory
        factory = new JestClientFactory();
        factory.setHttpClientConfig(new HttpClientConfig
                .Builder("http://" + esHost)
                .Builder("http://" + esHost + ":" + port)
                .multiThreaded(true)
                .discoveryEnabled(true)
                .readTimeout(30000)//30秒
                .build());//得到链接
    }
    //-----------------------------------TransportClient----------------------------------------
    private Client transportClient;
    public Client getTransportClient() {
        try {
            initTranClient();
            return transportClient;
        } catch (Exception e) {
            e.printStackTrace();
        }
        return null;
    }
    private synchronized void initTranClient() throws UnknownHostException {
        if (transportClient == null) {
            Settings settings = Settings.settingsBuilder()
                    .put("client.transport.sniff", true)
                    .put("cluster.name", StringUtils.isEmpty(clusterName) ? "jkzl" : clusterName)
                    .build();
            transportClient = TransportClient.builder().settings(settings).build()
                    .addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(esHost), Integer.valueOf(tPort)));
        }
    }
}

patient-co-statistics-es/src/main/java/com/yihu/wlyy/statistics/etl/save/es/ElastricSearchSave.java

@@ -36,7 +36,7 @@ public class ElastricSearchSave {
    public Boolean save(List<SaveModel> sms) {
        try {
            //得到链接
            JestClient jestClient = elasticFactory.getClient();
            JestClient jestClient = elasticFactory.getJestClient();
            Bulk.Builder bulk = new Bulk.Builder().defaultIndex(esIndex).defaultType(esType);
            for (SaveModel obj : sms) {
@@ -44,7 +44,6 @@ public class ElastricSearchSave {
                bulk.addAction(index);
            }
            BulkResult br = jestClient.execute(bulk.build());
            //关闭链接
            logger.info("save data count:" + sms.size());
            logger.info("save flag:" + br.isSucceeded());
            return br.isSucceeded();

patient-co-statistics-es/src/main/java/com/yihu/wlyy/statistics/job/business/CurrentMysqlToEsQuotaJob.java

@@ -170,7 +170,7 @@ public class CurrentMysqlToEsQuotaJob implements Job {
     */
    private boolean updateData(List<SaveModel> sms, Date quotaDate, String quotaCode, String timeLevel) {
        try {
            JestClient jestClient = elasticFactory.getClient();
            JestClient jestClient = elasticFactory.getJestClient();
            //先根据条件查找出来
            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
            searchSourceBuilder.query(
@@ -182,6 +182,7 @@ public class CurrentMysqlToEsQuotaJob implements Job {
            Search search = new Search.Builder(searchSourceBuilder.toString()).addIndex(esIndex).addType(esType)
                    .build();
            SearchResult result = jestClient.execute(search);
            List<SaveModel> quarySaveModels = result.getSourceAsObjectList(SaveModel.class);
            //如果之前有值就用查询出来的然后修改数目即可
            if (quarySaveModels != null && quarySaveModels.size() > 0) {

patient-co-statistics-es/src/main/java/com/yihu/wlyy/statistics/job/business/MysqlToEsQuotaJob.java

@@ -167,7 +167,7 @@ public class MysqlToEsQuotaJob implements Job {
     */
    private void deleteData(Date quotaDate, String quotaCode, String timeLevel) {
        try {
            JestClient jestClient = elasticFactory.getClient();
            JestClient jestClient = elasticFactory.getJestClient();
            //先根据条件查找出来
            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
            searchSourceBuilder.query(

patient-co-statistics-es/src/main/java/com/yihu/wlyy/statistics/model/job/ElasticsearchService.java

@@ -0,0 +1,142 @@
package com.yihu.wlyy.statistics.model.job;
import com.alibaba.druid.pool.ExceptionSorter;
import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.expr.SQLQueryExpr;
import com.alibaba.druid.sql.parser.SQLExprParser;
import com.yihu.wlyy.statistics.etl.save.es.ElasticFactory;
import com.yihu.wlyy.statistics.vo.SaveModel;
import io.searchbox.client.JestClient;
import io.searchbox.core.Search;
import io.searchbox.core.SearchResult;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.nlpcn.es4sql.domain.Select;
import org.nlpcn.es4sql.jdbc.ObjectResult;
import org.nlpcn.es4sql.jdbc.ObjectResultsExtractor;
import org.nlpcn.es4sql.parse.ElasticSqlExprParser;
import org.nlpcn.es4sql.parse.SqlParser;
import org.nlpcn.es4sql.query.AggregationQueryAction;
import org.nlpcn.es4sql.query.DefaultQueryAction;
import org.nlpcn.es4sql.query.SqlElasticSearchRequestBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.List;
/**
 * Created by chenweida on 2017/7/14.
 */
@Service
public class ElasticsearchService {
    @Autowired
    private ElasticFactory elasticFactory;
    @Value("${es.type}")
    private String esType;
    @Value("${es.index}")
    private String esIndex;
    /**
     * 查询一级维度的指标
     *
     * @param quotaCode 指标id
     * @param timeLevel 1增量 2到达量
     * @param areaLevel 1 省 2 市 3 区县 4 机构 5团队
     * @param code      如果areaLevel是3就是区的code 如果是4就是机构的code
     * @param quotaDate 时间 yyyy-MM-dd
     * @return
     */
    public SaveModel findDimension1Quota(String quotaCode, String timeLevel, String areaLevel, String code, String quotaDate) {
        try {
            JestClient jestClient = elasticFactory.getJestClient();
            //先根据条件查找出来
            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
            searchSourceBuilder.query(
                    new BoolQueryBuilder()
                            .must(QueryBuilders.matchQuery("quotaCode", quotaCode))
                            .must(QueryBuilders.matchQuery("timeLevel", timeLevel))
                            .must(QueryBuilders.matchQuery(SaveModel.getAreaLevelKey(areaLevel), code))
                            .must(QueryBuilders.matchQuery("quotaDate", quotaDate)))
                    .size(10);//一次取10条
            Search search = new Search.Builder(searchSourceBuilder.toString()).addIndex(esIndex).addType(esType)
                    .build();
            SearchResult result = jestClient.execute(search);
            List<SaveModel> saveModels = result.getSourceAsObjectList(SaveModel.class);
            if (saveModels != null && saveModels.size() > 0) {
                return saveModels.get(0);
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
        return null;
    }
    public SaveModel findDimension1QuotaList(String quotaCode, String timelevel, String parentAreaLevel, String code, String quotaDate, String chlAreaLevel) {
        return null;
    }
    public List<SaveModel> excute(String sql) {
        List<SaveModel> saveModels = new ArrayList<>();
        try {
            SQLExprParser parser = new ElasticSqlExprParser(sql);
            SQLExpr expr = parser.expr();
            SQLQueryExpr queryExpr = (SQLQueryExpr) expr;
            Select select = null;
            select = new SqlParser().parseSelect(queryExpr);
            //通过抽象语法树，封装成自定义的Select，包含了select、from、where group、limit等
            AggregationQueryAction action = null;
            DefaultQueryAction queryAction = null;
            SqlElasticSearchRequestBuilder requestBuilder = null;
            if (select.isAgg) {
                //包含计算的的排序分组的
                action = new AggregationQueryAction(elasticFactory.getTransportClient(), select);
                requestBuilder = action.explain();
            } else {
                //封装成自己的Select对象
                queryAction = new DefaultQueryAction(elasticFactory.getTransportClient(), select);
                requestBuilder = queryAction.explain();
            }
            SearchResponse response = (SearchResponse) requestBuilder.get();
            ObjectResult temp = new ObjectResultsExtractor(true, true, true).extractResults(response.getAggregations(), true);
            List<String> heads = temp.getHeaders();
            temp.getLines().stream().forEach(one -> {
                try {
                    SaveModel saveModel = new SaveModel();
                    for (int i = 0; i < one.size(); i++) {
                        String key = "set" + UpFirstStr(heads.get(i));
                        Object value = one.get(i);
                        if (value instanceof String) {
                            SaveModel.class.getMethod(key, String.class).invoke(saveModel,value);
                        }else if (value instanceof Integer) {
                            SaveModel.class.getMethod(key, String.class).invoke(saveModel,value);
                        }
                    }
                    saveModels.add(saveModel);
                } catch (Exception e) {
                    e.printStackTrace();
                }
            });
        } catch (Exception e) {
            e.printStackTrace();
        }
        return saveModels;
    }
    public String UpFirstStr(String str) {
        return str.toLowerCase().replaceFirst(str.substring(0, 1), str.substring(0, 1).toUpperCase());
    }
}

patient-co-statistics-es/src/main/java/com/yihu/wlyy/statistics/vo/SaveModel.java

@@ -11,6 +11,15 @@ import java.util.Date;
 * Created by chenweida on 2017/6/1.
 */
public class SaveModel {
    private final static String teamKey = "team";
    private final static String OrgKey = "hospital";
    private final static String townKey = "town";
    private final static String cityKey = "city";
    private final static String teamLevel = "5";
    private final static String OrgLevel = "4";
    private final static String townLevel = "3";
    private final static String cityLevel = "2";
    @JestId
    private String id;
@@ -21,7 +30,7 @@ public class SaveModel {
    private Date quotaDate;//统计时间
    private String city;//城市代码 350200
    
    private String cityName;//
    private String town;//区代码 350206
@@ -52,9 +61,9 @@ public class SaveModel {
    private String slaveKey4Name;
    private Integer result1=0;//统计结果
    private Integer result1 = 0;//统计结果
    private Integer result2=0;//统计结果
    private Integer result2 = 0;//统计结果
    private String areaLevel;// 1 省 2 市 3 区县 4 机构 5团队
    @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyyMMdd'T'HHmmss.SSS'Z'")
@@ -87,6 +96,7 @@ public class SaveModel {
    public void setTown(String town) {
        this.town = town;
    }
    //@Field(type = FieldType.String, analyzer="ngram_analyzer")//使用ngram进行单字分词
    public String getHospital() {
        return hospital;
@@ -264,4 +274,31 @@ public class SaveModel {
    public void setTimeLevel(String timeLevel) {
        this.timeLevel = timeLevel;
    }
    /**
     * 根据级别得到key
     * 1 省 2 市 3 区县 4 机构 5团队
     *
     * @param level
     * @return
     */
    public static String getAreaLevelKey(String level) {
        switch (level) {
            case teamLevel: {
                return teamKey;
            }
            case OrgLevel: {
                return OrgKey;
            }
            case townLevel: {
                return townKey;
            }
            case cityLevel: {
                return cityKey;
            }
            default: {
                return teamKey;
            }
        }
    }
}

patient-co-statistics-es/src/main/resources/application.yml

@@ -132,7 +132,10 @@ systemConfig:
es:
  index: wlyy_quota_test
  type: wlyy_quota_test
  host: 172.19.103.68:9200
  host: 172.19.103.68
  port: 9200
  tPort: 9300
  clusterName: jkzl
wlyy:
  im: