00

base/common-data-es/src/main/java/com/yihu/base/es/config/elasticsearch/ElasticSearchClient.java

@@ -0,0 +1,185 @@
package com.yihu.base.es.config.elasticsearch;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.engine.DocumentMissingException;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.config.ConfigurableBeanFactory;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Repository;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
 * Client - Es搜索服务
 * Created by progr1mmer on 2017/12/1.
 */
@Repository
@Scope(ConfigurableBeanFactory.SCOPE_SINGLETON)
public class ElasticSearchClient {
    @Autowired
    private ElasticSearchPool elasticSearchPool;
    public void mapping(String index, String type, XContentBuilder xContentBuilder) {
        TransportClient transportClient = elasticSearchPool.getClient();
        try {
            CreateIndexRequestBuilder createIndexRequestBuilder = transportClient.admin().indices().prepareCreate(index);
            createIndexRequestBuilder.addMapping(type, xContentBuilder);
            createIndexRequestBuilder.get();
        }finally {
            elasticSearchPool.releaseClient(transportClient);
        }
    }
    public void remove(String index) {
        TransportClient transportClient = elasticSearchPool.getClient();
        try {
            DeleteIndexRequestBuilder deleteIndexRequestBuilder = transportClient.admin().indices().prepareDelete(index);
            deleteIndexRequestBuilder.get();
        }finally {
            elasticSearchPool.releaseClient(transportClient);
        }
    }
    public Map<String, Object> index(String index, String type, Map<String, Object> source) {
        TransportClient transportClient = elasticSearchPool.getClient();
        try {
            IndexResponse response = transportClient.prepareIndex(index, type).setSource(source).get();
            source.put("_id", response.getId());
            return source;
        }finally {
            elasticSearchPool.releaseClient(transportClient);
        }
    }
    public void delete(String index, String type, String [] idArr) {
        TransportClient transportClient = elasticSearchPool.getClient();
        try {
            for (String id : idArr) {
                transportClient.prepareDelete(index, type, id).get();
            }
        }finally {
            elasticSearchPool.releaseClient(transportClient);
        }
    }
    public Map<String, Object> update(String index, String type, String id, Map<String, Object> source) throws DocumentMissingException {
        TransportClient transportClient = elasticSearchPool.getClient();
        try {
            transportClient.prepareUpdate(index, type, id).setDoc(source).get();
            return findById(index, type, id);
        }finally {
            elasticSearchPool.releaseClient(transportClient);
        }
    }
    public Map<String, Object> findById(String index, String type, String id) {
        TransportClient transportClient = elasticSearchPool.getClient();
        try {
            GetRequest getRequest = new GetRequest(index, type, id);
            GetResponse response = transportClient.get(getRequest).actionGet();
            Map<String, Object> source = response.getSource();
            if(source != null) {
                source.put("_id", response.getId());
            }
            return source;
        }finally {
            elasticSearchPool.releaseClient(transportClient);
        }
    }
    public List<Map<String, Object>> findByField(String index, String type, QueryBuilder queryBuilder) {
        TransportClient transportClient = elasticSearchPool.getClient();
        try {
            SearchRequestBuilder builder = transportClient.prepareSearch(index);
            builder.setTypes(type);
            builder.setSearchType(SearchType.DFS_QUERY_THEN_FETCH);
            builder.setQuery(queryBuilder);
            builder.setExplain(true);
            SearchResponse response = builder.get();
            SearchHits hits = response.getHits();
            List<Map<String, Object>> resultList = new ArrayList<Map<String, Object>>();
            for (SearchHit hit : hits.getHits()) {
                Map<String, Object> source = hit.getSource();
                source.put("_id", hit.getId());
                resultList.add(source);
            }
            return resultList;
        }finally {
            elasticSearchPool.releaseClient(transportClient);
        }
    }
    public List<Map<String, Object>> page(String index, String type, QueryBuilder queryBuilder, int page, int size){
        TransportClient transportClient = elasticSearchPool.getClient();
        try {
            SearchRequestBuilder builder = transportClient.prepareSearch(index);
            builder.setTypes(type);
            builder.setSearchType(SearchType.DFS_QUERY_THEN_FETCH);
            builder.setQuery(queryBuilder);
            builder.setFrom((page - 1) * size).setSize(size);
            builder.setExplain(true);
            SearchResponse response = builder.get();
            SearchHits hits = response.getHits();
            List<Map<String, Object>> resultList = new ArrayList<Map<String, Object>>();
            for (SearchHit hit : hits.getHits()) {
                Map<String, Object> source = hit.getSource();
                source.put("_id", hit.getId());
                resultList.add(source);
            }
            return resultList;
        }finally {
            elasticSearchPool.releaseClient(transportClient);
        }
    }
    public List<String> getIds(String index, String type, QueryBuilder queryBuilder){
        TransportClient transportClient = elasticSearchPool.getClient();
        try {
            SearchRequestBuilder builder = transportClient.prepareSearch(index);
            builder.setTypes(type);
            builder.setSearchType(SearchType.DFS_QUERY_THEN_FETCH);
            builder.setQuery(queryBuilder);
            builder.setFrom(0).setSize(10000);
            builder.setExplain(true);
            SearchResponse response = builder.get();
            SearchHits hits = response.getHits();
            List<String> resultList = new ArrayList<>();
            for (SearchHit hit : hits.getHits()) {
                resultList.add(hit.getId());
            }
            return resultList;
        }finally {
            elasticSearchPool.releaseClient(transportClient);
        }
    }
    public long count(String index, String type, QueryBuilder queryBuilder){
        TransportClient transportClient = elasticSearchPool.getClient();
        try {
            SearchRequestBuilder builder = transportClient.prepareSearch(index);
            builder.setTypes(type);
            builder.setSearchType(SearchType.DFS_QUERY_THEN_FETCH);
            builder.setQuery(queryBuilder);
            builder.setExplain(true);
            return builder.get().getHits().totalHits();
        }finally {
            elasticSearchPool.releaseClient(transportClient);
        }
    }
}

base/common-data-es/src/main/java/com/yihu/base/es/config/elasticsearch/ElasticSearchConfig.java

@@ -0,0 +1,34 @@
package com.yihu.base.es.config.elasticsearch;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
/**
 * Created by progr1mmer on 2017/12/1.
 */
@ConfigurationProperties(prefix = "elasticsearch")
@Configuration
public class ElasticSearchConfig {
    // 集群名称
    private String clusterName;
    // 节点
    private String clusterNodes;
    public String getClusterName() {
        return clusterName;
    }
    public void setClusterName(String clusterName) {
        this.clusterName = clusterName;
    }
    public String getClusterNodes() {
        return clusterNodes;
    }
    public void setClusterNodes(String clusterNodes) {
        this.clusterNodes = clusterNodes;
    }
}

base/common-data-es/src/main/java/com/yihu/base/es/config/elasticsearch/ElasticSearchPool.java

@@ -0,0 +1,89 @@
package com.yihu.base.es.config.elasticsearch;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.beans.factory.config.ConfigurableBeanFactory;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.List;
/**
 * Created by progr1mmer on 2018/1/4.
 */
@Component
@Scope(ConfigurableBeanFactory.SCOPE_SINGLETON)
public class ElasticSearchPool {
    @Value("${elasticsearch.pool.init-size}")
    private int initSize;
    @Value("${elasticsearch.pool.max-size}")
    private int maxSize;
    private List<TransportClient> clientPool;
    @Autowired
    private ElasticSearchConfig elasticSearchConfig;
    @PostConstruct
    private void init() {
        if(clientPool == null) {
            clientPool = new ArrayList<TransportClient>();
        }
        synchronized (ElasticSearchClient.class) {
            while (clientPool.size() < initSize) {
                Settings settings = Settings.builder()
                        .put("cluster.name", elasticSearchConfig.getClusterName())
                        .put("client.transport.sniff", true)
                        .build();
                String[] nodeArr = elasticSearchConfig.getClusterNodes().split(",");
                InetSocketTransportAddress[] socketArr = new InetSocketTransportAddress[nodeArr.length];
                for (int i = 0; i < socketArr.length; i++) {
                    if (!StringUtils.isEmpty(nodeArr[i])) {
                        String[] nodeInfo = nodeArr[i].split(":");
                        socketArr[i] = new InetSocketTransportAddress(new InetSocketAddress(nodeInfo[0], new Integer(nodeInfo[1])));
                    }
                }
                TransportClient transportClient = TransportClient.builder().settings(settings).build().addTransportAddresses(socketArr);
                clientPool.add(transportClient);
            }
        }
    }
    @PreDestroy
    private void destroy(){
        if (null != clientPool) {
            for (TransportClient transportClient : clientPool) {
                transportClient.close();
            }
        }
    }
    public synchronized TransportClient getClient() {
        int last_index = clientPool.size() - 1;
        TransportClient transportClient = clientPool.get(last_index);
        clientPool.remove(last_index);
        if(clientPool.isEmpty()) {
            init();
        }
        return transportClient;
    }
    public synchronized void releaseClient(TransportClient transportClient) {
        if(clientPool.size() > maxSize) {
            if (null != transportClient) {
                transportClient.close();
            }
        }else {
            clientPool.add(transportClient);
        }
    }
}

base/common-data-es/src/main/java/com/yihu/base/es/config/elasticsearch/ElasticSearchUtil.java

@@ -0,0 +1,137 @@
package com.yihu.base.es.config.elasticsearch;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.engine.DocumentMissingException;
import org.elasticsearch.index.query.*;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.config.ConfigurableBeanFactory;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.text.ParseException;
import java.util.List;
import java.util.Map;
/**
 * Util - Es搜索服务
 * Created by progr1mmer on 2017/12/2.
 */
@Service
@Scope(ConfigurableBeanFactory.SCOPE_SINGLETON)
public class ElasticSearchUtil {
    @Autowired
    private ElasticSearchClient elasticSearchClient;
    public void mapping(String index, String type, Map<String, Map<String, String>> source) throws IOException{
        XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("properties");
        for(String field : source.keySet()) {
            xContentBuilder.startObject(field);
            Map<String, String> propsMap = source.get(field);
            for(String prop : propsMap.keySet()) {
                xContentBuilder.field(prop, propsMap.get(prop));
            }
            xContentBuilder.endObject();
        }
        xContentBuilder.endObject().endObject();
        elasticSearchClient.mapping(index, type, xContentBuilder);
    }
    public void remove(String index){
        elasticSearchClient.remove(index);
    }
    public Map<String, Object> index(String index, String type, Map<String, Object> source) throws ParseException{
        return elasticSearchClient.index(index, type, source);
    }
    public void delete(String index, String type, String [] idArr) {
        elasticSearchClient.delete(index, type, idArr);
    }
    public void deleteByField(String index, String type, String field, Object value) {
        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
        MatchQueryBuilder matchQueryBuilder = QueryBuilders.matchPhraseQuery(field, value);
        boolQueryBuilder.must(matchQueryBuilder);
        List<String> idList = elasticSearchClient.getIds(index, type, boolQueryBuilder);
        String [] idArr = new String[idList.size()];
        idArr = idList.toArray(idArr);
        elasticSearchClient.delete(index, type, idArr);
    }
    public Map<String, Object> update(String index, String type, String id, Map<String, Object> source) throws DocumentMissingException {
        if(source.containsKey("_id")) {
            source.remove("_id");
        }
        return elasticSearchClient.update(index, type, id, source);
    }
    public Map<String, Object> findById(String index, String type, String id) {
        return elasticSearchClient.findById(index, type, id);
    }
    public List<Map<String, Object>> findByField(String index, String type, String field, Object value) {
        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
        MatchQueryBuilder matchQueryBuilder = QueryBuilders.matchPhraseQuery(field, value);
        boolQueryBuilder.must(matchQueryBuilder);
        return elasticSearchClient.findByField(index, type, boolQueryBuilder);
    }
    public List<Map<String, Object>> page(String index, String type, List<Map<String, Object>> filter, int page, int size) {
        QueryBuilder boolQueryBuilder = getQueryBuilder(filter);
        return elasticSearchClient.page(index, type, boolQueryBuilder, page, size);
    }
    public long count(String index, String type, List<Map<String, Object>> filter) {
        QueryBuilder boolQueryBuilder = getQueryBuilder(filter);
        return elasticSearchClient.count(index, type, boolQueryBuilder);
    }
    private QueryBuilder getQueryBuilder(List<Map<String, Object>> filter) {
        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
        for(Map<String, Object> param : filter) {
            String andOr = String.valueOf(param.get("andOr"));
            String condition = String.valueOf(param.get("condition"));
            String field = String.valueOf(param.get("field"));
            Object value = param.get("value");
            if(condition.equals("=")) {
                MatchQueryBuilder matchQueryBuilder = QueryBuilders.matchPhraseQuery(field, value);
                if("and".equals(andOr)) {
                    boolQueryBuilder.must(matchQueryBuilder);
                }else if("or".equals(andOr)) {
                    boolQueryBuilder.should(matchQueryBuilder);
                }
            }else if (condition.equals("?")) {
                QueryStringQueryBuilder queryStringQueryBuilder = QueryBuilders.queryStringQuery(field + ":" + value);
                if("and".equals(andOr)) {
                    boolQueryBuilder.must(queryStringQueryBuilder);
                }else if("or".equals(andOr)) {
                    boolQueryBuilder.should(queryStringQueryBuilder);
                }
            }else {
                RangeQueryBuilder rangeQueryBuilder = QueryBuilders.rangeQuery(field);;
                if(field.endsWith("Date")) {
                    rangeQueryBuilder.format("yyyy-MM-dd HH:mm:ss");
                }
                if(condition.equals(">")) {
                    rangeQueryBuilder.gt(value);
                }else if(condition.equals(">=")) {
                    rangeQueryBuilder.gte(value);
                }else if(condition.equals("<=")) {
                    rangeQueryBuilder.lte(value);
                }else if(condition.equals("<")) {
                    rangeQueryBuilder.lt(value);
                }
                if("and".equals(andOr)) {
                    boolQueryBuilder.must(rangeQueryBuilder);
                }else if("or".equals(andOr)) {
                    boolQueryBuilder.should(rangeQueryBuilder);
                }
            }
        }
        return boolQueryBuilder;
    }
}

svr/svr-iot/pom.xml

@@ -135,5 +135,10 @@
            <artifactId>common-data-hbase</artifactId>
            <version>1.0.0</version>
        </dependency>
        <dependency>
            <groupId>com.yihu.base</groupId>
            <artifactId>common-data-es</artifactId>
            <version>1.0.0</version>
        </dependency>
    </dependencies>
</project>