package com.yihu.jw.elasticsearch; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.expr.SQLQueryExpr; import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock; import com.alibaba.druid.sql.parser.ParserException; import com.alibaba.druid.sql.parser.SQLExprParser; import com.alibaba.druid.sql.parser.Token; import org.apache.commons.lang3.StringUtils; import org.apache.http.HttpHost; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.client.Client; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.index.query.*; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.SortBuilder; import org.nlpcn.es4sql.domain.Select; import org.nlpcn.es4sql.domain.Where; import org.nlpcn.es4sql.exception.SqlParseException; import org.nlpcn.es4sql.jdbc.ObjectResult; import org.nlpcn.es4sql.jdbc.ObjectResultsExtractor; import org.nlpcn.es4sql.parse.ElasticSqlExprParser; import org.nlpcn.es4sql.parse.SqlParser; import org.nlpcn.es4sql.parse.WhereParser; import org.nlpcn.es4sql.query.AggregationQueryAction; import org.nlpcn.es4sql.query.DefaultQueryAction; import org.nlpcn.es4sql.query.SqlElasticSearchRequestBuilder; import org.nlpcn.es4sql.query.maker.QueryMaker; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Service; import javax.annotation.Resource; import java.io.IOException; import java.text.DecimalFormat; import java.util.*; /** * elasticsearch7.x版本数据获取 * Created by yeshijie on 2022/6/15. */ @Service @Scope(ConfigurableBeanFactory.SCOPE_SINGLETON) public class ElasticSearch7Util { private static final Logger logger = LoggerFactory.getLogger(ElasticSearch7Util.class); @Resource(name="esClient") private RestHighLevelClient restHighLevelClient; /** * 执行sql * @param sql * @return */ public Long executeCountSQL(String sql) throws Exception { //实例化查询请求对象 SearchRequest request = new SearchRequest(); //实例化SearchSourceBuilder SearchSourceBuilder searchBuilder = new SearchSourceBuilder(); //根据索引、查询条件构建查询构造器 BoolQueryBuilder boolQueryBuilder = createQueryBuilderBySql(sql); //将查询构造器注入SearchSourceBuilder searchBuilder.query(boolQueryBuilder); //设置请求查询的索引(查询构造器中已指定,无需重复设置) //request.indices(indexName); //将构建好的SearchSourceBuilder注入请求 request.source(searchBuilder); //带入请求执行查询 SearchResponse searchResponse = restHighLevelClient.search(request, RequestOptions.DEFAULT); //得到查询结果 SearchHits hits = searchResponse.getHits(); SearchHit[] searchHits = hits.getHits(); //遍历查询结果 for(SearchHit hit : searchHits){ Map datas = hit.getSourceAsMap(); for (Object o:datas.values()){ if (o instanceof Double){ Double valueTemp =Double.valueOf((Double)o); DecimalFormat df = new DecimalFormat("######0"); return Long.parseLong(df.format(valueTemp)); } } } return 0L; } /** * 执行sql */ public List> executeSQL(String sql) throws IOException { //实例化查询请求对象 SearchRequest request = new SearchRequest(); //实例化SearchSourceBuilder SearchSourceBuilder searchBuilder = new SearchSourceBuilder(); //根据索引、查询条件构建查询构造器 BoolQueryBuilder boolQueryBuilder = createQueryBuilderBySql(sql); //将查询构造器注入SearchSourceBuilder searchBuilder.query(boolQueryBuilder); //设置请求查询的索引(查询构造器中已指定,无需重复设置) //request.indices(indexName); //将构建好的SearchSourceBuilder注入请求 request.source(searchBuilder); //带入请求执行查询 SearchResponse searchResponse = restHighLevelClient.search(request, RequestOptions.DEFAULT); //得到查询结果 SearchHits hits = searchResponse.getHits(); SearchHit[] searchHits = hits.getHits(); List> listData = new ArrayList<>(); //遍历查询结果 for(SearchHit hit : searchHits){ Map datas = hit.getSourceAsMap(); listData.add(datas); logger.info(datas.toString()); } return listData; } /** * 执行sql */ public List executeSql(String sql,Class clazz) throws Exception { //实例化查询请求对象 SearchRequest request = new SearchRequest(); //实例化SearchSourceBuilder SearchSourceBuilder searchBuilder = new SearchSourceBuilder(); //根据索引、查询条件构建查询构造器 BoolQueryBuilder boolQueryBuilder = createQueryBuilderBySql(sql); //将查询构造器注入SearchSourceBuilder searchBuilder.query(boolQueryBuilder); //设置请求查询的索引(查询构造器中已指定,无需重复设置) //request.indices(indexName); //将构建好的SearchSourceBuilder注入请求 request.source(searchBuilder); //带入请求执行查询 SearchResponse searchResponse = restHighLevelClient.search(request, RequestOptions.DEFAULT); //得到查询结果 SearchHits hits = searchResponse.getHits(); SearchHit[] searchHits = hits.getHits(); List listData = new ArrayList<>(); //遍历查询结果 for(SearchHit hit : searchHits){ Map datas = hit.getSourceAsMap(); T t = clazz.newInstance(); BeanUtils.copyProperties(t,datas); listData.add(t); logger.info(datas.toString()); } return listData; } /** * 构建查询构造器 * @param indexName 索引名 * @param whereExpress 查询条件:(f1=2 and f2=1) or (f3=1 and f4=1) * @return */ public BoolQueryBuilder createQueryBuilderByWhere(String indexName, String whereExpress) { BoolQueryBuilder boolQuery = null; try { String sql = "select * from " + indexName; String whereTemp = ""; if (StringUtils.isNotBlank(whereExpress)) { whereTemp = " where 1=1 " + whereExpress; } SQLQueryExpr sqlExpr = (SQLQueryExpr) toSqlExpr(sql + whereTemp); SqlParser sqlParser = new SqlParser(); MySqlSelectQueryBlock query = (MySqlSelectQueryBlock) sqlExpr.getSubQuery().getQuery(); WhereParser whereParser = new WhereParser(sqlParser, query); Where where = whereParser.findWhere(); if (where != null) { boolQuery = QueryMaker.explan(where); } } catch (SqlParseException e) { logger.info("ReadES.createQueryBuilderByExpress-Exception,"+e.getMessage()); e.printStackTrace(); } return boolQuery; } /** * 构建查询构造器 * @return */ public BoolQueryBuilder createQueryBuilderBySql(String sql) { BoolQueryBuilder boolQuery = null; try { SQLQueryExpr sqlExpr = (SQLQueryExpr) toSqlExpr(sql); SqlParser sqlParser = new SqlParser(); MySqlSelectQueryBlock query = (MySqlSelectQueryBlock) sqlExpr.getSubQuery().getQuery(); WhereParser whereParser = new WhereParser(sqlParser, query); Where where = whereParser.findWhere(); if (where != null) { boolQuery = QueryMaker.explan(where); } } catch (SqlParseException e) { logger.info("ReadES.createQueryBuilderByExpress-Exception,"+e.getMessage()); e.printStackTrace(); } return boolQuery; } /** * 验证sql * * @param sql sql查询语句 * @return and (a=1 and b=1) or (c=1 and d=1) */ private SQLExpr toSqlExpr(String sql) { SQLExprParser parser = new ElasticSqlExprParser(sql); SQLExpr expr = parser.expr(); if (parser.getLexer().token() != Token.EOF) { throw new ParserException("illegal sql expr : " + sql); } return expr; } /** * 查询指定索引下的信息 * @param indexName 索引名称 * @param condition 查询条件 * queryIndexContent("user_site"," and phone_no in('12234567890') "); */ public List> queryIndexContent(String indexName, String condition) throws IOException { //实例化查询请求对象 SearchRequest request = new SearchRequest(); //实例化SearchSourceBuilder SearchSourceBuilder searchBuilder = new SearchSourceBuilder(); //根据索引、查询条件构建查询构造器 BoolQueryBuilder boolQueryBuilder = createQueryBuilderByWhere(indexName,condition); //将查询构造器注入SearchSourceBuilder searchBuilder.query(boolQueryBuilder); //设置请求查询的索引(查询构造器中已指定,无需重复设置) //request.indices(indexName); //将构建好的SearchSourceBuilder注入请求 request.source(searchBuilder); //带入请求执行查询 SearchResponse searchResponse = restHighLevelClient.search(request, RequestOptions.DEFAULT); //得到查询结果 SearchHits hits = searchResponse.getHits(); SearchHit[] searchHits = hits.getHits(); List> listData = new ArrayList<>(); //遍历查询结果 for(SearchHit hit : searchHits){ Map datas = hit.getSourceAsMap(); listData.add(datas); logger.info(datas.toString()); } return listData; } /** * 查询语句转换 * @param filters * @return */ public QueryBuilder getQueryBuilder(String filters) { BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); if (StringUtils.isEmpty(filters)) { return boolQueryBuilder; } String [] filterArr = filters.split(";"); for (String filter : filterArr) { if (filter.contains("||")){ String [] fields = filter.split("\\|\\|"); BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); for (String filed : fields) { String [] condition = filed.split("="); queryBuilder.should(QueryBuilders.termQuery(condition[0], condition[1])); } boolQueryBuilder.must(queryBuilder); } else if (filter.contains("?")) { String [] condition = filter.split("\\?"); MatchPhraseQueryBuilder matchPhraseQueryBuilder = QueryBuilders.matchPhraseQuery(condition[0], condition[1]); boolQueryBuilder.must(matchPhraseQueryBuilder); } else if (filter.contains("<>")) { String [] condition = filter.split("<>"); if (condition[1].contains(",")) { String [] inCondition = condition[1].split(","); TermsQueryBuilder termsQueryBuilder = QueryBuilders.termsQuery(condition[0], inCondition); boolQueryBuilder.mustNot(termsQueryBuilder); } else { TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery(condition[0], condition[1]); boolQueryBuilder.mustNot(termQueryBuilder); } } else if (filter.contains(">=")) { String [] condition = filter.split(">="); RangeQueryBuilder rangeQueryBuilder = QueryBuilders.rangeQuery(condition[0]); rangeQueryBuilder.gte(condition[1]); boolQueryBuilder.must(rangeQueryBuilder); } else if (filter.contains(">")) { String [] condition = filter.split(">"); RangeQueryBuilder rangeQueryBuilder = QueryBuilders.rangeQuery(condition[0]); rangeQueryBuilder.gt(condition[1]); boolQueryBuilder.must(rangeQueryBuilder); } else if (filter.contains("<=")) { String [] condition = filter.split("<="); RangeQueryBuilder rangeQueryBuilder = QueryBuilders.rangeQuery(condition[0]); rangeQueryBuilder.lte(condition[1]); boolQueryBuilder.must(rangeQueryBuilder); } else if (filter.contains("<")) { String [] condition = filter.split("<"); RangeQueryBuilder rangeQueryBuilder = QueryBuilders.rangeQuery(condition[0]); rangeQueryBuilder.lt(condition[1]); boolQueryBuilder.must(rangeQueryBuilder); } else if (filter.contains("=")) { String [] condition = filter.split("="); if (condition[1].contains(",")) { String [] inCondition = condition[1].split(","); TermsQueryBuilder termsQueryBuilder = QueryBuilders.termsQuery(condition[0], inCondition); boolQueryBuilder.must(termsQueryBuilder); } else { TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery(condition[0], condition[1]); boolQueryBuilder.must(termQueryBuilder); } } } return boolQueryBuilder; } /** * 多条件查询 * @param mustMap * @param length * @return */ public List multiSearch(Map mustMap,int length,String index) { // 根据多个条件 生成 boolQueryBuilder BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); // 循环添加多个条件 for (Map.Entry entry : mustMap.entrySet()) { boolQueryBuilder.must(QueryBuilders .matchQuery(entry.getKey(), entry.getValue())); } // boolQueryBuilder生效 SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.query(boolQueryBuilder); searchSourceBuilder.size(length); // 其中listSearchResult是自己编写的方法,以供多中查询方式使用。 return listSearchResult(searchSourceBuilder,index); } /** * 用来处理搜索结果,转换成链表 * @param builder * @return */ public List listSearchResult(SearchSourceBuilder builder,String index) { // 提交查询 SearchRequest searchRequest = new SearchRequest(index); searchRequest.source(builder); // 获得response SearchResponse searchResponse = null; try { searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); } catch (IOException e) { e.printStackTrace(); } // 从response中获得结果 List list = new LinkedList<>(); SearchHits hits = searchResponse.getHits(); Iterator iterator = hits.iterator(); while(iterator.hasNext()) { SearchHit next = iterator.next(); list.add(next.getSourceAsString()); } return list; } }