|
|
@ -2,26 +2,27 @@ package com.yihu.iot.datainput.service;
|
|
|
|
|
|
import com.alibaba.fastjson.JSONArray;
|
|
|
import com.alibaba.fastjson.JSONObject;
|
|
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
|
import com.google.gson.JsonArray;
|
|
|
import com.yihu.base.es.config.ElastricSearchHelper;
|
|
|
import com.yihu.base.hbase.HBaseHelper;
|
|
|
import com.yihu.iot.datainput.enums.DataTypeEnum;
|
|
|
import com.yihu.iot.datainput.util.ConstantUtils;
|
|
|
import com.yihu.iot.datainput.util.RowKeyUtils;
|
|
|
import io.searchbox.core.Search;
|
|
|
import io.searchbox.core.SearchResult;
|
|
|
import org.apache.commons.lang.StringUtils;
|
|
|
import org.apache.hadoop.hbase.Cell;
|
|
|
import org.apache.hadoop.hbase.CellUtil;
|
|
|
import org.apache.hadoop.hbase.client.Result;
|
|
|
import org.apache.hadoop.hbase.util.Bytes;
|
|
|
import org.elasticsearch.action.search.SearchRequestBuilder;
|
|
|
import org.elasticsearch.index.query.QueryBuilders;
|
|
|
import org.elasticsearch.index.query.TermQueryBuilder;
|
|
|
import org.apache.hadoop.hbase.util.CollectionUtils;
|
|
|
import org.elasticsearch.index.query.*;
|
|
|
import org.elasticsearch.search.builder.SearchSourceBuilder;
|
|
|
import org.slf4j.Logger;
|
|
|
import org.slf4j.LoggerFactory;
|
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
|
import org.springframework.stereotype.Component;
|
|
|
|
|
|
import java.io.IOException;
|
|
|
import java.util.ArrayList;
|
|
|
import java.util.HashMap;
|
|
|
import java.util.List;
|
|
|
@ -90,7 +91,7 @@ public class DataSearchService {
|
|
|
JSONObject matchQuery = new JSONObject();
|
|
|
JSONObject subQuery = new JSONObject();
|
|
|
String baseName = DataTypeEnum.body_sign_params.name().toString();
|
|
|
if(null != DataStandardConvertService.dataMap.get(baseName) && DataStandardConvertService.dataMap.get(baseName).contains(key)){
|
|
|
if(null != DataStandardConvertService.dataMap.get(baseName) && DataStandardConvertService.dataMap.get(baseName).contains(key) || StringUtils.equalsIgnoreCase("rid",key)){
|
|
|
subQuery.put("data."+key,jsonObject.get(key)); //data数据里内嵌的字段，真正的数据值内容
|
|
|
}else{
|
|
|
subQuery.put(key,jsonObject.get(key));
|
|
|
@ -105,34 +106,57 @@ public class DataSearchService {
|
|
|
return query.toJSONString();
|
|
|
}
|
|
|
|
|
|
public String getData(String jsonData){
|
|
|
String query = getQueryString(jsonData);
|
|
|
String esResult = elastricSearchHelper.search(ConstantUtils.esIndex,ConstantUtils.esType,query);
|
|
|
JSONArray jsonArray = (JSONArray)JSONArray.parse(esResult);
|
|
|
public String getData(String jsonData,int page, int size){
|
|
|
// String query = getQueryString(jsonData);
|
|
|
JSONObject resultJsonObj = new JSONObject();
|
|
|
JSONArray resultArray = new JSONArray();
|
|
|
List list = new ArrayList();
|
|
|
ObjectMapper objectMapper = new ObjectMapper();
|
|
|
try {
|
|
|
list = objectMapper.readValue(jsonData,(List.class));
|
|
|
} catch (IOException e) {
|
|
|
e.printStackTrace();
|
|
|
}
|
|
|
SearchSourceBuilder query = getQueryBuilder(list,page,size);
|
|
|
SearchResult esResult = elastricSearchHelper.search(ConstantUtils.esIndex,ConstantUtils.esType,query.toString());
|
|
|
if(esResult.getTotal() == 0){
|
|
|
return "";
|
|
|
}
|
|
|
List<String> rowkeys = new ArrayList<>();
|
|
|
for(Object object:jsonArray){
|
|
|
JSONObject jsonObject = (JSONObject)object;
|
|
|
for(Object object:esResult.getSourceAsStringList()){
|
|
|
JSONObject jsonObject = (JSONObject)JSONObject.parse(String.valueOf(object));
|
|
|
JSONArray datas = (JSONArray)jsonObject.get("data");
|
|
|
for(Object data:datas){
|
|
|
JSONObject dataJson = (JSONObject)data;
|
|
|
rowkeys.add(dataJson.getString("rid"));
|
|
|
if(null != dataJson.getString("rid")){
|
|
|
rowkeys.add(dataJson.getString("rid"));
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
|
|
|
if(CollectionUtils.isEmpty(rowkeys)){
|
|
|
resultArray.addAll(esResult.getSourceAsStringList());
|
|
|
resultJsonObj.put("data",resultArray);
|
|
|
resultJsonObj.put("count",esResult.getTotal());
|
|
|
return resultJsonObj.toJSONString();
|
|
|
}
|
|
|
List<Map<String, Object>> resultList = new ArrayList<>();
|
|
|
try {
|
|
|
//拿到rowkey后，去hbase读取数据内容
|
|
|
Result[] hbaseData = hBaseHelper.getResultList(ConstantUtils.tableName,rowkeys);
|
|
|
for(int i = 0;i < hbaseData.length; i++){
|
|
|
List<Cell> ceList = hbaseData[i].listCells();
|
|
|
for(Result res:hbaseData){
|
|
|
List<Cell> ceList = res.listCells();
|
|
|
if(null == ceList){
|
|
|
continue;
|
|
|
}
|
|
|
Map<String, Object> map = new HashMap<String, Object>();
|
|
|
String rowkey = Bytes.toString(hbaseData[i].getRow());
|
|
|
String rowkey = Bytes.toString(res.getRow());
|
|
|
//rowkey是根据一些头部数据加密而来，解密即可还原
|
|
|
String tag = RowKeyUtils.getMessageFromRowKey(rowkey);
|
|
|
String[] tags = tag.split(",");
|
|
|
map.put("accessToken", tags[0]);
|
|
|
map.put("deviceSn", tags[1]);
|
|
|
map.put("extCode", tags[2]);
|
|
|
map.put("access_token", tags[0]);
|
|
|
map.put("sn", tags[1]);
|
|
|
map.put("ext_code", tags[2]);
|
|
|
if (ceList != null && ceList.size() > 0) {
|
|
|
for (Cell cell : ceList) {
|
|
|
map.put(Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()),
|
|
|
@ -145,22 +169,75 @@ public class DataSearchService {
|
|
|
} catch (Exception e) {
|
|
|
e.printStackTrace();
|
|
|
logger.error("get data from hbase fail.",e.getMessage());
|
|
|
return esResult;
|
|
|
return esResult.getSourceAsString();
|
|
|
}
|
|
|
JSONArray resultArray = new JSONArray();
|
|
|
resultArray.addAll(resultList);
|
|
|
return resultArray.toJSONString();
|
|
|
resultJsonObj.put("data",resultArray);
|
|
|
resultJsonObj.put("count",esResult.getTotal());//count放最后最先读出来
|
|
|
return resultJsonObj.toJSONString();
|
|
|
}
|
|
|
|
|
|
public static void main(String args[]){
|
|
|
String str = "{\n" +
|
|
|
"\t\"or\":\"\",\n" +
|
|
|
"\t\"extCode\":\"1\",\n" +
|
|
|
"\t\"idcard\":\"test\",\n" +
|
|
|
"\t\"from\":1,\n" +
|
|
|
"\t\"size\":5,\n" +
|
|
|
"\t\"sort\":{\"measure_time\":\"desc\"}\n" +
|
|
|
"}";
|
|
|
System.out.println(getQueryString(str));
|
|
|
private SearchSourceBuilder getQueryBuilder(List<Map<String, Object>> filter,int page, int size) {
|
|
|
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
|
|
|
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
|
|
|
for(Map<String, Object> param : filter) {
|
|
|
String andOr = String.valueOf(param.get("andOr"));
|
|
|
String condition = String.valueOf(param.get("condition"));
|
|
|
String field = String.valueOf(param.get("field"));
|
|
|
Object value = param.get("value");
|
|
|
String baseName = DataTypeEnum.body_sign_params.name().toString();
|
|
|
if(null != DataStandardConvertService.dataMap.get(baseName) && DataStandardConvertService.dataMap.get(baseName).contains(field) || StringUtils.equalsIgnoreCase("id",field)){
|
|
|
field = "data." + field;
|
|
|
}
|
|
|
if(condition.equals("=")) {
|
|
|
MatchQueryBuilder matchQueryBuilder = QueryBuilders.matchPhraseQuery(field, value);
|
|
|
if("and".equals(andOr)) {
|
|
|
boolQueryBuilder.must(matchQueryBuilder);
|
|
|
}else if("or".equals(andOr)) {
|
|
|
boolQueryBuilder.should(matchQueryBuilder);
|
|
|
}
|
|
|
}else if (condition.equals("?")) {
|
|
|
QueryStringQueryBuilder queryStringQueryBuilder = QueryBuilders.queryStringQuery(field + ":" + value);
|
|
|
if("and".equals(andOr)) {
|
|
|
boolQueryBuilder.must(queryStringQueryBuilder);
|
|
|
}else if("or".equals(andOr)) {
|
|
|
boolQueryBuilder.should(queryStringQueryBuilder);
|
|
|
}
|
|
|
}else {
|
|
|
RangeQueryBuilder rangeQueryBuilder = QueryBuilders.rangeQuery(field);;
|
|
|
if(field.endsWith("Date")) {
|
|
|
rangeQueryBuilder.format("yyyy-MM-dd HH:mm:ss");
|
|
|
}
|
|
|
if(condition.equals(">")) {
|
|
|
rangeQueryBuilder.gt(value);
|
|
|
}else if(condition.equals(">=")) {
|
|
|
rangeQueryBuilder.gte(value);
|
|
|
}else if(condition.equals("<=")) {
|
|
|
rangeQueryBuilder.lte(value);
|
|
|
}else if(condition.equals("<")) {
|
|
|
rangeQueryBuilder.lt(value);
|
|
|
}
|
|
|
if("and".equals(andOr)) {
|
|
|
boolQueryBuilder.must(rangeQueryBuilder);
|
|
|
}else if("or".equals(andOr)) {
|
|
|
boolQueryBuilder.should(rangeQueryBuilder);
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
searchSourceBuilder.query(boolQueryBuilder);
|
|
|
searchSourceBuilder.from((page -1)*size);
|
|
|
searchSourceBuilder.size(size);
|
|
|
return searchSourceBuilder;
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
* 修改
|
|
|
* @param json
|
|
|
* @return
|
|
|
*/
|
|
|
public String updateFiled(String json){
|
|
|
return "success";
|
|
|
}
|
|
|
public static void main(String args[]) {
|
|
|
}
|
|
|
}
|
