|
@ -4,15 +4,19 @@ import com.yihu.base.SolrHelper;
|
|
import com.yihu.base.hbase.HBaseHelper;
|
|
import com.yihu.base.hbase.HBaseHelper;
|
|
import com.yihu.wlyy.figure.label.model.DataModel;
|
|
import com.yihu.wlyy.figure.label.model.DataModel;
|
|
import com.yihu.wlyy.figure.label.util.TimeUtil;
|
|
import com.yihu.wlyy.figure.label.util.TimeUtil;
|
|
|
|
import org.apache.hadoop.hbase.Cell;
|
|
|
|
import org.apache.hadoop.hbase.HColumnDescriptor;
|
|
|
|
import org.apache.hadoop.hbase.client.Result;
|
|
|
|
import org.apache.hadoop.hbase.util.Bytes;
|
|
|
|
import org.apache.hadoop.hbase.util.CollectionUtils;
|
|
import org.apache.solr.common.SolrDocumentList;
|
|
import org.apache.solr.common.SolrDocumentList;
|
|
import org.slf4j.Logger;
|
|
import org.slf4j.Logger;
|
|
import org.slf4j.LoggerFactory;
|
|
import org.slf4j.LoggerFactory;
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
|
|
import org.springframework.data.hadoop.hbase.HbaseTemplate;
|
|
import org.springframework.stereotype.Component;
|
|
import org.springframework.stereotype.Component;
|
|
|
|
|
|
import java.util.ArrayList;
|
|
|
|
import java.util.List;
|
|
|
|
import java.util.Map;
|
|
|
|
|
|
import java.util.*;
|
|
import java.util.concurrent.Callable;
|
|
import java.util.concurrent.Callable;
|
|
import java.util.concurrent.ExecutorService;
|
|
import java.util.concurrent.ExecutorService;
|
|
import java.util.concurrent.Executors;
|
|
import java.util.concurrent.Executors;
|
|
@ -41,18 +45,48 @@ public class HbaseExtracter implements Extracter{
|
|
return null;
|
|
return null;
|
|
}
|
|
}
|
|
|
|
|
|
public List<Map<String,Object>> extractData(String core, String q, String fq) {
|
|
|
|
List<Map<String,Object>> result = new ArrayList<>();
|
|
|
|
|
|
public List<Map<String,Object>> extractData(String core, String q, String fq,String extractColumn) {
|
|
|
|
List<Map<String,Object>> resultList = new ArrayList<>();
|
|
String[] arr = core.split(";");
|
|
String[] arr = core.split(";");
|
|
String solrCore = arr[0];
|
|
String solrCore = arr[0];
|
|
String table = arr[1];
|
|
String table = arr[1];
|
|
|
|
|
|
|
|
//组装提取的列
|
|
|
|
String basicFl = "";
|
|
|
|
String dFl = "";
|
|
|
|
String[] columnArr = extractColumn.split(";");
|
|
|
|
if(null != columnArr){
|
|
|
|
for(int i = 0;i < columnArr.length; i++){
|
|
|
|
if (columnArr[i].contains("basic=")) {
|
|
|
|
basicFl = columnArr[i].split("=")[1];
|
|
|
|
} else if (columnArr[i].contains("d=")) {
|
|
|
|
dFl = columnArr[i].split("=")[1];
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
List<String> rowkeys = new ArrayList<>();
|
|
List<String> rowkeys = new ArrayList<>();
|
|
rowkeys = getRowkeys(solrCore,q,fq);
|
|
rowkeys = getRowkeys(solrCore,q,fq);
|
|
for(String rowkey : rowkeys){
|
|
|
|
Map<String,Object> map = hBaseHelper.getResultMap(table,rowkey);
|
|
|
|
result.add(map);
|
|
|
|
|
|
long start = System.currentTimeMillis();
|
|
|
|
String logTitle = "get data from hbase";
|
|
|
|
TimeUtil.start(logger, logTitle, start);
|
|
|
|
Result[] resultArr = hBaseHelper.getResultList(table,rowkeys,basicFl,dFl);
|
|
|
|
for(int i = 0; i < resultArr.length; i++){
|
|
|
|
List<Cell> ceList = resultArr[i].listCells();
|
|
|
|
if(CollectionUtils.isEmpty(ceList)){
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
Map<String, Object> map = new HashMap();
|
|
|
|
map.put("rowkey",rowkeys.get(i));
|
|
|
|
Iterator var5 = ceList.iterator();
|
|
|
|
while(var5.hasNext()) {
|
|
|
|
Cell cell = (Cell)var5.next();
|
|
|
|
map.put(Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()), Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
|
|
|
|
}
|
|
|
|
resultList.add(map);
|
|
}
|
|
}
|
|
return result;
|
|
|
|
|
|
TimeUtil.finish(logger, logTitle, start, System.currentTimeMillis());
|
|
|
|
logger.info("hbase data counts:" + resultList.size());
|
|
|
|
return resultList;
|
|
}
|
|
}
|
|
|
|
|
|
public List<String> getRowkeys(String core, String q, String fq) {
|
|
public List<String> getRowkeys(String core, String q, String fq) {
|
|
@ -66,46 +100,32 @@ public class HbaseExtracter implements Extracter{
|
|
e.printStackTrace();
|
|
e.printStackTrace();
|
|
logger.error("get solr query error");
|
|
logger.error("get solr query error");
|
|
}
|
|
}
|
|
/* int number = (int) count / numPerPage + 1;
|
|
|
|
List<String> rowkeys = new ArrayList<>();
|
|
|
|
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(number);
|
|
|
|
List<Callable<List<String>>> threadList = new ArrayList<>();
|
|
|
|
for (int i = 0; i < number; i++) {
|
|
|
|
MutilThreadSearchSolrIndexExtracter searchSolrIndexExtracter = new MutilThreadSearchSolrIndexExtracter(solrHelper, core, q, fq, i * numPerPage, numPerPage);
|
|
|
|
threadList.add(searchSolrIndexExtracter);
|
|
|
|
}
|
|
|
|
long start = System.currentTimeMillis();
|
|
|
|
String logTitle = "therad get data";
|
|
|
|
TimeUtil.start(logger, logTitle, start);
|
|
|
|
try {
|
|
|
|
List<Future<List<String>>> futureList = fixedThreadPool.invokeAll(threadList);
|
|
|
|
//取回线程执行的结果
|
|
|
|
for (Future future : futureList) {
|
|
|
|
rowkeys.addAll((List<String>) future.get());
|
|
|
|
}
|
|
|
|
} catch (Exception e) {
|
|
|
|
e.printStackTrace();
|
|
|
|
logger.error("get futureList result error");
|
|
|
|
}
|
|
|
|
long finish = System.currentTimeMillis();
|
|
|
|
TimeUtil.finish(logger, logTitle, start, finish);
|
|
|
|
return rowkeys;*/
|
|
|
|
return rowkeyList;
|
|
return rowkeyList;
|
|
}
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* 从solr中获取rowkey
|
|
|
|
* @param core
|
|
|
|
* @param q
|
|
|
|
* @param fq
|
|
|
|
* @param start
|
|
|
|
* @param rows
|
|
|
|
* @return
|
|
|
|
*/
|
|
public List<String> getRowkeysFromSolr(String core,String q,String[] fq,long start,long rows){
|
|
public List<String> getRowkeysFromSolr(String core,String q,String[] fq,long start,long rows){
|
|
List<String> rowkeyList = new ArrayList<>();
|
|
List<String> rowkeyList = new ArrayList<>();
|
|
SolrDocumentList solrDocumentList = null;
|
|
SolrDocumentList solrDocumentList = null;
|
|
String logTitle = "get data from solr";
|
|
|
|
TimeUtil.start(logger, logTitle, start);
|
|
|
|
|
|
String logTitle = "get rowkeys from solr";
|
|
|
|
long startTime = System.currentTimeMillis();
|
|
|
|
TimeUtil.start(logger, logTitle, startTime);
|
|
try {
|
|
try {
|
|
solrDocumentList = solrHelper.queryfl(core,q,fq,null,"rowkey",start,rows);
|
|
solrDocumentList = solrHelper.queryfl(core,q,fq,null,"rowkey",start,rows);
|
|
} catch (Exception e) {
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
|
e.printStackTrace();
|
|
}
|
|
}
|
|
long finish = System.currentTimeMillis();
|
|
long finish = System.currentTimeMillis();
|
|
TimeUtil.finish(logger, logTitle, start, finish);
|
|
|
|
logger.info("job get data counts:" + solrDocumentList.size());
|
|
|
|
|
|
TimeUtil.finish(logger, logTitle, startTime, finish);
|
|
|
|
logger.info("rowkeys counts:" + solrDocumentList.size());
|
|
solrDocumentList.forEach(
|
|
solrDocumentList.forEach(
|
|
document -> {
|
|
document -> {
|
|
rowkeyList.add(document.get("rowkey").toString());
|
|
rowkeyList.add(document.get("rowkey").toString());
|