|
@ -4,6 +4,7 @@ import com.yihu.base.SolrHelper;
|
|
|
import com.yihu.base.hbase.HBaseHelper;
|
|
|
import com.yihu.wlyy.figure.label.model.DataModel;
|
|
|
import com.yihu.wlyy.figure.label.util.TimeUtil;
|
|
|
import org.apache.solr.common.SolrDocumentList;
|
|
|
import org.slf4j.Logger;
|
|
|
import org.slf4j.LoggerFactory;
|
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
@ -42,49 +43,75 @@ public class HbaseExtracter implements Extracter{
|
|
|
|
|
|
public List<Map<String,Object>> extractData(String core, String q, String fq) {
|
|
|
List<Map<String,Object>> result = new ArrayList<>();
|
|
|
String[] arr = core.split(";");
|
|
|
String solrCore = arr[0];
|
|
|
String table = arr[1];
|
|
|
List<String> rowkeys = new ArrayList<>();
|
|
|
rowkeys = getRowkeys(core,q,fq);
|
|
|
rowkeys = getRowkeys(solrCore,q,fq);
|
|
|
for(String rowkey : rowkeys){
|
|
|
Map<String,Object> map = hBaseHelper.getResultMap(rowkey,"HealthProfile");
|
|
|
Map<String,Object> map = hBaseHelper.getResultMap(table,rowkey);
|
|
|
result.add(map);
|
|
|
}
|
|
|
return result;
|
|
|
}
|
|
|
|
|
|
public List<String> getRowkeys(String core, String q, String fq) {
|
|
|
List<String> rowkeyList = new ArrayList<>();
|
|
|
long count = 0;
|
|
|
try {
|
|
|
count = solrHelper.count(core, q);
|
|
|
solrHelper.query(core, q, fq, null, 0, 0);
|
|
|
String[] fqs = {fq};
|
|
|
rowkeyList = getRowkeysFromSolr(core, q, fqs, 0, count);
|
|
|
} catch (Exception e) {
|
|
|
e.printStackTrace();
|
|
|
logger.error("get solr query error");
|
|
|
}
|
|
|
int number = (int) count / numPerPage + 1;
|
|
|
/* int number = (int) count / numPerPage + 1;
|
|
|
List<String> rowkeys = new ArrayList<>();
|
|
|
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(number);
|
|
|
List<Callable<List<String>>> threadList = new ArrayList<>();
|
|
|
for (int i = 0; i < number; i++) {
|
|
|
MutilThreadSearchSolrIndexExtracter searchSolrIndexExtracter = new MutilThreadSearchSolrIndexExtracter(solrHelper, core, q, fq, i * numPerPage, numPerPage);
|
|
|
threadList.add(searchSolrIndexExtracter);
|
|
|
|
|
|
long start = System.currentTimeMillis();
|
|
|
String logTitle = "therad get data";
|
|
|
TimeUtil.start(logger, logTitle, start);
|
|
|
try {
|
|
|
List<Future<List<String>>> futureList = fixedThreadPool.invokeAll(threadList);
|
|
|
//取回线程执行的结果
|
|
|
for (Future future : futureList) {
|
|
|
rowkeys.addAll((List<String>) future.get());
|
|
|
}
|
|
|
} catch (Exception e) {
|
|
|
e.printStackTrace();
|
|
|
logger.error("get futureList result error");
|
|
|
}
|
|
|
long start = System.currentTimeMillis();
|
|
|
String logTitle = "therad get data";
|
|
|
TimeUtil.start(logger, logTitle, start);
|
|
|
try {
|
|
|
List<Future<List<String>>> futureList = fixedThreadPool.invokeAll(threadList);
|
|
|
//取回线程执行的结果
|
|
|
for (Future future : futureList) {
|
|
|
rowkeys.addAll((List<String>) future.get());
|
|
|
}
|
|
|
long finish = System.currentTimeMillis();
|
|
|
TimeUtil.finish(logger, logTitle, start, finish);
|
|
|
} catch (Exception e) {
|
|
|
e.printStackTrace();
|
|
|
logger.error("get futureList result error");
|
|
|
}
|
|
|
return null;
|
|
|
long finish = System.currentTimeMillis();
|
|
|
TimeUtil.finish(logger, logTitle, start, finish);
|
|
|
return rowkeys;*/
|
|
|
return rowkeyList;
|
|
|
}
|
|
|
|
|
|
public List<String> getRowkeysFromSolr(String core,String q,String[] fq,long start,long rows){
|
|
|
List<String> rowkeyList = new ArrayList<>();
|
|
|
SolrDocumentList solrDocumentList = null;
|
|
|
String logTitle = "get data from solr";
|
|
|
TimeUtil.start(logger, logTitle, start);
|
|
|
try {
|
|
|
solrDocumentList = solrHelper.queryfl(core,q,fq,null,"rowkey",start,rows);
|
|
|
} catch (Exception e) {
|
|
|
e.printStackTrace();
|
|
|
}
|
|
|
long finish = System.currentTimeMillis();
|
|
|
TimeUtil.finish(logger, logTitle, start, finish);
|
|
|
logger.info("job get data counts:" + solrDocumentList.size());
|
|
|
solrDocumentList.forEach(
|
|
|
document -> {
|
|
|
rowkeyList.add(document.get("rowkey").toString());
|
|
|
}
|
|
|
);
|
|
|
return rowkeyList;
|
|
|
}
|
|
|
|
|
|
}
|