Hbase Solr

.gitignore

@@ -7,7 +7,7 @@
*.iws
*.pyc
*.pyo
.idea/
.idea/*
.idea_modules/
build/*.jar
.settings

.idea/inspectionProfiles/Project_Default.xml

@@ -1,36 +0,0 @@
<component name="InspectionProjectProfileManager">
  <profile version="1.0">
    <option name="myName" value="Project Default" />
    <inspection_tool class="JavaDoc" enabled="true" level="WARNING" enabled_by_default="true">
      <option name="TOP_LEVEL_CLASS_OPTIONS">
        <value>
          <option name="ACCESS_JAVADOC_REQUIRED_FOR" value="none" />
          <option name="REQUIRED_TAGS" value="" />
        </value>
      </option>
      <option name="INNER_CLASS_OPTIONS">
        <value>
          <option name="ACCESS_JAVADOC_REQUIRED_FOR" value="none" />
          <option name="REQUIRED_TAGS" value="" />
        </value>
      </option>
      <option name="METHOD_OPTIONS">
        <value>
          <option name="ACCESS_JAVADOC_REQUIRED_FOR" value="none" />
          <option name="REQUIRED_TAGS" value="@return@param@throws or @exception" />
        </value>
      </option>
      <option name="FIELD_OPTIONS">
        <value>
          <option name="ACCESS_JAVADOC_REQUIRED_FOR" value="none" />
          <option name="REQUIRED_TAGS" value="" />
        </value>
      </option>
      <option name="IGNORE_DEPRECATED" value="false" />
      <option name="IGNORE_JAVADOC_PERIOD" value="true" />
      <option name="IGNORE_DUPLICATED_THROWS" value="false" />
      <option name="IGNORE_POINT_TO_ITSELF" value="false" />
      <option name="myAdditionalJavadocTags" value="date" />
    </inspection_tool>
  </profile>
</component>

.idea/inspectionProfiles/profiles_settings.xml

@@ -1,7 +0,0 @@
<component name="InspectionProjectProfileManager">
  <settings>
    <option name="PROJECT_PROFILE" value="Project Default" />
    <option name="USE_PROJECT_PROFILE" value="true" />
    <version value="1.0" />
  </settings>
</component>

.idea/modules.xml

@@ -2,6 +2,7 @@
<project version="4">
  <component name="ProjectModuleManager">
    <modules>
      <module fileurl="file://$PROJECT_DIR$/common-activemq-starter/common-activemq-starter.iml" filepath="$PROJECT_DIR$/common-activemq-starter/common-activemq-starter.iml" />
      <module fileurl="file://$PROJECT_DIR$/common-async-starter/common-async-starter.iml" filepath="$PROJECT_DIR$/common-async-starter/common-async-starter.iml" />
      <module fileurl="file://$PROJECT_DIR$/common-cache-starter/common-cache-starter.iml" filepath="$PROJECT_DIR$/common-cache-starter/common-cache-starter.iml" />
      <module fileurl="file://$PROJECT_DIR$/common-data-es-starter/common-data-es-starter.iml" filepath="$PROJECT_DIR$/common-data-es-starter/common-data-es-starter.iml" />
@@ -11,10 +12,9 @@
      <module fileurl="file://$PROJECT_DIR$/common-data-redis-starter/common-data-redis-starter.iml" filepath="$PROJECT_DIR$/common-data-redis-starter/common-data-redis-starter.iml" />
      <module fileurl="file://$PROJECT_DIR$/common-data-solr-starter/common-data-solr-starter.iml" filepath="$PROJECT_DIR$/common-data-solr-starter/common-data-solr-starter.iml" />
      <module fileurl="file://$PROJECT_DIR$/common-logback-starter/common-logback-starter.iml" filepath="$PROJECT_DIR$/common-logback-starter/common-logback-starter.iml" />
      <module fileurl="file://$PROJECT_DIR$/common-activemq-starter/common-mq.iml" filepath="$PROJECT_DIR$/common-activemq-starter/common-mq.iml" />
      <module fileurl="file://$PROJECT_DIR$/common-quartz-starter/common-quartz.iml" filepath="$PROJECT_DIR$/common-quartz-starter/common-quartz.iml" />
      <module fileurl="file://$PROJECT_DIR$/common-security-starter/common-security.iml" filepath="$PROJECT_DIR$/common-security-starter/common-security.iml" />
      <module fileurl="file://$PROJECT_DIR$/common-swagger-starter/common-swagger.iml" filepath="$PROJECT_DIR$/common-swagger-starter/common-swagger.iml" />
      <module fileurl="file://$PROJECT_DIR$/common-quartz-starter/common-quartz-starter.iml" filepath="$PROJECT_DIR$/common-quartz-starter/common-quartz-starter.iml" />
      <module fileurl="file://$PROJECT_DIR$/common-security-starter/common-security-starter.iml" filepath="$PROJECT_DIR$/common-security-starter/common-security-starter.iml" />
      <module fileurl="file://$PROJECT_DIR$/common-swagger-starter/common-swagger-starter.iml" filepath="$PROJECT_DIR$/common-swagger-starter/common-swagger-starter.iml" />
      <module fileurl="file://$PROJECT_DIR$/demo/demo.iml" filepath="$PROJECT_DIR$/demo/demo.iml" />
      <module fileurl="file://$PROJECT_DIR$/jkzl-start.iml" filepath="$PROJECT_DIR$/jkzl-start.iml" />
    </modules>

common-data-hbase-starter/src/main/java/com/yihu/base/hbase/AbstractHBaseClient.java

@@ -5,6 +5,7 @@ import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.springframework.data.hadoop.hbase.HbaseTemplate;
/**
 * AbstractHBaseClient - 基类
 * @author hzp
 * @created 2017.05.03
 */

common-data-hbase-starter/src/main/java/com/yihu/base/hbase/HBaseAdmin.java

@@ -16,6 +16,7 @@ import java.util.ArrayList;
import java.util.List;
/**
 * Hbase - DDL
 * @author hzp
 * @created 2017.05.03
 */
@@ -27,70 +28,103 @@ public class HBaseAdmin extends AbstractHBaseClient {
    /**
     * 判断表是否存在
     * @param tableName
     * @return
     * @throws Exception
     */
    public boolean isTableExists(String tableName) throws Exception {
        Connection connection = getConnection();
        Admin admin = connection.getAdmin();
        boolean ex = admin.tableExists(TableName.valueOf(tableName));
        Connection connection = null;
        Admin admin = null;
        try {
            connection = getConnection();
            admin = connection.getAdmin();
            return admin.tableExists(TableName.valueOf(tableName));
        } finally {
            if (admin != null) {
                admin.close();
            }
            if (connection != null) {
                connection.close();
            }
        }
        admin.close();
        connection.close();
        return ex;
    }
    /**
     * 创建表
     * 新建表
     * @param tableName
     * @param columnFamilies
     * @throws Exception
     */
    public void createTable(String tableName, String... columnFamilies) throws Exception {
        Connection connection = getConnection();
        Admin admin = connection.getAdmin();
        if (!admin.tableExists(TableName.valueOf(tableName))) {
            HTableDescriptor tableDescriptor = new HTableDescriptor(TableName.valueOf(tableName));
            for (String fc : columnFamilies) {
                tableDescriptor.addFamily(new HColumnDescriptor(fc));
        Connection connection = null;
        Admin admin = null;
        try {
            connection = getConnection();
            admin = connection.getAdmin();
            if (!admin.tableExists(TableName.valueOf(tableName))) {
                HTableDescriptor tableDescriptor = new HTableDescriptor(TableName.valueOf(tableName));
                for (String fc : columnFamilies) {
                    tableDescriptor.addFamily(new HColumnDescriptor(fc));
                }
                admin.createTable(tableDescriptor);
            }
        }finally {
            if (admin != null) {
                admin.close();
            }
            if (connection != null) {
                connection.close();
            }
            admin.createTable(tableDescriptor);
        }
        admin.close();
        connection.close();
    }
    /**
     * 模糊匹配表名
     * @param regex 表达式
     * @param includeSysTables 是否包含系统表
     * @return
     * @throws Exception
     */
    public List<String> getTableList(String regex, boolean includeSysTables) throws Exception {
        Connection connection = getConnection();
        Admin admin = connection.getAdmin();
        TableName[] tableNames;
        if (regex == null || regex.length() == 0) {
            tableNames = admin.listTableNames();
        } else {
            tableNames = admin.listTableNames(regex, includeSysTables);
        }
        Connection connection = null;
        Admin admin = null;
        List<String> tables = new ArrayList<>();
        for (TableName tableName : tableNames) {
            tables.add(tableName.getNameAsString());
        try {
            connection = getConnection();
            admin = connection.getAdmin();
            TableName[] tableNames;
            if (regex == null || regex.length() == 0) {
                tableNames = admin.listTableNames();
            } else {
                tableNames = admin.listTableNames(regex, includeSysTables);
            }
            for (TableName tableName : tableNames) {
                tables.add(tableName.getNameAsString());
            }
            return tables;
        }finally {
            if (admin != null) {
                admin.close();
            }
            if (connection != null) {
                connection.close();
            }
        }
        admin.close();
        connection.close();
        return tables;
    }
    /**
     * 批量清空表数据
     * 批量清空表数据 （直接删除相关表，再新建）
     * @param tables
     * @throws Exception
     */
    public void truncate(List<String> tables) throws Exception {
        Connection connection = getConnection();
        Admin admin = connection.getAdmin();
    public void cleanTable(List<String> tables) throws Exception {
        Connection connection = null;
        Admin admin = null;
        try {
            connection = getConnection();
            admin = connection.getAdmin();
            for (String tableName : tables) {
                TableName tn = TableName.valueOf(tableName);
                if (admin.tableExists(TableName.valueOf(tableName))) {
@@ -110,46 +144,71 @@ public class HBaseAdmin extends AbstractHBaseClient {
                }*/
            }
        } finally {
            admin.close();
            connection.close();
            if (admin != null) {
                admin.close();
            }
            if (connection != null) {
                connection.close();
            }
        }
    }
    /**
     * 删除表结构
     * 删除表
     * @param tableName
     * @throws Exception
     */
    public void dropTable(String tableName) throws Exception {
        Connection connection = getConnection();
        Admin admin = connection.getAdmin();
        Connection connection = null;
        Admin admin = null;
        try {
            connection = getConnection();
            admin = connection.getAdmin();
            admin.disableTable(TableName.valueOf(tableName));
            admin.deleteTable(TableName.valueOf(tableName));
        } finally {
            admin.close();
            connection.close();
            if (admin != null) {
                admin.close();
            }
            if (connection != null) {
                connection.close();
            }
        }
    }
    public ObjectNode getTableMetaData(String tableName) {
        return hbaseTemplate.execute(tableName, new TableCallback<ObjectNode>() {
            public ObjectNode doInTable(HTableInterface table) throws Throwable {
    /**
     * 获取表结构
     * @param tableName
     * @return
     * @throws Exception
     */
    public ObjectNode getTableMetaData(String tableName) throws Exception{
        Connection connection = null;
        Admin admin = null;
        try {
            connection = getConnection();
            admin = connection.getAdmin();
            TableName tn = TableName.valueOf(tableName);
            if (admin.tableExists(tn)) {
                ObjectMapper objectMapper = new ObjectMapper();
                ObjectNode root = objectMapper.createObjectNode();
                HTableDescriptor tableDescriptor = table.getTableDescriptor();
                ObjectNode objectNode = objectMapper.createObjectNode();
                HTableDescriptor tableDescriptor = admin.getTableDescriptor(tn);
                HColumnDescriptor[] columnDescriptors = tableDescriptor.getColumnFamilies();
                for (int i = 0; i < columnDescriptors.length; ++i) {
                    HColumnDescriptor columnDescriptor = columnDescriptors[i];
                    root.put(Integer.toString(i), Bytes.toString(columnDescriptor.getName()));
                    objectNode.put(Integer.toString(i), Bytes.toString(columnDescriptor.getName()));
                }
                return root;
                return objectNode;
            }
            return null;
        }finally {
            if (admin != null) {
                admin.close();
            }
        });
            if (connection != null) {
                connection.close();
            }
        }
    }
}

common-data-hbase-starter/src/main/java/com/yihu/base/hbase/HBaseHelper.java

@@ -2,6 +2,7 @@ package com.yihu.base.hbase;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.filter.CompareFilter;
import org.apache.hadoop.hbase.filter.RegexStringComparator;
@@ -10,81 +11,207 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.springframework.data.hadoop.hbase.HbaseTemplate;
import org.springframework.data.hadoop.hbase.RowMapper;
import org.springframework.data.hadoop.hbase.TableCallback;
import org.springframework.util.StringUtils;
import java.util.*;
/**
 * Hbase - DML
 * 数据增删改查
 */
public class HBaseHelper extends AbstractHBaseClient {
    ObjectMapper objectMapper=new ObjectMapper();
    private ObjectMapper objectMapper = new ObjectMapper();
    public HBaseHelper(HbaseTemplate hbaseTemplate) {
        super(hbaseTemplate);
    }
    /**
     *模糊匹配rowkey
     * 新增数据 - 多列族
     * @param tableName
     * @param rowKey
     * @param family
     * @throws Exception
     */
    public String[] findRowKeys(String tableName, String rowkeyRegEx) throws Exception {
    public void add(String tableName , String rowKey, Map<String, Map<String, String>> family) {
        hbaseTemplate.execute(tableName, new TableCallback<Void>() {
            @Override
            public Void doInTable(HTableInterface table) throws Throwable {
                Put p = new Put(rowKey.getBytes());
                for (String familyName : family.keySet()) {
                    Map<String, String> map = family.get(familyName);
                    for (String qualifier : map.keySet()) {
                        String value = map.get(qualifier);
                        p.addColumn(familyName.getBytes(), qualifier.getBytes(), value.getBytes());
                    }
                }
                table.put(p);
                return null;
            }
        });
    }
    /**
     * 新增数据 - 单列族
     * @param tableName
     * @param rowKey
     * @param family
     * @param columns
     * @param values
     */
    public void add(String tableName, String rowKey, String family, Object[] columns, Object[] values) {
        hbaseTemplate.execute(tableName, new TableCallback<Void>() {
            @Override
            public Void doInTable(HTableInterface table) throws Throwable {
                Put put = new Put(Bytes.toBytes(rowKey));
                for (int j = 0; j < columns.length; j++) {
                    //为空字段不保存
                    if (values[j] != null) {
                        String column = String.valueOf(columns[j]);
                        String value = String.valueOf(values[j]);
                        put.addColumn(Bytes.toBytes(family), Bytes.toBytes(column), Bytes.toBytes(value));
                    }
                }
                table.put(put);
                return null;
            }
        });
    }
    /**
     * 删除记录
     * @param tableName
     * @param rowKey
     */
    public void delete(String tableName, String rowKey)  {
        hbaseTemplate.execute(tableName, new TableCallback<Void>() {
            @Override
            public Void doInTable(HTableInterface table) throws Throwable {
                Delete d = new Delete(rowKey.getBytes());
                table.delete(d);
                return null;
            }
        });
    }
    /**
     * 批量删除数据
     * @param tableName
     * @param rowKeys
     * @return
     * @throws Exception
     */
    public Object[] deleteBatch(String tableName, String[] rowKeys) {
        return hbaseTemplate.execute(tableName, new TableCallback<Object[]>() {
            @Override
            public Object[] doInTable(HTableInterface table) throws Throwable {
                List<Delete> deletes = new ArrayList<>(rowKeys.length);
                for (String rowKey : rowKeys) {
                    Delete delete = new Delete(Bytes.toBytes(rowKey));
                    deletes.add(delete);
                }
                Object[] results = new Object[deletes.size()];
                table.batch(deletes, results);
                return results;
            }
        });
    }
    /**
     * 删除列族
     * @param tableName
     * @param rowKey
     * @param familyName
     * @throws Exception
     */
    public void deleteFamily(String tableName, String rowKey, String familyName) throws Exception {
        hbaseTemplate.delete(tableName, rowKey, familyName);
    }
    /**
     * 删除某列
     * @param tableName
     * @param rowKey
     * @param familyName
     * @param columnName
     * @throws Exception
     */
    public void deleteColumn(String tableName, String rowKey, String familyName, String columnName) throws Exception {
        hbaseTemplate.delete(tableName, rowKey, familyName, columnName);
    }
    /**
     * 修改某行某列值
     */
    public void put(String tableName ,String rowKey, String familyName, String qualifier, String value) throws Exception {
        hbaseTemplate.put(tableName, rowKey, familyName, qualifier, value.getBytes());
    }
    /**
     * 模糊匹配rowKey
     * @param tableName 表名
     * @param rowKeyRegEx 表达式
     * @return
     * @throws Exception
     */
    public String[] findRowKeys(String tableName, String rowKeyRegEx) throws Exception {
        Scan scan = new Scan();
        scan.addFamily(Bytes.toBytes("basic"));
        scan.setFilter(new RowFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator(rowkeyRegEx)));
        scan.setStartRow(rowKeyRegEx.substring(1, rowKeyRegEx.length()).getBytes());
        //scan.setStopRow(rowKeyRegEx.substring(1, rowKeyRegEx.length()).getBytes());
        scan.setFilter(new RowFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator(rowKeyRegEx)));
        List<String> list = new LinkedList<>();
        hbaseTemplate.find(tableName, scan, new RowMapper<Void>() {
            @Override
            public Void mapRow(Result result, int rowNum) throws Exception {
                list.add(Bytes.toString(result.getRow()));
                return null;
            }
        });
        return list.toArray(new String[list.size()]);
    }
    /**
     *表总条数
     * 表总条数
     * @param tableName
     * @return
     * @throws Exception
     */
    public Integer count(String tableName) throws Exception {
        Scan scan = new Scan();
        scan.addFamily(Bytes.toBytes("basic"));
        scan.setFilter(new RowFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator("^")));
        List<String> list = new LinkedList<>();
        hbaseTemplate.find(tableName, scan, new RowMapper<Void>() {
            @Override
            public Void mapRow(Result result, int rowNum) throws Exception {
                list.add(Bytes.toString(result.getRow()));
                return null;
            }
        });
        return list.size();
    }
    /**
     * 根据 rowkey获取一条记录
     * 根据rowKey获取一条记录
     * @param tableName
     * @param rowKey
     * @return 字符串
     */
    public String get(String tableName, String rowkey) {
        return hbaseTemplate.get(tableName, rowkey,new RowMapper<String>() {
    public String get(String tableName, String rowKey) {
        return hbaseTemplate.get(tableName, rowKey, new RowMapper<String>() {
            @Override
            public String mapRow(Result result, int rowNum) throws Exception {
                if(!result.isEmpty())
                {
                if(!result.isEmpty()) {
                    List<Cell> ceList = result.listCells();
                    Map<String, Object> map = new HashMap<String, Object>();
                    map.put("rowkey",rowkey);
                    if (ceList != null && ceList.size() > 0) {
                        for (Cell cell : ceList) {
                            //默认不加列族
                            // Bytes.toString(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength()) +"_"
                            map.put(Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()),
                                    Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
                        }
                    map.put("rowkey", rowKey);
                    for (Cell cell : ceList) {
                        // 默认不加列族
                        // Bytes.toString(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength()) +"_"
                        map.put(Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()),
                                Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
                    }
                    return objectMapper.writeValueAsString(map);
                }
@@ -96,29 +223,43 @@ public class HBaseHelper extends AbstractHBaseClient {
    }
    /**
     * 通过表名  key 和 列族 和列 获取一个数据
     * 通过rowKey获取某行数据
     * @param tableName
     * @param rowKey
     * @return Map
     */
    public String get(String tableName ,String rowkey, String familyName, String qualifier) {
        return hbaseTemplate.get(tableName, rowkey,familyName,qualifier ,new RowMapper<String>(){
            public String mapRow(Result result, int rowNum) throws Exception {
                List<Cell> ceList =   result.listCells();
                String res = "";
                if(ceList!=null&&ceList.size()>0){
                    for(Cell cell:ceList){
                        res = Bytes.toString( cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
    public Map<String, Object> getResultMap(String tableName, String rowKey) {
        return hbaseTemplate.get(tableName, rowKey, new RowMapper<Map<String, Object>>() {
            @Override
            public Map<String, Object> mapRow(Result result, int rowNum) throws Exception {
                if(!result.isEmpty()) {
                    List<Cell> ceList = result.listCells();
                    Map<String, Object> map = new HashMap<String, Object>();
                    map.put("rowkey", rowKey);
                    for (Cell cell : ceList) {
                        //默认不加列族
                        // Bytes.toString(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength()) +"_"
                        map.put(Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()),
                                Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
                    }
                    return map;
                }else {
                    return null;
                }
                return res;
            }
        });
    }
    /**
     * 通过rowkey获取某行数据
     * 通过rowKey获取某行数据
     * @param tableName
     * @param rowKey
     * @return
     * @throws Exception
     */
    public Result getResult(String tableName, String rowKey) throws Exception {
        return hbaseTemplate.get(tableName, rowKey, new RowMapper<Result>() {
            @Override
            public Result mapRow(Result result, int rowNum) throws Exception {
                return result;
            }
@@ -126,40 +267,23 @@ public class HBaseHelper extends AbstractHBaseClient {
    }
    /**
     * 通过rowkey获取多行数据
     * 通过表名和rowKey获取指定列族下的值
     * @param tableName 表名
     * @param rowKey rowKey
     * @param familyName 列族
     * @return
     */
    public Result[] getResultList(String tableName,List<String> rowKeys) throws Exception {
        return hbaseTemplate.execute(tableName, new TableCallback<Result[]>() {
            public Result[] doInTable(HTableInterface table) throws Throwable {
                List<Get> list = new ArrayList<Get>();
                for (String rowKey : rowKeys) {
                    Get get = new Get(Bytes.toBytes(rowKey));
                    list.add(get);
                }
                return  table.get(list);
            }
        });
    }
    /**
     * 通过rowkey获取某行数据
     */
    public Map<String, Object> getResultMap(String tableName, String rowKey) throws Exception {
        return hbaseTemplate.get(tableName, rowKey, new RowMapper<Map<String, Object>>() {
            public Map<String, Object> mapRow(Result result, int rowNum) throws Exception {
                Map<String, Object> map = null;
                if(result!=null) {
                    List<Cell> ceList = result.listCells();
                    if (ceList != null && ceList.size() > 0) {
                        map = new HashMap<String, Object>();
                        map.put("rowkey", rowKey);
                        for (Cell cell : ceList) {
                            //默认不加列族
                            // Bytes.toString(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength()) +"_"
                            map.put(Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()),
                                    Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
                        }
    public Map<String, String> get(String tableName, String rowKey, String familyName) {
        return hbaseTemplate.get(tableName, rowKey, familyName, new RowMapper<Map<String, String>>(){
            @Override
            public Map<String, String> mapRow(Result result, int rowNum) throws Exception {
                Map<String, String> map = new HashMap<>();
                NavigableMap<byte[], byte[]> navigableMaps = result.getFamilyMap(familyName.getBytes());
                if(null != navigableMaps) {
                    for (byte[] key : navigableMaps.keySet()) {
                        String keys = new String(key);
                        String values = new String(navigableMaps.get(key));
                        map.put(keys, values);
                    }
                }
                return map;
@@ -167,177 +291,86 @@ public class HBaseHelper extends AbstractHBaseClient {
        });
    }
    /**
     * 修改某行某列值
     * 通过表名和rowKey获取指定列族下的列名的值
     * @param tableName 表名
     * @param rowKey rowKey
     * @param familyName 列族
     * @param qualifier 列名
     * @return
     */
    public void put(String tableName ,String rowkey, String familyName, String qualifier,String value) throws Exception
    {
        hbaseTemplate.execute(tableName, new TableCallback<String>() {
            public String doInTable(HTableInterface table) throws Throwable {
                Put p = new Put(rowkey.getBytes());
                p.add(familyName.getBytes(), qualifier.getBytes(), value.getBytes());
                table.put(p);
                return null;
    public String get(String tableName, String rowKey, String familyName, String qualifier) {
        return hbaseTemplate.get(tableName, rowKey, familyName, qualifier, new RowMapper<String>(){
            @Override
            public String mapRow(Result result, int rowNum) throws Exception {
                Cell cell = result.getColumnLatestCell(familyName.getBytes(), qualifier.getBytes());
                return new String(CellUtil.cloneValue(cell));
            }
        });
    }
    /**
     * 新增行
     * 通过rowKey集合获取指定列名下的多条数据
     * @param tableName 表名
     * @param rowKeys rowKeys
     * @param basicFl basic列族下的列名
     * @param dFl d列族下的列名
     * @return
     * @throws Exception
     */
    public void add(String tableName , String rowkey, Map<String,Map<String,String>> family) throws Exception
    {
        hbaseTemplate.execute(tableName, new TableCallback<String>() {
            public String doInTable(HTableInterface table) throws Throwable {
                Put p = new Put(rowkey.getBytes());
                for(String familyName : family.keySet())
                {
                    Map<String,String> map = family.get(familyName);
                    for (String qualifier : map.keySet())
                    {
                        String value = map.get(qualifier);
                        if(value == null){
                            continue;
    public Result[] getResultList(String tableName, List<String> rowKeys, String basicFl, String dFl) {
        return hbaseTemplate.execute(tableName, new TableCallback<Result[]>() {
            @Override
            public Result[] doInTable(HTableInterface table) throws Throwable {
                List<Get> list = new ArrayList<Get>();
                for (String rowKey : rowKeys) {
                    Get get = new Get(Bytes.toBytes(rowKey));
                    if (!StringUtils.isEmpty(basicFl)) {
                        String[] basicArr = basicFl.split(",");
                        for (String basicStr : basicArr) {
                            get.addColumn(Bytes.toBytes("basic"), Bytes.toBytes(basicStr));
                        }
                        p.add(familyName.getBytes(), qualifier.getBytes(), value.getBytes());
                    }
                }
                table.put(p);
                return null;
            }
        });
    }
    /**
     * 批量新增行
     */
    public void addBulk(String tableName , List<String> rowkeyList, List<Map<String,Map<String,String>>> familyList) throws Exception
    {
        hbaseTemplate.execute(tableName, new TableCallback<String>() {
            public String doInTable(HTableInterface table) throws Throwable {
                List<Put> list = new ArrayList<>();
                for(int i = 0; i < rowkeyList.size();i++){
                    Put p = new Put(rowkeyList.get(i).getBytes());
                    Map<String,Map<String,String>> family = familyList.get(i);
                    for(String familyName : family.keySet())
                    {
                        Map<String,String> map = family.get(familyName);
                        for (String qualifier : map.keySet())
                        {
                            String value = map.get(qualifier);
                            if(value == null){
                                continue;
                            }
                            p.add(familyName.getBytes(), qualifier.getBytes(), value.getBytes());
                    if (!StringUtils.isEmpty(dFl)) {
                        String[] dArr = dFl.split(",");
                        for (String dStr : dArr) {
                            get.addColumn(Bytes.toBytes("d"), Bytes.toBytes(dStr));
                        }
                    }
                    list.add(p);
                    list.add(get);
                }
                table.put(list);
                return null;
                return table.get(list);
            }
        });
    }
    /**
     * 新增数据
     */
    public void add(String tableName, String rowKey, String family, Object[] columns, Object[] values) throws Exception {
        hbaseTemplate.execute(tableName, new TableCallback<Object>() {
            public Object doInTable(HTableInterface htable) throws Throwable {
                Put put = new Put(Bytes.toBytes(rowKey));
                for (int j = 0; j < columns.length; j++) {
                    //为空字段不保存
                    if(values[j]!=null)
                    {
                        String column = String.valueOf(columns[j]);
                        String value = String.valueOf(values[j]);
                        put.addColumn(Bytes.toBytes(family),
                                Bytes.toBytes(column),
                                Bytes.toBytes(value));
                    }
                }
                htable.put(put);
                return null;
            }
        });
    }
    /************************************* Bean使用原型模式 ***************************************************************/
    /**
     * 根据 rowkey删除一条记录
     * 保存数据 原型模式
     */
    public void delete(String tableName, String rowkey)  {
        hbaseTemplate.execute(tableName, new TableCallback<String>() {
            public String doInTable(HTableInterface table) throws Throwable {
                Delete d = new Delete(rowkey.getBytes());
                table.delete(d);
    public void save(String tableName, TableBundle tableBundle) {
        hbaseTemplate.execute(tableName, new TableCallback<Void>() {
            @Override
            public Void doInTable(HTableInterface table) throws Throwable {
                List<Put> puts = tableBundle.putOperations();
                Object[] results = new Object[puts.size()];
                table.batch(puts, results);
                return null;
            }
        });
    }
    /**
     * 批量删除数据
     * 删除数据 原型模式
     */
    public Object[] deleteBatch(String tableName, String[] rowKeys) throws Exception {
        return hbaseTemplate.execute(tableName, new TableCallback<Object[]>() {
    public void delete(String tableName, TableBundle tableBundle) {
        hbaseTemplate.execute(tableName, new TableCallback<Object[]>() {
            @Override
            public Object[] doInTable(HTableInterface table) throws Throwable {
                List<Delete> deletes = new ArrayList<>(rowKeys.length);
                for (String rowKey : rowKeys) {
                    Delete delete = new Delete(Bytes.toBytes(rowKey));
                    deletes.add(delete);
                }
                List<Delete> deletes = tableBundle.deleteOperations();
                Object[] results = new Object[deletes.size()];
                table.batch(deletes, results);
                return results;
            }
        });
    }
    /**
     * 删除列族
     */
    public void deleteFamily(String tableName, String rowKey, String familyName) throws Exception {
        hbaseTemplate.delete(tableName, rowKey, familyName);
    }
    /**
     * 删除某列
     */
    public void deleteColumn(String tableName, String rowKey, String familyName, String columnName) throws Exception {
        hbaseTemplate.delete(tableName, rowKey, familyName, columnName);
    }
    /************************************* Bean使用原型模式 ***************************************************************/
    /**
     * 保存数据 原型模式
     */
    public void save(String tableName, TableBundle tableBundle) throws Exception {
        hbaseTemplate.execute(tableName, new TableCallback<Object>() {
            public Object doInTable(HTableInterface htable) throws Throwable {
                List<Put> puts = tableBundle.putOperations();
                Object[] results = new Object[puts.size()];
                htable.batch(puts, results);
                return null;
            }
        });
@@ -348,34 +381,16 @@ public class HBaseHelper extends AbstractHBaseClient {
     */
    public Object[] get(String tableName, TableBundle tableBundle) {
        return hbaseTemplate.execute(tableName, new TableCallback<Object[]>() {
            @Override
            public Object[] doInTable(HTableInterface table) throws Throwable {
                List<Get> gets = tableBundle.getOperations();
                Object[] results = new Object[gets.size()];
                table.batch(gets, results);
                if (results.length > 0 && results[0].toString().equals("keyvalues=NONE")) return null;
                if (results.length > 0 && results[0].toString().equals("keyvalues=NONE")) {
                    return null;
                }
                return results;
            }
        });
    }
    /**
     * 删除数据 原型模式
     */
    public void delete(String tableName, TableBundle tableBundle) {
        hbaseTemplate.execute(tableName, new TableCallback<Object[]>() {
            public Object[] doInTable(HTableInterface table) throws Throwable {
                List<Delete> deletes = tableBundle.deleteOperations();
                Object[] results = new Object[deletes.size()];
                table.batch(deletes, results);
                return null;
            }
        });
    }
}

common-data-hbase-starter/src/main/java/com/yihu/base/hbase/config/HbaseConfig.java

@@ -1,10 +1,8 @@
package com.yihu.base.hbase.config;
import com.yihu.base.hbase.HBaseHelper;
import com.yihu.base.hbase.HBaseAdmin;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
@@ -12,7 +10,6 @@ import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.hadoop.hbase.HbaseTemplate;
import org.springframework.data.hadoop.hbase.TableCallback;
import java.util.HashMap;
import java.util.HashSet;
@@ -27,7 +24,8 @@ import java.util.Set;
@Configuration
@ConfigurationProperties(prefix = "hadoop")
public class HbaseConfig {
    private static Logger logger = LoggerFactory.getLogger(HbaseConfig.class);
    private static final Logger logger = LoggerFactory.getLogger(HbaseConfig.class);
    private Map<String, String> hbaseProperties = new HashMap<>();
    public Map<String, String> getHbaseProperties() {
@@ -46,23 +44,20 @@ public class HbaseConfig {
            hbaseProperties.put(key, value);
        }
        org.apache.hadoop.conf.Configuration configuration = HBaseConfiguration.create();
        hbaseProperties.keySet().stream().filter(key -> hbaseProperties.get(key) != null).forEach(key -> {
            configuration.set(key, hbaseProperties.get(key));
        });
        return configuration;
    }
    @Bean
    public HbaseTemplate hbaseTemplate(org.apache.hadoop.conf.Configuration configuration) {
        logger.info("set System property for hbase ---", user);
        System.setProperty("HADOOP_USER_NAME", user);
        HbaseTemplate hbaseTemplate = new HbaseTemplate();
        hbaseTemplate.setConfiguration(configuration);
        try {
            logger.info("set System property for hbase ---", user);
            System.setProperty("HADOOP_USER_NAME", user);
        /*try {
            String tableName = "HealthProfile";
            //覆盖默认的配置文件
            org.apache.hadoop.conf.Configuration.addDefaultResource("core-site.xml");
@@ -87,14 +82,20 @@ public class HbaseConfig {
            connection.close();
        } catch (Exception ex) {
            logger.info("Hbase createConnection failure", ex.getMessage());
        }
        }*/
        return hbaseTemplate;
    }
    @Bean
    public HBaseHelper hBaseHelper(HbaseTemplate hbaseTemplate) {
        HBaseHelper hBaseHelper = new HBaseHelper(hbaseTemplate);
        return hBaseHelper;
    }
    @Bean
    public HBaseAdmin hBaseAdmin(HbaseTemplate hbaseTemplate) {
        HBaseAdmin hBaseAdmin = new HBaseAdmin(hbaseTemplate);
        return hBaseAdmin;
    }
}

common-data-hbase-starter/src/main/resources/template.yml

@@ -1,7 +1,7 @@
hadoop:
  hbase-properties:
    hbase.zookeeper.property.clientPort: 2181 ##端口号
    zookeeper.znode.parent: /hbase-unsecure
    hbase.zookeeper.quorum:  node1.hde.h3c.com,node2.hde.h3c.com,node3.hde.h3c.com ##地址
    hbase.zookeeper.property.clientPort: 2181 #zookeeper端口号
    zookeeper.znode.parent: /hbase-unsecure #zookeeper节点
    hbase.zookeeper.quorum:  node1.hde.h3c.com,node2.hde.h3c.com,node3.hde.h3c.com #zookeeper地址
  user:
    name: root

common-data-solr-starter/src/main/java/com/yihu/base/SolrAdmin.java

@@ -8,12 +8,15 @@ import org.apache.solr.client.solrj.response.UpdateResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrInputDocument;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.config.ConfigurableBeanFactory;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -27,11 +30,12 @@ import java.util.Map;
@Service
@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE)
public class SolrAdmin {
    @Autowired
    SolrPool pool;
    private static final Logger logger = LoggerFactory.getLogger(SolrAdmin.class);
    @Autowired
    private SolrPool pool;
    /************************* 基础操作 **************************************************/
    /**
     * 新建单条索引
     */
@@ -39,21 +43,16 @@ public class SolrAdmin {
        SolrClient client = pool.getConnection(core);
        SolrInputDocument doc = new SolrInputDocument();
        //注意date的格式，要进行适当的转化
        for(String key:map.keySet())
        {
        for(String key:map.keySet()) {
            doc.addField(key, map.get(key));
        }
        UpdateResponse re = client.add(doc);
        client.commit();
        pool.close(core); //释放连接
        if(re.getStatus()!=0) {
            System.out.print("create index cost " + re.getQTime());
        if (re.getStatus() != 0) {
            logger.info("create index cost " + re.getQTime());
            return true;
        }
        else{
            System.out.print("create index faild!");
        } else{
            logger.warn("create index failed!");
            return false;
        }
    }
@@ -62,9 +61,9 @@ public class SolrAdmin {
     * 修改单条索引单字段
     */
    public Boolean update(String core,String uniqueKey,String uniqueKeyValue,String key,Object value) throws Exception {
        Map<String,Object> map = new HashedMap();
        map.put(key,value);
        return update(core,uniqueKey+":"+uniqueKeyValue,map);
        Map<String,Object> map = new HashMap();
        map.put(key, value);
        return update(core,uniqueKey + ":" + uniqueKeyValue, map);
    }
    /**
@@ -75,37 +74,28 @@ public class SolrAdmin {
        QueryResponse qr = client.query(new SolrQuery(keyQuery));
        SolrDocumentList docs = qr.getResults();
        if(docs!=null && docs.size()>0)
        {
        if(docs != null && docs.size() > 0) {
            List<SolrInputDocument> solrList = new ArrayList<>();
            for(int i=0;i<docs.size();i++)
            {
            for(int i = 0; i < docs.size(); i++) {
                SolrDocument doc = docs.get(i);
                SolrInputDocument newItem = new SolrInputDocument();
                newItem.addField("rowkey",doc.get("rowkey"));
                for(String key :map.keySet())
                {
                for(String key :map.keySet()) {
                    newItem.addField(key,map.get(key));
                }
                solrList.add(newItem);
            }
            UpdateResponse re = client.add(solrList);
            client.commit();
            pool.close(core); //释放连接
            if(re.getStatus()!=0) {
                System.out.print("update index cost " + re.getQTime());
            if(re.getStatus() != 0) {
                logger.info("update index cost " + re.getQTime());
                return true;
            }
            else{
                System.out.print("update index faild!");
            } else{
                logger.warn("update index failed!");
                return false;
            }
        }
        else{
            System.out.print("Null result!");
        } else{
            logger.warn("Null result!");
        }
        return true;
@@ -119,14 +109,11 @@ public class SolrAdmin {
        SolrClient client = pool.getConnection(core);
        UpdateResponse de = client.deleteByQuery(keyQuery);
        client.commit();
        pool.close(core); //释放连接
        if(de.getStatus()!=0) {
            System.out.print("delete index cost " + de.getQTime());
        if (de.getStatus() != 0) {
            logger.info("delete index cost " + de.getQTime());
            return true;
        }
        else{
            System.out.print("delete index faild!");
        } else{
            logger.warn("delete index failed!");
            return false;
        }
    }

common-data-solr-starter/src/main/java/com/yihu/base/SolrContext.java

@@ -14,7 +14,8 @@ import org.springframework.data.solr.core.SolrTemplate;
 * @version 1.0
 * @created 2016.04.18 18:47
 */
@Configuration
//@Configuration
@Deprecated
public class SolrContext {
    @Value("${spring.data.solr.zk-host}")
    String zkHost;

common-data-solr-starter/src/main/java/com/yihu/base/SolrHelper.java

@@ -12,10 +12,7 @@ import org.springframework.beans.factory.config.ConfigurableBeanFactory;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Service;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
/**
 * Solr底层查询类
@@ -27,26 +24,18 @@ import java.util.Map;
@Service
@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE)
public class SolrHelper {
    @Autowired
    SolrPool pool;
    private final static String ASC = "asc";
    /**
     * 获取查询耗时
     */
    private int qtime;
    private static final String ASC = "asc";
    private int getQtime() {
        return qtime;
    }
    @Autowired
    private SolrPool pool;
    /************************** 查询操作 *****************************************************/
    /**
     * 简单查询方法
     */
    public SolrDocumentList query(String tablename, String q, Map<String, String> sort, long start, long rows) throws Exception {
        return query(tablename, q, null, sort, start, rows);
    public SolrDocumentList query(String tableName, String q, Map<String, String> sort, long start, long rows) throws Exception {
        return query(tableName, q, null, sort, start, rows);
    }
    /**
@@ -62,21 +51,18 @@ public class SolrHelper {
    public SolrDocumentList query(String core, String q, String fq, Map<String, String> sort, long start, long rows) throws Exception {
        SolrClient conn = pool.getConnection(core);
        SolrQuery query = new SolrQuery();
        if (null != q && !q.equals("")) //设置查询条件
        {
        if (null != q && !q.equals("")) { //设置查询条件
            query.setQuery(q);
        } else {
            query.setQuery("*:*");
        }
        if (null != fq && !fq.equals("")) //设置过滤条件
        {
        if (null != fq && !fq.equals("")) { //设置过滤条件
            query.setFilterQueries(fq);
        }
        query.setStart(Integer.parseInt(String.valueOf(start)));//设置查询起始行
        query.setRows(Integer.parseInt(String.valueOf(rows)));//设置查询行数
        //设置排序
        if (sort != null) {
            for (Object co : sort.keySet()) {
@@ -87,18 +73,54 @@ public class SolrHelper {
                }
            }
        }
        QueryResponse rsp = conn.query(query);
        qtime = rsp.getQTime();
        System.out.print("Solr Query Time:" + qtime);
        SolrDocumentList docs = rsp.getResults();
        pool.close(core); //释放连接
        return docs;
    }
    /******************************* Count 统计 ***********************************************/
    /**
     * Solr查询方法 多个过滤条件
     *
     * @param q     查询字符串
     * @param fq    过滤查询  多个过滤条件
     * @param sort  过滤条件
     * @param start 查询起始行
     * @param rows  查询行数
     * @return
     */
    public SolrDocumentList queryByfqs(String core, String q, String[] fq, Map<String, String> sort, long start, long rows) throws Exception {
        SolrClient conn = pool.getConnection(core);
        SolrQuery query = new SolrQuery();
        if (null != q && !q.equals("")) {
            query.setQuery(q);
        } else {
            query.setQuery("*:*");
        }
        if (null != fq && fq.length > 0) {
            query.setFilterQueries(fq);
        }
        query.setStart(Integer.parseInt(String.valueOf(start)));//设置查询起始行
        query.setRows(Integer.parseInt(String.valueOf(rows)));//设置查询行数
        //设置排序
        if (sort != null) {
            for (Object co : sort.keySet()) {
                if (ASC == sort.get(co).toLowerCase() || ASC.equals(sort.get(co).toLowerCase())) {
                    query.addSort(co.toString(), SolrQuery.ORDER.asc);
                } else {
                    query.addSort(co.toString(), SolrQuery.ORDER.desc);
                }
            }
        }
        QueryResponse rsp = conn.query(query);
        return rsp.getResults();
    }
    /**
     * 总数查询方法
     */
@@ -112,146 +134,169 @@ public class SolrHelper {
    public long count(String core, String q, String fq) throws Exception {
        SolrClient conn = pool.getConnection(core);
        SolrQuery query = new SolrQuery();
        if (null != q && !q.equals("")) //设置查询条件
        {
        if (null != q && !q.equals("")) { //设置查询条件
            query.setQuery(q);
        } else {
            query.setQuery("*:*");
        }
        if (null != fq && !fq.equals("")) //设置过滤条件
        {
        if (null != fq && !fq.equals("")) {
            query.setFilterQueries(fq);
        }
        query.setStart(0);
        query.setRows(0);
        QueryResponse rsp = conn.query(query);
        Integer start =  (int)rsp.getResults().getNumFound();
        query.setStart(start);
        rsp = conn.query(query);
        qtime = rsp.getQTime();
        System.out.print("Solr Count Time:" + qtime);
        SolrDocumentList docs = rsp.getResults();
        long count =  rsp.getResults().getNumFound();
        //query.setStart(start);
        //rsp = conn.query(query);
        //SolrDocumentList docs = rsp.getResults();
        return count;
        pool.close(core);
        return docs.getNumFound();
    }
    /**
     * 单组分组Count统计（start从0开始）
     *
     * @param core       core名
     * @param q          查询条件
     * @param fq         筛选条件
     * @param groupField 分组字段名
     * @param start      起始偏移位
     * @param limit      结果条数，为负数则不限制
     */
    public Map<String, Long> groupCount(String core, String q, String fq, String groupField, int start, int rows) throws Exception {
    public Map<String, Long> groupCount(String core, String q, String fq, String groupField, int start, int limit) throws Exception {
        SolrClient conn = pool.getConnection(core);
        SolrQuery query = new SolrQuery();
        if (null != q && !q.equals("")) //设置查询条件
        {
        if (null != q && !q.equals("")) {
            query.setQuery(q);
        } else {
            query.setQuery("*:*");
        }
        if (null != fq && !fq.equals("")) //设置过滤条件
        {
        if (null != fq && !fq.equals("")) {
            query.setFilterQueries(fq);
        }
        query.setFacet(true);//设置facet=on
        query.setRows(0);
        query.addFacetField(groupField);
        query.setFacetLimit(rows);//限制每次返回结果数
        query.setFacetLimit(limit);//限制每次返回结果数
        query.set(FacetParams.FACET_OFFSET, start);
        query.setFacetMissing(false);//不统计null的值
        query.setFacetMinCount(0);// 设置返回的数据中每个分组的数据最小值，比如设置为0，则统计数量最小为0，不然不显示
        QueryResponse rsp = conn.query(query);
        List<FacetField.Count> countList = rsp.getFacetField(groupField).getValues();
        qtime = rsp.getQTime();
        System.out.print("Solr Group Time:" + qtime);
        Map<String, Long> rmap = new HashMap<String, Long>();
        Map<String, Long> rmap = new HashMap<>();
        for (FacetField.Count count : countList) {
            if (count.getCount() > 0)
                rmap.put(count.getName(), (long) count.getCount());
        }
        pool.close(core);
        return rmap;
    }
    /**
     * 多组分组Count(独立计算)
     *
     * @param core        core名
     * @param q           查询条件
     * @param fq          筛选条件
     * @param groupFields 分组字段名
     */
    public List<FacetField> groupCount(String core, String q, String fq, String[] groups) throws Exception {
    public List<FacetField> groupCount(String core, String q, String fq, String[] groupFields) throws Exception {
        SolrClient conn = pool.getConnection(core);
        SolrQuery query = new SolrQuery();
        if (null != q && !q.equals("")) //设置查询条件
        {
        if (null != q && !q.equals("")) {
            query.setQuery(q);
        } else {
            query.setQuery("*:*");
        }
        if (null != fq && !fq.equals("")) //设置过滤条件
        {
        if (null != fq && !fq.equals("")) {
            query.setFilterQueries(fq);
        }
        query.setFacet(true);//设置facet=on
        query.setRows(0);
        query.addFacetField(groups);
        query.setFacetLimit(1000);//限制每次返回结果数
        query.addFacetField(groupFields);
        query.setFacetLimit(-1); // 限制每次返回结果数
        query.set(FacetParams.FACET_OFFSET, 0);
        query.setFacetMissing(true);//不统计null的值
        query.setFacetMinCount(0);// 设置返回的数据中每个分组的数据最小值，比如设置为0，则统计数量最小为0，不然不显示
        query.setFacetMissing(false); // 不统计null的值
        query.setFacetMinCount(0); // 设置返回的数据中每个分组的数据最小值，比如设置为0，则统计数量最小为0，不然不显示
        QueryResponse rsp = conn.query(query);
        qtime = rsp.getQTime();
        System.out.print("Solr Group Time:" + qtime);
        List<FacetField> facets = rsp.getFacetFields();
        return rsp.getFacetFields();
        pool.close(core);
        return facets;
    }
    /**
     * 多组分组Count统计（关联计算）
     *
     * @return
     * @param core        core名
     * @param q           查询条件
     * @param fq          筛选条件
     * @param groupFields 分组字段名
     * @param start       起始偏移位
     * @param limit       结果条数，为负数则不限制
     */
    public List<PivotField> groupCountMult(String core, String q, String fq, String groupFields, int start, int rows) throws Exception {
    public List<PivotField> groupCountMult(String core, String q, String fq, String groupFields, int start, int limit) throws Exception {
        SolrClient conn = pool.getConnection(core);
        SolrQuery query = new SolrQuery();
        if (null != q && !q.equals("")) //设置查询条件
        {
        if (null != q && !q.equals("")) {
            query.setQuery(q);
        } else {
            query.setQuery("*:*");
        }
        if (null != fq && !fq.equals("")) //设置过滤条件
        {
        if (null != fq && !fq.equals("")) {
            query.setFilterQueries(fq);
        }
        query.setFacet(true);//设置facet=on
        query.setRows(0);
        query.addFacetPivotField(groupFields);
        query.setFacetLimit(rows);//限制每次返回结果数
        query.set(FacetParams.FACET_OFFSET, start);
        query.setFacetLimit(limit);//限制每次返回结果数
        query.setFacetMissing(false);//不统计null的值
        query.setFacetMinCount(0);// 设置返回的数据中每个分组的数据最小值，比如设置为0，则统计数量最小为0，不然不显示
        QueryResponse rsp = conn.query(query);
        qtime = rsp.getQTime();
        System.out.print("Solr Group Time:" + qtime);
        NamedList<List<PivotField>> namedList = rsp.getFacetPivot();
        pool.close(core);
        if (namedList != null && namedList.size() > 0) {
            return namedList.getVal(0);
        } else
        } else {
            return null;
        }
    }
    /**
     * 分组数值统计
     *
     * @param core       表名
     * @param q          查询条件
     * @param statsField 统计字段
     * @return
     */
    public FieldStatsInfo getStats(String core, String q, String fq, String statsField) throws Exception {
        SolrClient conn = pool.getConnection(core);
        SolrQuery query = new SolrQuery();
        if (null != q && !q.equals("")) {
            query.setQuery(q);
        } else {
            query.setQuery("*:*");
        }
        if (null != fq && !fq.equals("")) {
            query.setFilterQueries(fq);
        }
        query.addGetFieldStatistics(statsField);
        query.setRows(0);
        QueryResponse rsp = conn.query(query);
        Map<String, FieldStatsInfo> stats = rsp.getFieldStatsInfo();
        if (stats != null && stats.size() > 0) {
            return stats.get(statsField);
        }
        return null;
    }
    /**************************** 数值统计 ******************************************/
    /**
     * 分组数值统计
     *
@@ -264,14 +309,12 @@ public class SolrHelper {
    public List<FieldStatsInfo> getStats(String core, String q, String fq, String statsField, String groupField) throws Exception {
        SolrClient conn = pool.getConnection(core);
        SolrQuery query = new SolrQuery();
        if (null != q && !q.equals("")) //设置查询条件
        {
        if (null != q && !q.equals("")) {
            query.setQuery(q);
        } else {
            query.setQuery("*:*");
        }
        if (null != fq && !fq.equals("")) //设置过滤条件
        {
        if (null != fq && !fq.equals("")) {
            query.setFilterQueries(fq);
        }
@@ -280,18 +323,13 @@ public class SolrHelper {
        query.setRows(0);
        QueryResponse rsp = conn.query(query);
        qtime = rsp.getQTime();
        System.out.print("Solr Stats Time:" + qtime);
        Map<String, FieldStatsInfo> stats = rsp.getFieldStatsInfo();
        pool.close(core);
        if (stats != null && stats.size() > 0) {
            Map<String, List<FieldStatsInfo>> map = stats.get(statsField).getFacets();
            if (map != null) {
                return map.get(groupField);
            }
        }
        return null;
    }
@@ -299,20 +337,16 @@ public class SolrHelper {
    /**
     * 查询统计
     *
     * @param core       表名
     * @param core       core名
     * @param facetQuery 查询条件
     * @return
     * @throws Exception
     */
    public Map<String, Integer> getFacetQuery(String core, String facetQuery) throws Exception {
        SolrClient conn = pool.getConnection(core);
        SolrQuery query = new SolrQuery();
        query.setQuery("*:*");
        query.setFacet(true);
        query.addFacetQuery(facetQuery);
        QueryResponse resp = conn.query(query);
        return resp.getFacetQuery();
    }
@@ -327,13 +361,10 @@ public class SolrHelper {
     * @param start
     * @param limit
     * @param missing
     * @return
     * @throws Exception
     */
    public FacetField getFacetField(String core, String facetField, String fq, int minCount, int start, int limit, boolean missing) throws Exception {
        SolrClient conn = pool.getConnection(core);
        SolrQuery query = new SolrQuery();
        query.setQuery("*:*");
        if (!StringUtils.isBlank(fq)) {
@@ -351,66 +382,67 @@ public class SolrHelper {
        QueryResponse resp = conn.query(query);
        return resp.getFacetField(facetField);
    }
    /**
     * 日期范围分组统计
     *
     * @param core
     * @param dateField
     * @param startTime
     * @param endTime
     * @param grap
     * @param fq
     * @return
     * @throws Exception
     */
    public List<RangeFacet> getFacetDateRange(String core, String dateField, Date startTime, Date endTime, String grap, String fq) throws Exception {
    public List<RangeFacet> getFacetDateRange(String core, String dateField, Date startTime, Date endTime, String gap, String fq) throws Exception {
        SolrClient conn = pool.getConnection(core);
        SolrQuery query = new SolrQuery();
        query.setQuery("*:*");
        if (!StringUtils.isBlank(fq)) {
            query.setFilterQueries(fq);
        }
        query.setRows(0)
               .setFacet(true)
               .addDateRangeFacet(dateField, startTime, endTime, grap);
                .setFacet(true)
                .addDateRangeFacet(dateField, startTime, endTime, gap);
        QueryResponse resp = conn.query(query);
        return resp.getFacetRanges();
    }
    /**
     * 日期范围分组统计
     */
    public List<RangeFacet> getFacetDateRange(String core, String field, String start, String end, String gap, String fq, String q) throws Exception {
        SolrClient conn = pool.getConnection(core);
        SolrQuery query = new SolrQuery();
        if (StringUtils.isEmpty(q)) {
            query.setQuery("*:*");
        } else {
            query.setQuery(q);
        }
        if (!StringUtils.isEmpty(fq)) {
            query.setFilterQueries(fq);
        }
        query.setRows(0)
                .setFacet(true)
                .setFacetMissing(false)
                .add("facet.range", new String[]{field})
                .add(String.format(Locale.ROOT, "f.%s.%s", new Object[]{field, "facet.range.start"}), new String[]{start})
                .add(String.format(Locale.ROOT, "f.%s.%s", new Object[]{field, "facet.range.end"}), new String[]{end})
                .add(String.format(Locale.ROOT, "f.%s.%s", new Object[]{field, "facet.range.gap"}), new String[]{gap});
        QueryResponse resp = conn.query(query);
        return resp.getFacetRanges();
    }
    /**
     * 数值型字段范围统计
     *
     * @param core
     * @param numField
     * @param start
     * @param end
     * @param grap
     * @param fq
     * @return
     * @throws Exception
     */
    public List<RangeFacet> getFacetNumRange(String core, String numField, int start, int end, int grap, String fq) throws Exception {
    public List<RangeFacet> getFacetNumRange(String core, String field, int start, int end, int gap, String fq) throws Exception {
        SolrClient conn = pool.getConnection(core);
        SolrQuery query = new SolrQuery();
        query.setQuery("*:*");
        if (!StringUtils.isBlank(fq)) {
            query.setFilterQueries(fq);
        }
        query.setRows(0)
                .setFacet(true)
                .addNumericRangeFacet(numField, start, end, grap);
                .addNumericRangeFacet(field, start, end, gap);
        QueryResponse resp = conn.query(query);
        return resp.getFacetRanges();
    }
}

common-data-solr-starter/src/main/java/com/yihu/base/SolrPool.java

@@ -8,6 +8,8 @@ import org.springframework.context.annotation.Scope;
import org.springframework.data.solr.server.support.MulticoreSolrClientFactory;
import org.springframework.stereotype.Service;
import javax.annotation.PreDestroy;
/**
 * Solr连接池
@@ -20,33 +22,37 @@ import org.springframework.stereotype.Service;
public class SolrPool {
    @Value("${spring.data.solr.zk-host}")
    String zkHost;
    private String zkHost;
    private MulticoreSolrClientFactory factory;
    protected MulticoreSolrClientFactory getFactory(){
        if(factory==null)
        {
    protected synchronized MulticoreSolrClientFactory getFactory(){
        if (null == factory) {
            CloudSolrClient client = new CloudSolrClient(zkHost);
            factory = new MulticoreSolrClientFactory(client);
        }
        return factory;
    }
    /**
     * 获取连接
     */
    public SolrClient getConnection(String core) throws Exception{
        if (factory != null) {
            return factory.getSolrClient(core);
        }
        return getFactory().getSolrClient(core);
    }
    @PreDestroy
    private void destroy() {
        if (factory != null) {
            factory.destroy();
        }
    }
    /**
     * 关闭连接
     public void close(SolrClient solrClient) throws Exception{
     solrClient.close();
     }
     */
    public void close(String core) throws Exception{
        getFactory().removeSolrClient(core);
    }
}