完善数据清洗es

patient-co/patient-co-wlyy/src/main/java/com/yihu/wlyy/web/third/jkedu/service/EduArticleService.java

@@ -749,29 +749,24 @@ public class EduArticleService {
     */
    private void deleteEsDataByBatch(String batchNo) {
        JestClient jestClient = null;
        //List<HealthEduArticleES> saveModels = new ArrayList<>();
        try {
            int i = 0;
            jestClient = elasticFactory.getJestClient();
            //先根据条件查找出来
            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
            searchSourceBuilder.query(
                    new BoolQueryBuilder()
                            .must(QueryBuilders.matchQuery("batchNo", batchNo))
            ).size(500000);//一次取10000条
            Search search = new Search.Builder(searchSourceBuilder.toString()).addIndex(esIndex).addType(esType)
                    .build();
            SearchResult result = jestClient.execute(search);
            List<HealthEduArticleES> healthEduArticleESs = result.getSourceAsObjectList(HealthEduArticleES.class);
            //根据id批量删除
            Bulk.Builder bulk = new Bulk.Builder().defaultIndex(esIndex).defaultType(esType);
            for (HealthEduArticleES obj : healthEduArticleESs) {
                Delete index = new Delete.Builder(obj.getId()).build();
                bulk.addAction(index);
            String sql = "select id from " + esIndex + " where batchNo='" + batchNo + "'";
            List<Map<String, Object>> returnList = elasticsearchUtil.excuteDataModel(sql);
            //根据id批量删除
            for (Map<String, Object> obj : returnList) {
                if (obj.containsKey("id")) {
                    i++;
                    Delete index = new Delete.Builder(obj.get("id").toString()).build();
                    bulk.addAction(index);
                }
            }
            BulkResult br = jestClient.execute(bulk.build());
            logger.info("delete data count:" + healthEduArticleESs.size());
            logger.info("delete data count:" + i);
            logger.info("delete flag:" + br.isSucceeded());
        } catch (Exception e) {
            e.printStackTrace();
@@ -1127,39 +1122,23 @@ public class EduArticleService {
     * @throws Exception
     */
    public Map<String, Object> deleteEsOldArticlePatient() throws Exception {
        Map<String, Object> returnMap = new HashedMap();
        int resultSize = 0;
        String countSql ="SELECT " +
                " COUNT(DISTINCT(batch_no)) AS batchCount " +
        String batchNoSql ="SELECT " +
                " DISTINCT(batch_no) batchNo " +
                " FROM " +
                " wlyy_health_edu_article_patient_copy " +
                " WHERE " +
                " admin_team_code IS NOT NULL " +
                " AND batch_no IS NOT NULL";
        Map<String,Object> countMap = jdbcTemplate.queryForMap(countSql);
        if (countMap!=null){
            int size = 1024;
            int batchCount = Integer.valueOf(String.valueOf(countMap.get("batchCount")));
            double pageCount = Math.ceil(Double.valueOf(batchCount) / Double.valueOf(size));
            if (pageCount > 0) {
                String sql = "SELECT batch_no AS batchNo,COUNT(1) batchCount " +
                        "FROM wlyy_health_edu_article_patient_copy " +
                        "WHERE admin_team_code IS NOT NULL AND batch_no IS NOT NULL GROUP BY batch_no limit ?,?";
                for (int i = 1; i <= pageCount; i++) {
                    List<String> batchNoList = new ArrayList<>();
                    int start = (i - 1) * size;
                    List<Map<String,Object>> batchMapList = jdbcTemplate.queryForList(sql,new Object[]{start,size});
                    if (batchMapList!=null && batchMapList.size()>0){
                        for (Map<String,Object> map : batchMapList){
                            batchNoList.add(String.valueOf(map.get("batchNo")));
                        }
                        resultSize += deleteEsByBatchNo(batchNoList);
                    }
                }
        List<Map<String,Object>> batchList = jdbcTemplate.queryForList(batchNoSql);
        List<String> batchNoList =  new ArrayList<>();
        if (batchList!=null && batchList.size()>0){
            for (Map<String,Object> map : batchList){
                batchNoList.add(String.valueOf(map.get("batchNo")));
            }
            resultSize += deleteEsByBatchNo(batchNoList);
        }
        returnMap.put("deleteCount",Integer.valueOf(resultSize));
        return returnMap;
    }