指标job 执行 ，增加抽取入库时间条件

src/main/java/com/yihu/quota/etl/extract/ExtractUtil.java

@@ -12,6 +12,7 @@ import com.yihu.quota.vo.DictModel;
import com.yihu.quota.vo.OrgHealthCategoryShowModel;
import com.yihu.quota.vo.QuotaVo;
import com.yihu.quota.vo.SaveModel;
import org.apache.commons.lang3.time.DateUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeanUtils;
@@ -39,6 +40,7 @@ public class ExtractUtil {
    private static String slave_sex = "sex";
    private static String slave_age = "age";
    private static  String unknown = "未知";
    private static String create_date = "create_date";
    @Autowired
    private JdbcTemplate jdbcTemplate;
@@ -209,6 +211,13 @@ public class ExtractUtil {
                        }
                        saveModel.setQuotaDate(date);
                    }
                    if(!StringUtils.isEmpty(create_date)) {
                            if (map.get(create_date) instanceof Date) {
                                Date createDate = (Date)map.get(create_date);
                                Date createDateVal = DateUtils.addHours(createDate, -8);
                                saveModel.setCreateDate(createDateVal);
                            }
                    }
                }
                if(!StringUtils.isEmpty(aggregationKey)){
                    if(map.get(aggregationKey) != null){

src/main/java/com/yihu/quota/etl/extract/solr/SolrExtract.java

@@ -52,6 +52,8 @@ public class SolrExtract {
    @Autowired
    private ObjectMapper objectMapper;
    private static String create_date = "create_date";
    private QuotaVo quotaVo;
    private String startTime;
    private String endTime;
@@ -84,11 +86,14 @@ public class SolrExtract {
        solrQuery.initParams(this.startTime, this.endTime);
        String core = esConfig.getTable(); // solr的core名
        String q = null; // 过滤条件
        String createDateFilter = "";//入库时间过滤条件
        // 统计数据数量
        String timeKey = esConfig.getTimekey();
        if (!StringUtils.isEmpty(timeKey)) {
            if (!StringUtils.isEmpty(startTime) && !StringUtils.isEmpty(endTime)) {
                q = String.format("%s:[%s TO %s]", timeKey, startTime, endTime);
                createDateFilter = String.format("%s:[%s TO %s]", create_date, startTime, endTime);
                q = q + " OR " + createDateFilter;
            } else {
                q = timeKey + ":[* TO *]";
            }
@@ -105,6 +110,7 @@ public class SolrExtract {
        String core = esConfig.getTable(); // solr的core名
        String q = null; // 查询条件
        String fq = null; // 过滤条件
        String createDateFilter = "";//入库时间过滤条件
        if (esConfig.getFilter() != null) {
            fq = esConfig.getFilter();
        }
@@ -145,7 +151,7 @@ public class SolrExtract {
            dimensionGroupList.add(new SolrGroupEntity(key, SolrGroupEntity.GroupType.FIELD_VALUE));
            fl += key + ",";
        }
        fl += timeKey + ",rowkey";
        fl += timeKey + "," + create_date + ",rowkey";
        if (StringUtils.isEmpty(esConfig.getAggregation()) || (!esConfig.getAggregation().equals(Contant.quota.aggregation_list)  && !esConfig.getAggregation().equals(Contant.quota.aggregation_distinct))) {
            // 默认追加一个日期字段作为细维度，方便按天统计作为最小单位统计值。
@@ -161,6 +167,8 @@ public class SolrExtract {
        if (!StringUtils.isEmpty(timeKey)) {
            if (!StringUtils.isEmpty(startTime) && !StringUtils.isEmpty(endTime)) {
                q = String.format("%s:[%s TO %s]", timeKey, startTime, endTime);
                createDateFilter = String.format("%s:[%s TO %s]", create_date, startTime, endTime);
                q = q + " OR " + createDateFilter;
            } else {
                q = timeKey + ":[* TO *]";
            }

src/main/java/com/yihu/quota/job/EsQuotaJob.java

@@ -16,10 +16,7 @@ import com.yihu.quota.vo.QuotaVo;
import com.yihu.quota.vo.SaveModel;
import net.sf.json.JSONObject;
import org.elasticsearch.client.Client;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.QueryStringQueryBuilder;
import org.elasticsearch.index.query.RangeQueryBuilder;
import org.elasticsearch.index.query.*;
import org.joda.time.LocalDate;
import org.quartz.*;
import org.slf4j.Logger;
@@ -188,8 +185,8 @@ public class EsQuotaJob implements Job {
                logger.info(content + dataModels.size());
                haveThreadCount++;
            } else {
                status = Contant.save_status.fail;
                content = time + "没有抽取到数据";
                status = Contant.save_status.success;
                content = "没有抽取到数据" + time ;
                haveThreadCount++;
            }
            // 初始执行时，更新该指标为已初始执行过
@@ -227,9 +224,7 @@ public class EsQuotaJob implements Job {
    private void deleteRecord(QuotaVo quotaVo) throws Exception {
        EsConfig esConfig = extractHelper.getEsConfig(quotaVo.getCode());
        EsConfig sourceEsConfig = extractHelper.getDataSourceEsConfig(quotaVo.getCode());
        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
        QueryStringQueryBuilder termQueryQuotaCode = QueryBuilders.queryStringQuery("quotaCode:" + quotaVo.getCode().replaceAll("_", ""));
        boolQueryBuilder.must(termQueryQuotaCode);
        String start = "";
        String end = "";
        if(sourceEsConfig.getFullQuery() !=null && sourceEsConfig.getFullQuery().equals("true")){
@@ -243,10 +238,20 @@ public class EsQuotaJob implements Job {
                end = endTime;
            }
        }
        RangeQueryBuilder rangeQueryStartTime = QueryBuilders.rangeQuery("quotaDate").gte(start.substring(0, 10));
        boolQueryBuilder.must(rangeQueryStartTime);
        RangeQueryBuilder rangeQueryEndTime = QueryBuilders.rangeQuery("quotaDate").lte(end.substring(0, 10));
        boolQueryBuilder.must(rangeQueryEndTime);
        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
        RangeQueryBuilder rangeQuotaTime = QueryBuilders.rangeQuery("quotaDate").gte(start.substring(0, 10)).lte(end.substring(0, 10));
        RangeQueryBuilder rangeCreateTime = QueryBuilders.rangeQuery("createDate").gte(start.substring(0, 10) + "T00:00:00+0800").lte(end.substring(0, 10) + "T23:59:59+0800");
        String quotaCodeTerm = "quotaCode:" + quotaVo.getCode().replaceAll("_", "");
        QueryStringQueryBuilder termQueryQuotaCode = QueryBuilders.queryStringQuery(quotaCodeTerm);
        QueryBuilder qb = QueryBuilders.boolQuery()
                .must(termQueryQuotaCode)
                .must(QueryBuilders.boolQuery()
                        .should(rangeQuotaTime)
                        .should(rangeCreateTime)
                );
        boolQueryBuilder.filter(qb);
        boolean flag = true ;
        Client talClient = elasticSearchPool.getClient();
        Client client = elasticSearchPool.getClient();

src/main/java/com/yihu/quota/vo/SaveModel.java

@@ -55,6 +55,8 @@ public class SaveModel {
    @CreatedDate
    private Date createTime;//创建时间
    private Date createDate;//档案入库时间
    public String getQuotaDate() {
        return quotaDate;
    }
@@ -374,4 +376,12 @@ public class SaveModel {
    public void setDistinctFieldValue(String distinctFieldValue) {
        this.distinctFieldValue = distinctFieldValue;
    }
    public Date getCreateDate() {
        return createDate;
    }
    public void setCreateDate(Date createDate) {
        this.createDate = createDate;
    }
}