多数据源配置，

common/common-entity/src/main/java/com/yihu/figure_label/entity/FlJobConfig.java

@@ -25,6 +25,7 @@ public class FlJobConfig extends IdEntity {
    private String del;
    private String sourceType;
    private String source;
    private String datasource;
    public String getJobName() {
        return jobName;
@@ -113,4 +114,12 @@ public class FlJobConfig extends IdEntity {
    public void setSource(String source) {
        this.source = source;
    }
    public String getDatasource() {
        return datasource;
    }
    public void setDatasource(String datasource) {
        this.datasource = datasource;
    }
}

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/config/db/DruidConfig.java

@@ -39,6 +39,7 @@ public class DruidConfig {
        //忽略过滤的形式
        filterRegistrationBean.addInitParameter("exclusions", "*.js,*.gif,*.jpg,*.png,*.css,*.ico,/druid/*");
        return filterRegistrationBean;
    }
    @Bean

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/config/db/WlyyJpa.java

@@ -10,6 +10,7 @@ import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.orm.jpa.JpaTransactionManager;
import org.springframework.orm.jpa.JpaVendorAdapter;
import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
@@ -32,7 +33,7 @@ import java.util.Properties;
        entityManagerFactoryRef = "wlyyEntityManagerFactory",
        transactionManagerRef = "wlyyTransactionManager",
        basePackages = {"com.yihu.wlyy.figure.label.dao"})   //设置Repository所在位置
public class WlyyJpa {
public class FigureLabelJpa {
    @Autowired
    private HibernateProperties hibernateProperties;
@@ -41,15 +42,7 @@ public class WlyyJpa {
    @Autowired
    private DruidConfig druidConfig;
    @Bean(name = "wlyyDataSource")
    @Primary
    public DataSource wlyyDataSource() throws SQLException {
        DruidDataSource datasource = new DruidDataSource();
        datasource.setUrl(dataSourceProperties.getWlyy().getUrl());
        datasource.setUsername(dataSourceProperties.getWlyy().getUsername());
        datasource.setPassword(dataSourceProperties.getWlyy().getPassword());
        datasource.setDriverClassName(dataSourceProperties.getDriverClassName());
    private DataSource buildDataSource(DruidDataSource datasource) throws SQLException{
        //configuration
        datasource.setInitialSize(dataSourceProperties.getInitialSize());
        datasource.setMinIdle(dataSourceProperties.getMinIdle());
@@ -76,12 +69,31 @@ public class WlyyJpa {
        return datasource;
    }
    @Bean(name = "wlyyDataSource")
    @Primary
    public DataSource wlyyDataSource() throws SQLException {
        DruidDataSource datasource = new DruidDataSource();
        datasource.setUrl(dataSourceProperties.getWlyy().getUrl());
        datasource.setUsername(dataSourceProperties.getWlyy().getUsername());
        datasource.setPassword(dataSourceProperties.getWlyy().getPassword());
        datasource.setDriverClassName(dataSourceProperties.getDriverClassName());
        return buildDataSource(datasource);
    }
    @Bean(name = "healtharchiveDataSource")
    public DataSource healtharchiveDataSource() throws SQLException {
        DruidDataSource datasource = new DruidDataSource();
        datasource.setUrl(dataSourceProperties.getHealtharchive().getUrl());
        datasource.setUsername(dataSourceProperties.getHealtharchive().getUsername());
        datasource.setPassword(dataSourceProperties.getHealtharchive().getPassword());
        datasource.setDriverClassName(dataSourceProperties.getDriverClassName());
        return buildDataSource(datasource);
    }
    @Bean(name = "wlyyEntityManagerFactory")
    @Primary
    public LocalContainerEntityManagerFactoryBean entityManagerFactoryPrimary(
            @Qualifier("wlyyDataSource") DataSource dataSource) {
    public LocalContainerEntityManagerFactoryBean entityManagerFactoryPrimary(@Qualifier("wlyyDataSource") DataSource dataSource) {
        LocalContainerEntityManagerFactoryBean emfb = new LocalContainerEntityManagerFactoryBean();
        emfb.setDataSource(dataSource);
        emfb.setPackagesToScan("com.yihu.figure_label.entity");
@@ -89,17 +101,13 @@ public class WlyyJpa {
        JpaVendorAdapter vendorAdapter = new HibernateJpaVendorAdapter();
        emfb.setJpaVendorAdapter(vendorAdapter);
        emfb.setJpaProperties(hibernateProperties.hibProperties());
        return emfb;
    }
    @Bean(name = "wlyyTransactionManager")
    @Primary
    JpaTransactionManager transactionManagerSecondary(
            @Qualifier("wlyyEntityManagerFactory") EntityManagerFactory builder) {
    JpaTransactionManager transactionManagerSecondary(@Qualifier("wlyyEntityManagerFactory") EntityManagerFactory builder) {
        return new JpaTransactionManager(builder);
    }
@@ -109,4 +117,15 @@ public class WlyyJpa {
        properties.put("slowSqlMillis", "1000");
        return properties;
    }
    @Bean
    public JdbcTemplate jdbcTemplate(@Qualifier("wlyyDataSource") DataSource dataSource) {
        return new JdbcTemplate(dataSource);
    }
    @Bean(name = "healtharchiveTemplate")
    public JdbcTemplate secondJdbcTemplate(@Qualifier("healtharchiveDataSource") DataSource dataSource) {
        return new JdbcTemplate(dataSource);
    }
}

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/config/db/properties/DataSourceProperties.java

@@ -27,7 +27,9 @@ public class DataSourceProperties {
    private Boolean logAbandoned;
    private String filters;
    private DataSourceLoginProperties wlyy=new DataSourceLoginProperties();
    private DataSourceLoginProperties wlyy = new DataSourceLoginProperties();
    private DataSourceLoginProperties healtharchive = new DataSourceLoginProperties();
    public String getDriverClassName() {
        return driverClassName;
@@ -172,4 +174,12 @@ public class DataSourceProperties {
    public void setWlyy(DataSourceLoginProperties wlyy) {
        this.wlyy = wlyy;
    }
    public DataSourceLoginProperties getHealtharchive() {
        return healtharchive;
    }
    public void setHealtharchive(DataSourceLoginProperties healtharchive) {
        this.healtharchive = healtharchive;
    }
}

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/convert/ConvertHelper.java

@@ -125,6 +125,15 @@ public class ConvertHelper {
        if(dataModel.getId4() != 0){
            ids.append(dataModel.getId4());
        }
        if(!StringUtils.isEmpty(dataModel.getIdstr())){
            ids.append(dataModel.getIdstr());
        }
        if(!StringUtils.isEmpty(dataModel.getIdstr1())){
            ids.append(dataModel.getIdstr1());
        }
        if(!StringUtils.isEmpty(dataModel.getIdstr2())){
            ids.append(dataModel.getIdstr2());
        }
        String[] idArr = ids.toString().split(",");
        String[]sourceArr = sources.split(",");
        if(idArr.length != sourceArr.length){

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/convert/SexConvert.java

@@ -34,10 +34,13 @@ public class SexConvert implements Convert{
        );
        models.forEach(
                model -> {
                    String sex = IdCardUtil.getSexForIdcard(model.getIdcard());
                    SaveModel saveModel = new SaveModel();
                    saveModel.setId(MakeIDUtil.makeSaveModelID(model,""));
                    saveModel.setIdcard(model.getIdcard());
                    saveModel.setLabelType(one.getParentCode());
                    String sex = IdCardUtil.getSexForIdcard(model.getIdcard());
                    saveModel.setLabelCode(sex);
                    saveModel.setLabelName(map.get(sex));
                    saveModel.setCreateTime(DateFormatUtils.format(new Date(), ConstantUtil.date_format));

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/extract/Extracter.java

@@ -16,7 +16,7 @@ public interface Extracter {
     */
//    List<ExtractModel> extract();
    List<DataModel> extractDataByJobConfigsql(String sql);
    List<DataModel> extractDataByJobConfigsql(String sql,String datasource);
}

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/extract/MysqlExtracter.java

@@ -4,10 +4,16 @@ import com.yihu.wlyy.figure.label.model.DataModel;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;
import javax.annotation.PostConstruct;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.*;
/**
@@ -21,12 +27,36 @@ public class MysqlExtracter implements Extracter {
    private int numPerPage = 100000; //一次性最多查询10万条
    private Map<String,JdbcTemplate> templateMap = new HashMap<>();
    @Autowired
    private JdbcTemplate jdbcTemplate;
    @Qualifier("jdbcTemplate")
    private JdbcTemplate wlyydefaultJdbcTemplate;
    @Autowired
    @Qualifier("healtharchiveTemplate")
    private JdbcTemplate healtharchiveTemplate;
    @PostConstruct
    public void initTemplateMap(){
        templateMap.put("wlyy",wlyydefaultJdbcTemplate);
        templateMap.put("healtharchive",healtharchiveTemplate);
    }
    @Override
    public List<DataModel> extractDataByJobConfigsql(String sql) {
        return  getDataByThread(sql);
    public List<DataModel> extractDataByJobConfigsql(String sql,String datasource) {
        JdbcTemplate jdbcTemplate = null;
        // 默认的
        if(StringUtils.isEmpty(datasource)){
            jdbcTemplate = templateMap.get("wlyy");
            return  getDataByThread(sql,jdbcTemplate);
        }
        jdbcTemplate = templateMap.get(datasource);
        if(null == jdbcTemplate){
            logger.error("datasource not exist,check in fl_job_config: "+ datasource);
            return null;
        }
        return  getDataByThread(sql,jdbcTemplate);
    }
    /**
@@ -34,9 +64,9 @@ public class MysqlExtracter implements Extracter {
     * @param sql
     * @return
     */
    public List<DataModel> getDataByThread(String sql){
    public List<DataModel> getDataByThread(String sql,JdbcTemplate jdbcTemplate){
        List<DataModel> datas = new ArrayList<>();
        int size = this.getCount(sql);
        int size = this.getCount(sql,jdbcTemplate);
        int number = size / numPerPage + 1;
        ExecutorService fixedThreadPool =  Executors.newFixedThreadPool(number);
        List<Callable<List<DataModel>>> threadList = new ArrayList<>();
@@ -72,10 +102,9 @@ public class MysqlExtracter implements Extracter {
     * @param sql
     * @return
     */
    public Integer getCount(String sql){
    public Integer getCount(String sql,JdbcTemplate jdbcTemplate){
        String countSql = "select count(*) from " + sql.split("from|FROM")[1];
        Integer count = jdbcTemplate.queryForObject(countSql,Integer.class);
        return count;
    }
}

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/job/Mysql2ESJob.java

@@ -74,6 +74,8 @@ public class Mysql2ESJob implements Job {
    private String source;
    private String datasource;
    /**
     * 数据表的id，有些数据是按时间增量查询，有些数据是按表的主键id增量查询
     */
@@ -111,7 +113,7 @@ public class Mysql2ESJob implements Job {
        this.source = String.valueOf(paramsMap.get("source"));
        this.flJobConfig = flJobConfigDao.findById(this.flJobConfigId);
//        this.sqlFiledValue = String.valueOf(paramsMap.get("sqlFiledValue"));
        this.datasource = flJobConfig.getDatasource();
        this.sqlFiledCondition = paramsMap.getString("sqlFiledCondition");
        this.sqlFiledValue = flJobConfig.getSqlFieldValue();
        this.sqlFiledCondition = ">";
@@ -131,7 +133,7 @@ public class Mysql2ESJob implements Job {
     * 提取数据，按数据库中配置的增量条件提取
     */
    public void extract(){
        this.dataModelList = mysqlExtracter.extractDataByJobConfigsql(this.finalSql);
        this.dataModelList = mysqlExtracter.extractDataByJobConfigsql(this.finalSql,this.datasource);
    }
    /**

patient-co/patient-co-figure-label/src/main/java/com/yihu/wlyy/figure/label/model/DataModel.java

@@ -13,6 +13,13 @@ public class DataModel {
    private int id2;
    private int id3;
    private int id4;
    /**
     * 有些表的id为uuid，是string类型，所以为了兼容所有的，此种类型id命名为idstr，进行数据抽取时需多注意
     */
    private String idstr;
    private String idstr1;
    private String idstr2;
    /**
     * 居民身份证号
     */
@@ -96,6 +103,30 @@ public class DataModel {
        this.id4 = id4;
    }
    public String getIdstr() {
        return idstr;
    }
    public void setIdstr(String idstr) {
        this.idstr = idstr;
    }
    public String getIdstr1() {
        return idstr1;
    }
    public void setIdstr1(String idstr1) {
        this.idstr1 = idstr1;
    }
    public String getIdstr2() {
        return idstr2;
    }
    public void setIdstr2(String idstr2) {
        this.idstr2 = idstr2;
    }
    public String getIdcard() {
        return idcard;
    }

patient-co/patient-co-figure-label/src/main/resources/application-dev.yml

@@ -11,6 +11,11 @@ spring:
      url: jdbc:mysql://172.19.103.77/figure_lable?useUnicode:true&characterEncoding=utf-8&autoReconnect=true
      username: root
      password: 123456
    healtharchive:
      url: jdbc:mysql://172.19.103.50:3306/healtharchive?useUnicode=true&characterEncoding=UTF-8&useSSL=false
      username: chenweishan
      password: chenweishan
  data:
    elasticsearch:
      cluster-nodes-jest: http://172.19.103.45:9200,http://172.19.103.68:9200  #多个逗号分割