Explorar el Código

mini项目采集线程优化

hzp hace 9 años
padre
commit
8e1f081221

+ 14 - 0
Hos-Resource-Mini/src/main/java/com.yihu.ehr/service/crawler/CrawlerManager.java

@ -55,6 +55,13 @@ public class CrawlerManager {
    }
    public void dataCrawlerFrequency() {
        List<JSONObject> versionList = db.query("select code from std_inner_version order by code desc");
        if(versionList==null)
        {
            LogUtil.info("标准数据还未初始化,请确认!");
            return;
        }
        String uuid = UUID.randomUUID().toString();
        Date lastCralwerTime = DateUtil.getSysDateTime();
        lastCralwerTime = DateUtil.formatYMDToYMDHMS(lastCralwerTime.toString());
@ -74,6 +81,13 @@ public class CrawlerManager {
    }
    public void dataCrawlerSupply() {
        List<JSONObject> versionList = db.query("select code from std_inner_version order by code desc");
        if(versionList==null)
        {
            LogUtil.info("标准数据还未初始化,请确认!");
            return;
        }
        List<JSONObject> jsonList = db.query("select * from crawler_supply where status != 1");
        if (!CollectionUtil.isEmpty(jsonList)) {
            for (JSONObject json : jsonList) {

+ 1 - 0
Hos-Resource-Mini/src/main/java/com.yihu.ehr/service/crawler/DataCollectDispatcher.java

@ -78,6 +78,7 @@ public class DataCollectDispatcher {
            result = false;
        }
        //上传档案
        try {
            if (!CollectionUtil.isEmpty(dataList)) {
                if (!upload(dataList, patient, dataSetMap, token)) {

+ 6 - 55
Hos-Resource-Mini/src/main/java/com.yihu.ehr/service/standard/StandardManager.java

@ -1,19 +1,13 @@
package com.yihu.ehr.service.standard;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.yihu.ehr.common.config.SysConfig;
import com.yihu.ehr.common.constants.Constants;
import com.yihu.ehr.dbhelper.jdbc.DBHelper;
import com.yihu.ehr.util.compress.Zipper;
import com.yihu.ehr.util.encode.Base64;
import com.yihu.ehr.util.encrypt.RSA;
import com.yihu.ehr.util.file.FileUtil;
import com.yihu.ehr.util.httpclient.EsbHttp;
import com.yihu.ehr.util.httpclient.HttpClientUtil;
import com.yihu.ehr.util.httpclient.Response;
import com.yihu.ehr.util.log.LogUtil;
import com.yihu.ehr.util.operator.CollectionUtil;
import com.yihu.ehr.util.operator.SqlCreate;
import com.yihu.ehr.util.operator.StringUtil;
import org.dom4j.Document;
@ -29,15 +23,12 @@ public class StandardManager {
    public static String versionMethod;
    public static String downMethod;
    private static volatile StandardManager instance;
    private DBHelper db;
    private SysConfig sysConfig;
    private File[] zipFiles;
    private String localVersion;
    public StandardManager(){
        if (db == null) {
            db = new DBHelper();
        }
        sysConfig = SysConfig.getInstance();
    }
@ -76,7 +67,7 @@ public class StandardManager {
                LogUtil.warn("总支撑平台尚无可用版本进行标准升级");
                return false;
            }
            localVersion = getCurrentVersion();
            String localVersion = getCurrentVersion();
            if (StringUtil.isEmpty(localVersion)) {
                return true;
            }
@ -106,6 +97,7 @@ public class StandardManager {
                return false;
            }
            versionUpdate(encryptPwd);
            localVersion = remoteVersion;
        }
        return true;
    }
@ -116,14 +108,9 @@ public class StandardManager {
     * @return String
     */
    public synchronized String getCurrentVersion() {
        if (!StringUtil.isEmpty(localVersion)){
            return localVersion;
        }
        List<JSONObject> list = db.query("select code from std_inner_version order by code desc");
        if (!CollectionUtil.isEmpty(list)) {
            localVersion = list.get(0).getString("code");
        if (StringUtil.isEmpty(localVersion)){
            localVersion = new StandardService().getCurrentVersion();
        }
        return localVersion;
    }
@ -149,7 +136,7 @@ public class StandardManager {
            return false;
        }
        result = genData();
        result = new StandardService().genData(zipFiles);
        if (!result) {
            LogUtil.fatal("生成标准数据失败.");
            return false;
@ -231,42 +218,6 @@ public class StandardManager {
        return true;
    }
    private boolean genData() {
        List<String> sqlList = new ArrayList<>();
        List<String> fileNameList = new ArrayList<>();
        List<String> deleteDataList = new ArrayList<>();
        for (File file : zipFiles) {
            if (!file.getName().contains("_cda")) {
                if (file.getName().contains("org_")||file.getName().contains("adapter_")
                        ||file.getName().contains("std_")) {
                    fileNameList.add(StringUtil.substring(file.getName(), 0, file.getName().indexOf(Constants.DOT)));
                }
                if (!file.getParentFile().getName().equals(Constants.CDA_FILE)) {
                    createData(file, sqlList);
                }
            }
        }
        SqlCreate sqlCreate = new SqlCreate();
        for (String fileName : fileNameList) {
            sqlCreate.setTableName(fileName);
            deleteDataList.add(sqlCreate.deleteData());
        }
        String[] deleteArray = new String[deleteDataList.size()];
        deleteDataList.toArray(deleteArray);
        String[] sqlArray = new String[sqlList.size()];
        sqlList.toArray(sqlArray);
        try {
            db.executeBatch(deleteArray);
            db = new DBHelper();
            db.executeBatch(sqlArray);
        } catch (Exception e) {
            e.printStackTrace();
        }
        return true;
    }
    private void createData(File file, List<String> sqlList) {
        try {
            SAXReader reader = new SAXReader();

+ 189 - 0
Hos-Resource-Mini/src/main/java/com.yihu.ehr/service/standard/StandardService.java

@ -0,0 +1,189 @@
package com.yihu.ehr.service.standard;
import com.yihu.ehr.common.constants.Constants;
import com.yihu.ehr.dbhelper.jdbc.DBConfig;
import com.yihu.ehr.dbhelper.jdbc.DBHelper;
import com.yihu.ehr.util.log.LogUtil;
import com.yihu.ehr.util.operator.CollectionUtil;
import com.yihu.ehr.util.operator.SqlCreate;
import com.yihu.ehr.util.operator.StringUtil;
import org.dom4j.Document;
import org.dom4j.Element;
import org.dom4j.io.SAXReader;
import org.json.JSONObject;
import java.io.File;
import java.sql.Connection;
import java.util.*;
/**
 * 线程取数据,不走连接池
 */
public class StandardService {
    private DBHelper db = new DBHelper();
    /**
     * 获取当前自版本号
     */
    public synchronized String getCurrentVersion() {
        try {
            String localVersion = "";
            List<JSONObject> list = db.query("select code from std_inner_version order by code desc");
            if (!CollectionUtil.isEmpty(list)) {
                localVersion = list.get(0).getString("code");
            }
            return localVersion;
        }
        catch (Exception e)
        {
            LogUtil.error(e);
            return "";
        }
    }
    /**
     * 读取标准数据
     */
    public synchronized boolean genData(File[] zipFiles) {
        List<String> sqlList = new ArrayList<>();
        List<String> fileNameList = new ArrayList<>();
        List<String> deleteDataList = new ArrayList<>();
        for (File file : zipFiles) {
            if (!file.getName().contains("_cda")) {
                if (file.getName().contains("org_")||file.getName().contains("adapter_")
                        ||file.getName().contains("std_")) {
                    fileNameList.add(StringUtil.substring(file.getName(), 0, file.getName().indexOf(Constants.DOT)));
                }
                if (!file.getParentFile().getName().equals(Constants.CDA_FILE)) {
                    createData(file, sqlList);
                }
            }
        }
        SqlCreate sqlCreate = new SqlCreate();
        for (String fileName : fileNameList) {
            sqlCreate.setTableName(fileName);
            deleteDataList.add(sqlCreate.deleteData());
        }
        String[] deleteArray = new String[deleteDataList.size()];
        deleteDataList.toArray(deleteArray);
        String[] sqlArray = new String[sqlList.size()];
        sqlList.toArray(sqlArray);
        try {
            db.executeBatch(deleteArray);
            db = new DBHelper();
            db.executeBatch(sqlArray);
        } catch (Exception e) {
            e.printStackTrace();
        }
        return true;
    }
    private void createData(File file, List<String> sqlList) {
        try {
            SAXReader reader = new SAXReader();
            Document doc = reader.read(file);
            Element root = doc.getRootElement();
            String tableName = root.attributeValue("name");
            if (tableName == null) {
                throw new Exception("错误的标准格式,无表名信息.");
            }
            Element metadata = root.element("metadata");
            if (metadata == null) {
                throw new Exception("错误的标准格式,无表字段信息.");
            }
            HashMap<String, String> colMap = getColMetaData(metadata);
            for (Iterator rowIterator = root.elementIterator("row"); rowIterator.hasNext();) {
                SqlCreate sqlCreate = new SqlCreate();
                sqlCreate.setTableName(tableName);
                Element rowElement = (Element) rowIterator.next();
                createDataSql(rowElement, colMap, sqlList, sqlCreate);
            }
        } catch (Exception e) {
            LogUtil.fatal("生成标准表异常:");
            LogUtil.error(e);
        }
    }
    private HashMap<String, String> getColMetaData(Element rowElem) {
        HashMap<String, String> hashMap = new HashMap<>();
        for (Iterator colIterator = rowElem.elementIterator(); colIterator.hasNext(); ) {
            Element colElement = (Element) colIterator.next();
            String name = colElement.attributeValue("name");
            String type = colElement.attributeValue("type");
            hashMap.put(name, type);
        }
        return hashMap;
    }
    private void createDataSql(Element rowElement, HashMap<String, String> colMap, List<String> sqlList, SqlCreate sqlCreate) {
        if (rowElement == null) {
            return;
        }
        Map<String, String> keyValueMap = new HashMap<>();
        List<String> itemList = new ArrayList<>();
        List<String> itemValueList = new ArrayList<>();
        for (Iterator colIterator = rowElement.elementIterator(); colIterator.hasNext(); ) {
            Element colElement = (Element) colIterator.next();
            if (colElement.getText().equals("")) {
                continue;
            }
            String key = colElement.getName();
            itemList.add(key);
            String type = colMap.get(key);
            if (type == null) {
                LogUtil.fatal("createDataSql无效的类型.");
            }
            String value;
            switch (type) {
                case "N":
                    value = colElement.getText();
                    break;
                case "S":
                    value = "'" + colElement.getText().replace("'","''") + "'";
                    break;
                case "D":
                    value = "'" + colElement.getText() + "'";
                    break;
                default:
                    value = colElement.getText();
            }
            itemValueList.add(value);
            keyValueMap.put(key, value);
            if (key.equals(Constants.TABLE_KEY)) {
                sqlCreate.setTableKey(value);
            }
        }
        sqlCreate.setItemList(itemList);
        sqlCreate.setItemValueList(itemValueList);
        sqlCreate.setKeyValueMap(keyValueMap);
        if (Constants.ADD_TYPE.equals(rowElement.attributeValue("type"))) {
            sqlList.add(sqlCreate.insertData());
        } else if (Constants.UPDATE_TYPE.equals(rowElement.attributeValue("type"))) {
            sqlList.add(sqlCreate.updateDataByTableKey());
            if (sqlCreate.isToAdapter() && sqlCreate.isCodeOrValue()) {
                sqlList.add(sqlCreate.updateAdapterData());
            }
        } else if (Constants.DELETE_TYPE.equals(rowElement.attributeValue("type"))) {
            sqlList.add(sqlCreate.deleteDataByTableKey());
            if (sqlCreate.isToAdapter()) {
                sqlList.add(sqlCreate.deleteAdapterData());
            }
            if (sqlCreate.isToAdapterDataset()) {
                sqlList.add(sqlCreate.deleteAdapterOriginData());
            }
        }
    }
}

+ 1 - 5
Hos-Resource-Mini/src/main/java/com.yihu.ehr/service/thread/CrawlerSupplyThread.java

@ -12,11 +12,7 @@ public class CrawlerSupplyThread implements Runnable {
        while (ThreadManage.crawlerSupplyIsRunning) {
            try {
                LogUtil.info("补采线程开始");
                if (StandardManager.getInstance().isDataInit()) {
                    CrawlerManager.getInstance().dataCrawlerSupply();
                } else {
                    LogUtil.info("标准数据还未初始化,请确认!");
                }
                CrawlerManager.getInstance().dataCrawlerSupply();
                LogUtil.info("补采线程结束");
                sleep();
            } catch (InterruptedException e) {

+ 4 - 6
Hos-Resource-Mini/src/main/java/com.yihu.ehr/service/thread/CrawlerThread.java

@ -1,6 +1,7 @@
package com.yihu.ehr.service.thread;
import com.yihu.ehr.common.config.ThreadConfig;
import com.yihu.ehr.dbhelper.jdbc.DBConfig;
import com.yihu.ehr.dbhelper.jdbc.DBHelper;
import com.yihu.ehr.service.crawler.CrawlerManager;
import com.yihu.ehr.service.standard.StandardManager;
@ -8,16 +9,14 @@ import com.yihu.ehr.util.log.LogUtil;
public class CrawlerThread implements Runnable {
    DBHelper db = new DBHelper(Thread.currentThread().getName(), DBConfig.get("defaultUri"));
    @Override
    public void run() {
        while (ThreadManage.crawlerIsRunning) {
            try {
                LogUtil.info("采集线程开始");
                if (StandardManager.getInstance().isDataInit()) {
                    CrawlerManager.getInstance().dataCrawlerFrequency();
                } else {
                    LogUtil.info("标准数据还未初始化,请确认!");
                }
                CrawlerManager.getInstance().dataCrawlerFrequency();
                LogUtil.info("采集线程结束");
                sleep();
            } catch (InterruptedException e) {
@ -40,7 +39,6 @@ public class CrawlerThread implements Runnable {
    private void sleep() throws Exception {
        if (ThreadConfig.CRAWLER_THREAD_SLEEP_TIME == 0) {
            DBHelper db = new DBHelper();
            Object obj = db.scalar("select param_value from system_param where param_key='INTERVAL'");
            if(obj!=null) {
                ThreadConfig.CRAWLER_THREAD_SLEEP_TIME = Integer.parseInt(obj.toString());

+ 1 - 0
Hos-Resource-Mini/src/main/java/com.yihu.ehr/service/thread/StandardUpdateThread.java

@ -2,6 +2,7 @@ package com.yihu.ehr.service.thread;
import com.yihu.ehr.common.config.SysConfig;
import com.yihu.ehr.common.config.ThreadConfig;
import com.yihu.ehr.dbhelper.jdbc.DBConfig;
import com.yihu.ehr.dbhelper.jdbc.DBHelper;
import com.yihu.ehr.service.standard.StandardManager;
import com.yihu.ehr.util.log.LogUtil;

+ 6 - 5
Hos-Resource-Mini/src/main/java/com.yihu.ehr/util/httpclient/EsbHttp.java

@ -231,8 +231,12 @@ public class EsbHttp {
            if(json!=null && json.has("data"))
            {
                JSONObject p = (JSONObject)json.getJSONArray("data").get(0);
                if(p.has(colName) && p.getString(colName).length()>0)
                if(!p.has(colName) || p.get(colName).equals(null) || p.getString(colName).length()==0)
                {
                    LogUtil.info("注册病人信息请求失败:身份证号码为空,patient_id=" + patient.getPatientId() + ", event_no=" + patient.getEventNo());
                    return false;
                }
                else{
                    String idCord = p.getString(colName);
                    String registerMethod = HttpHelper.defaultHttpUrl + "/patients/"+idCord;
                    if (StringUtil.isEmpty(data)) {
@ -259,10 +263,7 @@ public class EsbHttp {
                        return false;
                    }
                }
                else{
                    LogUtil.info("注册病人信息请求失败:身份证号码为空,patient_id=" + patient.getPatientId() + ", event_no=" + patient.getEventNo());
                    return false;
                }
            }
            else{
                LogUtil.info("注册病人信息请求失败:传入数据无效,patient_id=" + patient.getPatientId() + ", event_no=" + patient.getEventNo());