Merge branch 'master' of demon/esb into master

Hos-Framework/src/main/java/com/yihu/ehr/framework/util/file/FileUtil.java

@@ -254,4 +254,34 @@ public class FileUtil {
        // 目录此时为空，可以删除
        return dir.delete();
    }
    /**
     * InputStream转string
     * @param is 输入流
     * @return Base64编码 文件字节流
     */
    public static String streamToBase64String(InputStream is) {
        byte[] bytes= FileUtil.getBytesByStream(is);
        return Base64.encode(bytes);
    }
    public static  byte[] getBytesByStream(InputStream inputStream){
        byte[] buffer = null;
        try {
            ByteArrayOutputStream bos = new ByteArrayOutputStream(1000);
            byte[] b = new byte[1024];
            int n;
            while ((n = inputStream.read(b)) != -1) {
                bos.write(b, 0, n);
            }
            bos.close();
            buffer = bos.toByteArray();
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return buffer;
    }
}

Hos-Framework/src/main/java/com/yihu/ehr/framework/util/file/FtpFileUtil.java

@@ -0,0 +1,460 @@
package com.yihu.ehr.framework.util.file;
import com.yihu.ehr.framework.util.log.LogService;
import org.apache.commons.net.ftp.FTP;
import org.apache.commons.net.ftp.FTPClient;
import org.apache.commons.net.ftp.FTPFile;
import org.apache.commons.net.ftp.FTPReply;
import java.io.*;
import java.net.SocketException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
 * ftp 操作工具类
 * Created by HZY on 2015/8/12.
 */
public class FtpFileUtil {
    private int port = 21;      // 端口号
    private String username;    //ftp  登录名
    private String password;    //ftp  登录密码
    private String ftpHostName; //ftp  主机名（IP）
    private FTPClient ftpClient = new FTPClient();
    private FileOutputStream fos = null;
    public FtpFileUtil(String username, String password, String ftpHostName, int port) {
        this.username = username;
        this.password = password;
        this.ftpHostName = ftpHostName;
        this.port = port;
    }
    /**
     * 建立ftp连接
     */
    public void connect() {
        try {
            // 连接
            ftpClient.connect(ftpHostName, port);
            int reply = ftpClient.getReplyCode();
            if (!FTPReply.isPositiveCompletion(reply)) {
                ftpClient.disconnect();
            }
            // 登录
            ftpClient.login(username, password);
            ftpClient.setBufferSize(1024 * 1024 * 1);
            ftpClient.setFileType(FTP.BINARY_FILE_TYPE);
//            ftpClient.setControlEncoding("GBK");
            ftpClient.setDefaultTimeout(60 * 1000);
            ftpClient.setConnectTimeout(60 * 1000);
            ftpClient.setDataTimeout(60 * 1000);
            ftpClient.setRemoteVerificationEnabled(false);
        } catch (SocketException e) {
            LogService.getLogger().error("连接Ftp服务器异常：", e);
        } catch (IOException e) {
            LogService.getLogger().error("连接Ftp服务器异常：", e);
        }
    }
    /**
     * 关闭输入输出流
     */
    public void closeConnect() {
        try {
            if (fos != null) {
                fos.close();
            }
            ftpClient.logout();
            ftpClient.disconnect();
        } catch (IOException e) {
            LogService.getLogger().error("关闭Ftp连接失败：", e);
        }
    }
    /**
     * 下载文件
     *
     * @param ftpFileName ftp文件路径
     * @param localDir    本地保存路径
     */
    public boolean down(String ftpFileName, String localDir) {
        connect();
        boolean flag = downFile(ftpFileName, localDir);
        closeConnect();
        return flag;
    }
    /**
     * 上传文件
     *
     * @param localDir    本地文件
     * @param ftpFileName ftp上传路径
     * @return  是否上传成功
     * @throws Exception
     */
    public boolean upload(String localDir, String ftpFileName) throws Exception {
        connect();
        boolean flag = uploadFile(localDir, ftpFileName);
        closeConnect();
        return flag;
    }
    /**
     * 下载单个ftp上文件
     *
     * @param ftpFileName   ftp文件路径
     * @param localDir  本地保存路径
     * @return 是否下载成功
     */
    public boolean downFile(String ftpFileName, String localDir) {
        boolean success = false;
        try {
            File file = new File(ftpFileName);
            File temp = new File(localDir);
            if (!temp.exists()) {
                temp.mkdirs();
            }
            File localfile = new File(localDir + File.separator + file.getName());
            if (!localfile.exists()) {
                fos = new FileOutputStream(localfile);
                success = ftpClient.retrieveFile(ftpFileName, fos);
            }
            ftpClient.changeToParentDirectory();
        } catch (SocketException e) {
            LogService.getLogger().error("Ftp服务器连接失败！", e);
        } catch (IOException e) {
            LogService.getLogger().error("Ftp文件下载失败！", e);
        }
        return success;
    }
    /**
     * 写入文件至ftp
     *
     * @param local  写入到FTP服务器上的文件
     * @param remote FTP服务器保存目录
     * @return 是否上传成功
     * @throws Exception
     */
    public boolean uploadFile(String local, String remote) {
        boolean success = false;
        File localFile = new File(local);
        InputStream in = null;
        try {
            if (!localFile.exists()) {
                return success;
            } else {
                in = new BufferedInputStream(new FileInputStream(localFile));
            }
            int reply = ftpClient.getReplyCode();
            if (!FTPReply.isPositiveCompletion(reply)) {
                ftpClient.disconnect();
                return success;
            }
            createDir(remote);
            ftpClient.changeWorkingDirectory(remote);
            success = ftpClient.storeFile(localFile.getName(), in);
            in.close();
        } catch (IOException e) {
            LogService.getLogger().error("Ftp文件上传失败！", e);
        }
        return success;
    }
    /**
     * 判断是否是目录
     *
     * @param fileName ftp文件路径
     * @return  true 是目录 false 不是
     */
    public boolean isDir(String fileName) {
        try {
            //切换目录，若当前是目录则返回true,否则返回false。
            return ftpClient.changeWorkingDirectory(fileName);
        } catch (Exception e) {
            LogService.getLogger().error("Ftp判断是否是目录失败！", e);
        }
        return false;
    }
    /**
     * 递归创建远程服务器目录
     *
     * @param remote 远程服务器文件绝对路径
     * @return 目录创建是否成功
     * @throws IOException
     */
    public boolean createDir(String remote) throws IOException {
        boolean success = true;
        String directory = remote.substring(0, remote.lastIndexOf("/") + 1);
        // 如果远程目录不存在，则递归创建远程服务器目录
        if (!directory.equalsIgnoreCase("/")
                && !ftpClient.changeWorkingDirectory(new String(directory))) {
            int start = 0;
            int end = 0;
            if (directory.startsWith("/")) {
                start = 1;
            } else {
                start = 0;
            }
            end = directory.indexOf("/", start);
            while (true) {
                String subDirectory = new String(remote.substring(start, end));
                if (!ftpClient.changeWorkingDirectory(subDirectory)) {
                    if (ftpClient.makeDirectory(subDirectory)) {
                        ftpClient.changeWorkingDirectory(subDirectory);
                    } else {
                        LogService.getLogger().debug("创建Ftp目录失败");
                        return false;
                    }
                }
                start = end + 1;
                end = directory.indexOf("/", start);
                // 检查所有目录是否创建完毕
                if (end <= start) {
                    break;
                }
            }
        }
        return success;
    }
    /**
     * 删除文件-FTP方式
     *
     * @param path     FTP服务器上传地址
     * @param fileName FTP服务器上要删除的文件名
     * @return
     */
    public boolean deleteFile(String path, String fileName) {
        boolean success = false;
        try {
            ftpClient.changeWorkingDirectory(path);//转移到指定FTP服务器目录
            ftpClient.deleteFile(fileName);
            ftpClient.logout();
            success = true;
        } catch (Exception e) {
            LogService.getLogger().error("Ftp文件删除失败！", e);
        }
        return success;
    }
    /**
     * 下载ftp文件夹
     *
     * @param ftpFileName ftp文件或文件夹名
     * @param localDir    本地保存路径
     */
    private boolean downDir(String ftpFileName, String localDir) {
        boolean success = false;
        try {
            File file = new File(ftpFileName);
            File temp = new File(localDir);
            if (!temp.exists()) {
                temp.mkdirs();
            }
            // 判断是否是目录
            if (isDir(ftpFileName)) {
                String dirPath = localDir + "/" + ftpFileName;
                File dirFile = new File(dirPath);
                if (!dirFile.exists()) {
                    dirFile.mkdirs();
                }
                String[] names = ftpClient.listNames();
                for (int i = 0; i < names.length; i++) {
                    if (isDir(names[i])) {
                        downDir(ftpFileName + '/' + names[i], dirPath
                                + File.separator + names[i]);
                        ftpClient.changeToParentDirectory();
                    } else {
                        File localfile = new File(dirPath + File.separator
                                + names[i]);
                        if (!localfile.exists()) {
                            fos = new FileOutputStream(localfile);
                            ftpClient.retrieveFile(names[i], fos);
                        }
                    }
                }
            } else {
                File localfile = new File(localDir + File.separator + file.getName());
                if (!localfile.exists()) {
                    fos = new FileOutputStream(localfile);
                    success = ftpClient.retrieveFile(ftpFileName, fos);
                }
                ftpClient.changeToParentDirectory();
            }
        } catch (SocketException e) {
            LogService.getLogger().error("Ftp服务器连接异常！", e);
        } catch (IOException e) {
            LogService.getLogger().error("Ftp文件下载失败！", e);
        }
        return success;
    }
    /**
     * 上传文件夹或文件至ftp
     *
     * @param filename   要上传的文件
     * @param uploadpath ftp文件路径
     * @return
     * @throws Exception
     */
    public boolean uploadDir(String filename, String uploadpath) {
        boolean success = false;
        File file = new File(filename);
        try {
            // 要上传的是否存在
            if (!file.exists()) {
                return success;
            }
            // 要上传的是否是文件夹
            if (!file.isDirectory()) {
                return uploadFile(filename, uploadpath);
            }
            File[] flles = file.listFiles();
            if (flles!=null){
            for (File files : flles) {
                if (files.exists()) {
                    if (files.isDirectory()) {
                        uploadDir(files.getAbsoluteFile().toString(),
                                uploadpath);
                    } else {
                        String local = files.getCanonicalPath().replaceAll("\\\\", "/");
                        String remote = uploadpath + local.substring(local.indexOf("/") + 1, local.lastIndexOf("/") + 1);
                        ftpClient.changeWorkingDirectory("/");
                        uploadFile(local, remote);
                        ftpClient.changeWorkingDirectory("/");
                    }
                }
            }
            }
        } catch (IOException e) {
            LogService.getLogger().error("Ftp文件上传失败！", e);
        }
        return true;
    }
    /**
     * 读取病人信息并封装成List<String>
     *
     * @param path 远程ftp文件路径
     * @return     List<String>病人信息集合
     */
    public List<String> readFileData(String path) {
        connect();
        List<String> list = new ArrayList<>();
        InputStream inputStream = null;
        try {
            ftpClient.changeWorkingDirectory(path);
            String[] filenames = ftpClient.listNames();
            if (filenames != null && filenames.length > 0) {
                for (String filenanme : filenames) {
                    inputStream = ftpClient.retrieveFileStream(path + "/" + filenanme);
                    if (inputStream==null){
                        LogService.getLogger().info("读取Ftp文件路径错误：" + (path + "/" + filenanme));
                        return list;
                    }//流为null，文件读取失败
                    String json = FileUtil.streamToBase64String(inputStream);
                    list.add(json);
                    inputStream.close();
                    ftpClient.completePendingCommand();
                }
            }
        } catch (SocketException e) {
            LogService.getLogger().error("Ftp服务器异常！", e);
        } catch (IOException e) {
            LogService.getLogger().error("Ftp文件读取失败！", e);
        }finally {
            closeConnect();
        }
        return list;
    }
    /**
     * 获取病人列表信息
     *
     * @param basePath  根目录下病人数据文件夹 eg：/home/test/patient/
     * @param agencyNo 机构编码
     * @return  病人信息map集合
     */
    public List<Map<String, String>> getPatientList(String basePath, String agencyNo) {
        connect();
        List<Map<String, String>> list = new ArrayList<>();
        Map<String, String> map = null;
        String base = basePath + agencyNo;
        try {
            ftpClient.changeWorkingDirectory(base);//病人ID集合
            int reply = ftpClient.getReplyCode();
            if (FTPReply.isPositiveCompletion(reply)) {
                String[] patientnames = ftpClient.listNames();
                if (patientnames != null && patientnames.length > 0) {
                    for (String patientname : patientnames) {
                        String jsonPath = base + "/" + patientname;//json文件夹
                        ftpClient.changeWorkingDirectory(jsonPath);
                        String[] eventNames = ftpClient.listNames();//获取json文件遍历
                        for (String event : eventNames) {
                            map = new HashMap<>();
                            map.put("agency_code", agencyNo);
                            map.put("patient_id", patientname);
                            map.put("event_no", event);
                            list.add(map);
                        }
//                    ftpClient.changeWorkingDirectory("..");
                    }
                }
            }else {
                LogService.getLogger().debug("agency编码：" + agencyNo + " 对应的数据不存在，获取病人列表信息失败");
            }
        } catch (SocketException e) {
            LogService.getLogger().error("Ftp-服务器连接失败！", e);
        } catch (IOException e) {
            LogService.getLogger().error("Ftp-获取病人数据失败！", e);
        }finally {
            closeConnect();
        }
        return list;
    }
    /**
     * 清空文件夹中文件
     * @param pathname ftp文件夹路径
     * @return
     */
    public boolean removeData(String pathname) {
        try {
            FTPFile[] files = ftpClient.listFiles(pathname);
            for (FTPFile f : files) {
                if (f.isDirectory()) {
                    removeData(pathname + "/" + f.getName());
                    ftpClient.removeDirectory(pathname);
                }
                if (f.isFile()) {
                    ftpClient.deleteFile(pathname + "/" + f.getName());
                }
            }
        } catch (IOException e) {
            LogService.getLogger().error("Ftp-清空文件夹失败！", e);
            return false;
        }
        return true;
    }
}

Hos-resource/src/main/java/com/yihu/ehr/crawler/origin/FileSystemOrigin.java

@@ -4,9 +4,6 @@ package com.yihu.ehr.crawler.origin;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.yihu.common.util.file.FtpFileUtil;
import com.yihu.common.util.log.BusinessLogger;
import com.yihu.common.util.log.DebugLogger;
import com.yihu.ehr.common.Services;
import com.yihu.ehr.crawler.format.AdapterScheme;
import com.yihu.ehr.crawler.model.adapter.AdapterDataSet;
@@ -14,6 +11,8 @@ import com.yihu.ehr.crawler.model.config.SysConfig;
import com.yihu.ehr.crawler.model.patient.Patient;
import com.yihu.ehr.crawler.model.transform.LogicValues;
import com.yihu.ehr.crawler.service.EsbHttp;
import com.yihu.ehr.framework.util.file.FtpFileUtil;
import com.yihu.ehr.framework.util.log.LogService;
import com.yihu.ehr.framework.util.operator.DateUtil;
import com.yihu.ehr.framework.util.operator.StringUtil;
import com.yihu.ehr.framework.util.springutil.SpringBeanUtil;
@@ -101,10 +100,10 @@ public class FileSystemOrigin implements IDataOrigin {
            return jsonObject.toString();
        } catch (SQLException e) {
//            e.printStackTrace();
            DebugLogger.fatal("", e);
            LogService.getLogger().error("", e);
        } catch (Exception e) {
//            e.printStackTrace();
            DebugLogger.fatal("", e);
            LogService.getLogger().error("", e);
        }
        return null;
    }
@@ -136,7 +135,7 @@ public class FileSystemOrigin implements IDataOrigin {
                    String patientId = patientMap.get("patient_id");
                    String eventNo = patientMap.get("event_no");
                    if (orgAgency == null) {
                        BusinessLogger.fatal("获取病人列表错误,无法获取机构代码.");
                        LogService.getLogger().error("获取病人列表错误,无法获取机构代码.");
                        continue;
                    }
                    patient.setPatientId(patientId);
@@ -149,7 +148,7 @@ public class FileSystemOrigin implements IDataOrigin {
            }
        } catch (Exception e) {
//            e.printStackTrace();
            DebugLogger.fatal("", e);
            LogService.getLogger().error("", e);
        }
        return patientList;
@@ -194,10 +193,10 @@ public class FileSystemOrigin implements IDataOrigin {
            return clear;
        } catch (SQLException e) {
            e.printStackTrace();
            DebugLogger.fatal("", e);
            LogService.getLogger().error("", e);
        } catch (Exception e) {
            e.printStackTrace();
            DebugLogger.fatal("", e);
            LogService.getLogger().error("", e);
        }
@@ -225,7 +224,7 @@ public class FileSystemOrigin implements IDataOrigin {
            int port = rootNode.path("port").asInt();
            ftpUtil = new FtpFileUtil(username, password, host, port);
        } catch (IOException e) {
            DebugLogger.fatal("获取Ftp服务器配置失败", e);
            LogService.getLogger().error("获取Ftp服务器配置失败", e);
            e.printStackTrace();
        }
        return ftpUtil;

Hos-resource/src/main/java/com/yihu/ehr/crawler/model/patient/PatientCDAUpload.java

@@ -1,8 +1,9 @@
package com.yihu.ehr.crawler.model.patient;
package com.yihu.ehr.crawler.service;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.yihu.ehr.crawler.model.config.SysConfig;
import com.yihu.ehr.crawler.model.patient.Patient;
import com.yihu.ehr.crawler.service.PatientCDAIndex;
import com.yihu.ehr.framework.util.compress.Zipper;
import com.yihu.ehr.framework.util.encrypt.MD5;
@@ -12,6 +13,7 @@ import com.yihu.ehr.framework.util.http.HOPClient;
import com.yihu.ehr.framework.util.http.Response;
import com.yihu.ehr.framework.util.httpclient.HttpHelper;
import com.yihu.ehr.framework.util.httpclient.HttpResponse;
import com.yihu.ehr.framework.util.log.LogService;
import org.apache.http.NameValuePair;
import org.apache.http.message.BasicNameValuePair;
@@ -27,7 +29,7 @@ import java.util.*;
 * @version 1.0
 * @created 2015.07.06 15:58
 */
public class PatientCDAUpload {
public class OldPatientCDAUpload {
    public static String uploadMethod;
@@ -41,20 +43,19 @@ public class PatientCDAUpload {
        ZipFile zipFile = zip(patient);
        try {
            if (zipFile == null || zipFile.file == null) {
                //BusinessLogger.fatal( "压缩病人档案失败,病人文档未生成,patient_id:" + patient.getPatientId() + ",event_no:" + patient.getEventNo(), LogAttribute.FAIL, patient.getJobTimeStamp());
                LogService.getLogger().info("压缩病人档案失败,病人文档未生成,patient_id:" + patient.getPatientId() + ",event_no:" + patient.getEventNo());
                return false;
            }
            boolean result = upload(patient, zipFile,token);
            if (!result) {
                //BusinessLogger.fatal("上传病人档案失败,patient_id:" + patient.getPatientId() + ",event_no:" + patient.getEventNo());
                LogService.getLogger().info("上传病人档案失败,patient_id:" + patient.getPatientId() + ",event_no:" + patient.getEventNo());
                FileUtil.deleteDirectory(new File(zipFile.directory));
                return false;
            }
            //DebugLogger.trace(zipFile.directory);
            result = FileUtil.deleteDirectory(new File(zipFile.directory));
            if (!result) {
                //BusinessLogger.fatal("删除临时文件失败,patient_id:" + patient.getPatientId() + ",event_no:" + patient.getEventNo());
                LogService.getLogger().info("删除临时文件失败,patient_id:" + patient.getPatientId() + ",event_no:" + patient.getEventNo());
            }
        } catch (Exception e) {
            FileUtil.deleteDirectory(new File(zipFile.directory));
@@ -76,7 +77,7 @@ public class PatientCDAUpload {
            String pwd = uuidPwd.toString();
            Key key = RSA.genPublicKey(SysConfig.getInstance().getPublicKey());
            if (key == null) {
                //BusinessLogger.fatal("压缩文件错误,无公钥信息.");
                LogService.getLogger().info("压缩文件错误,无公钥信息.");
                FileUtil.deleteDirectory(new File( patientCDAIndex.getDirectory()));
                return null;
            }
@@ -92,7 +93,7 @@ public class PatientCDAUpload {
            return zipFile;
        } catch (Exception e) {
            e.printStackTrace();
            //DebugLogger.fatal("从data目录生成zip数据时，压缩文件异常", e);
            LogService.getLogger().info("从data目录生成zip数据时，压缩文件异常", e);
        }
        return null;
@@ -111,12 +112,12 @@ public class PatientCDAUpload {
            header.put("Authorization", "Basic " + HttpHelper.clientKey);
            HttpResponse response = HttpHelper.postFile(uploadMethod, formParams, zipFile.file.getAbsolutePath(), header);
            if (response == null) {
                //BusinessLogger.fatal( "上传病人档案请求失败,patient_id:" + patient.getPatientId() + ",event_no:" + patient.getEventNo(), LogAttribute.FAIL, patient.getJobTimeStamp());
                LogService.getLogger().info( "上传病人档案请求失败,patient_id:" + patient.getPatientId() + ",event_no:" + patient.getEventNo());
                return false;
            }
            if (response.getStatusCode() != 200) {
                //BusinessLogger.fatal( "上传病人档案请求失败,错误代码:" + response.getStatusCode() + ",patient_id:" + patient.getPatientId() + ",event_no:" + patient.getEventNo(), LogAttribute.FAIL, patient.getJobTimeStamp());
                LogService.getLogger().info( "上传病人档案请求失败,错误代码:" + response.getStatusCode() + ",patient_id:" + patient.getPatientId() + ",event_no:" + patient.getEventNo());
                return false;
            }
@@ -125,16 +126,16 @@ public class PatientCDAUpload {
            JsonNode codeNode = rootNode.get("code");
            String result = codeNode.asText();
            if (!result.equals("0")) {
                //BusinessLogger.fatal("上传病人档案失败,错误代码:" + result + ",patient_id:" + patient.getPatientId() + ",event_no:" + patient.getEventNo(), LogAttribute.FAIL, patient.getJobTimeStamp());
                LogService.getLogger().info("上传病人档案失败,错误代码:" + result + ",patient_id:" + patient.getPatientId() + ",event_no:" + patient.getEventNo());
                return false;
            } else {
                //BusinessLogger.info( "上传病人档案成功,patient_id:" + patient.getPatientId() + ",event_no:" + patient.getEventNo(), LogAttribute.SUCCESS, patient.getJobTimeStamp());
                LogService.getLogger().info( "上传病人档案成功,patient_id:" + patient.getPatientId() + ",event_no:" + patient.getEventNo());
                return true;
            }
        } catch (Exception e) {
            e.printStackTrace();
            //BusinessLogger.fatal( "上传病人档案异常,patient_id:" + patient.getPatientId() + ",event_no:" + patient.getEventNo(), LogAttribute.FAIL, patient.getJobTimeStamp());
            LogService.getLogger().info( "上传病人档案异常,patient_id:" + patient.getPatientId() + ",event_no:" + patient.getEventNo());
            return false;
        }
    }