|
|
@ -0,0 +1,385 @@
|
|
|
package com.yihu.hos.rest.services.crawler;
|
|
|
|
|
|
import com.mongodb.client.MongoDatabase;
|
|
|
import com.mongodb.client.gridfs.GridFSBucket;
|
|
|
import com.mongodb.client.gridfs.GridFSBuckets;
|
|
|
import com.mongodb.client.gridfs.model.GridFSUploadOptions;
|
|
|
import com.mongodb.gridfs.GridFSDBFile;
|
|
|
import com.yihu.hos.core.file.FileUtil;
|
|
|
import eu.medsea.mimeutil.MimeUtil;
|
|
|
import org.bson.Document;
|
|
|
import org.bson.types.ObjectId;
|
|
|
import org.springframework.data.mongodb.core.query.Query;
|
|
|
import org.springframework.data.mongodb.gridfs.GridFsCriteria;
|
|
|
import org.springframework.data.mongodb.gridfs.GridFsOperations;
|
|
|
import org.springframework.stereotype.Component;
|
|
|
|
|
|
import java.io.*;
|
|
|
import java.sql.Blob;
|
|
|
import java.util.*;
|
|
|
|
|
|
/**
|
|
|
* MongoDB GridFS 操作类
|
|
|
*
|
|
|
* @author HZY
|
|
|
* @vsrsion 1.0
|
|
|
* Created at 2016/7/21.
|
|
|
*/
|
|
|
@Component
|
|
|
public class GridFSUtil {
|
|
|
|
|
|
public static final int defaultChunkSize = 1024 * 1024 * 4;
|
|
|
|
|
|
|
|
|
public static GridFsOperations gridFsOperations;
|
|
|
|
|
|
|
|
|
|
|
|
public static ObjectId uploadFile( Blob blob, String fileType, Map<String, Object> params) {
|
|
|
// String fileName = UUID.randomUUID().toString() + "." + fileType;
|
|
|
//自定义字段
|
|
|
Document metaDocument = new Document();
|
|
|
if (params != null && params.size() > 0) {
|
|
|
for (Map.Entry<String, Object> entry : params.entrySet()) {
|
|
|
String key = entry.getKey();
|
|
|
metaDocument.append(key, entry.getValue());
|
|
|
}
|
|
|
}
|
|
|
// Create some custom options
|
|
|
GridFSUploadOptions gridFSUploadOptions = new GridFSUploadOptions()
|
|
|
.chunkSizeBytes(defaultChunkSize).metadata(metaDocument);
|
|
|
try {
|
|
|
com.mongodb.gridfs.GridFSFile gridFSFile = gridFsOperations.store(blob.getBinaryStream(), gridFSUploadOptions);
|
|
|
if (gridFSFile != null) {
|
|
|
return (ObjectId) gridFSFile.getId();
|
|
|
}
|
|
|
|
|
|
} catch (Exception e) {
|
|
|
e.printStackTrace();
|
|
|
}
|
|
|
return null;
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
* 上传文件至Mongodb by GridFS
|
|
|
*
|
|
|
* @param filePath 上传的文件路径
|
|
|
* @param saveFileName 保存到mongo的文件名
|
|
|
* @param params 自定义保存字段
|
|
|
*/
|
|
|
public static String uploadFile(String filePath,String saveFileName, Map<String, Object> params) {
|
|
|
//自定义字段
|
|
|
|
|
|
Document metaDocument = new Document();
|
|
|
if (params != null && params.size() > 0) {
|
|
|
for (Map.Entry<String, Object> entry : params.entrySet()) {
|
|
|
String key = entry.getKey();
|
|
|
metaDocument.append(key, entry.getValue());
|
|
|
}
|
|
|
}
|
|
|
// Create some custom options
|
|
|
FileInputStream fileInputStream = null;
|
|
|
try {
|
|
|
File readFile = new File(filePath);
|
|
|
fileInputStream = new FileInputStream(readFile);
|
|
|
gridFsOperations.delete(Query.query(GridFsCriteria.where("filename").is(saveFileName)));//删除原来的文件，保证唯一
|
|
|
com.mongodb.gridfs.GridFSFile gridFSFile = gridFsOperations.store(fileInputStream,saveFileName,"",metaDocument);
|
|
|
if (gridFSFile != null) {
|
|
|
return saveFileName;
|
|
|
}
|
|
|
} catch (IOException e) {
|
|
|
e.printStackTrace();
|
|
|
} finally {
|
|
|
if (fileInputStream != null) {
|
|
|
try {
|
|
|
fileInputStream.close();
|
|
|
} catch (IOException e) {
|
|
|
e.printStackTrace();
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
return null;
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
* 从 mongodb GridFS 下载文件
|
|
|
*
|
|
|
* @param dbName 数据库名
|
|
|
* @param savePath 文件保存路径
|
|
|
* @param objectId GridFS文件保存ObjectId
|
|
|
* @return
|
|
|
*/
|
|
|
public static String downFile(String dbName, String savePath, ObjectId objectId) {
|
|
|
FileOutputStream fileOutputStream = null;
|
|
|
try {
|
|
|
GridFSDBFile gridFSDBFile = gridFsOperations.findOne(Query.query(GridFsCriteria.where("_id").is(objectId)));
|
|
|
|
|
|
fileOutputStream = new FileOutputStream(savePath);
|
|
|
gridFSDBFile.writeTo(fileOutputStream);
|
|
|
|
|
|
return savePath;
|
|
|
} catch (IOException e) {
|
|
|
e.printStackTrace();
|
|
|
} finally {
|
|
|
if (fileOutputStream != null) {
|
|
|
try {
|
|
|
fileOutputStream.close();
|
|
|
} catch (IOException e) {
|
|
|
e.printStackTrace();
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
return null;
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
* 批量下载文件保存(根据 fs.files集合）
|
|
|
*
|
|
|
* @param dbName 数据库名
|
|
|
* @param savePath 文件保存的路径
|
|
|
* @param fsFiles fs.files
|
|
|
* @return 以“，”分割的文件名
|
|
|
*/
|
|
|
public static Map<String, StringBuffer> downFileList(String dbName, String savePath, List<GridFSDBFile> fsFiles) {
|
|
|
StringBuffer stringBuffer = new StringBuffer();
|
|
|
Map<String, String> fileNames = new HashMap<>();
|
|
|
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
|
|
if (fsFiles != null && fsFiles.size() > 0) {
|
|
|
for (GridFSDBFile fsFile : fsFiles) {
|
|
|
Object objectId = fsFile.getId();
|
|
|
String fileType = fsFile.getFilename().substring(fsFile.getFilename().lastIndexOf("."));
|
|
|
String newName = UUID.randomUUID().toString() + "." + fileType;
|
|
|
GridFSDBFile gridFSDBFile = gridFsOperations.findOne(Query.query(GridFsCriteria.where("_id").is(objectId)));
|
|
|
try {
|
|
|
gridFSDBFile.writeTo(out);
|
|
|
boolean success = FileUtil.writeFile(savePath + "/" + newName, out.toByteArray(), "utf-8");
|
|
|
if (!success) {
|
|
|
break;
|
|
|
}
|
|
|
|
|
|
String type = getMimeType(out.toByteArray());
|
|
|
fileNames.put(newName, type);
|
|
|
} catch (IOException e) {
|
|
|
e.printStackTrace();
|
|
|
return null;
|
|
|
} finally {
|
|
|
if (out != null) {
|
|
|
try {
|
|
|
out.close();
|
|
|
} catch (IOException e) {
|
|
|
e.printStackTrace();
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
return groupDataMap(fileNames);
|
|
|
} else {
|
|
|
return null;
|
|
|
}
|
|
|
}
|
|
|
|
|
|
|
|
|
/**
|
|
|
* 删除 mongodb-GridFS文件
|
|
|
*
|
|
|
* @param dbName
|
|
|
* @param objectId
|
|
|
*/
|
|
|
public static void deleteFile(String dbName, ObjectId objectId) {
|
|
|
gridFsOperations.delete(Query.query(GridFsCriteria.where("_id").is(objectId)));
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
* 查询fs.files 数据 in GridFS
|
|
|
*
|
|
|
* @param filters 查询条件
|
|
|
* @return files集合
|
|
|
*/
|
|
|
public static List<GridFSDBFile> findFiles( Map<String, Object> filters) {
|
|
|
Query query = new Query();
|
|
|
if (filters != null) {
|
|
|
filters.forEach((key, value) -> query.addCriteria(GridFsCriteria.where(key).is(value)));
|
|
|
}
|
|
|
|
|
|
return gridFsOperations.find(query);
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
* 根据ObjectID集合查询GridFS 文件列表
|
|
|
*
|
|
|
* @param dbName 数据库名
|
|
|
* @param ids objectId集合
|
|
|
* @return
|
|
|
*/
|
|
|
public static List<GridFSDBFile> findFsFiles(String dbName, List<ObjectId> ids) {
|
|
|
List<GridFSDBFile> list = new ArrayList<>();
|
|
|
ids.forEach(objectId -> {
|
|
|
GridFSDBFile gridFSDBFile = gridFsOperations.findOne(Query.query(GridFsCriteria.where("_id").is(objectId)));
|
|
|
list.add(gridFSDBFile);
|
|
|
});
|
|
|
|
|
|
return list;
|
|
|
}
|
|
|
|
|
|
|
|
|
/*************************************** MineType 工具类 *********************************/
|
|
|
/**
|
|
|
* 获取文件Mine-Type
|
|
|
*
|
|
|
* @param file
|
|
|
* @return
|
|
|
*/
|
|
|
public static String getMimeType(File file) {
|
|
|
MimeUtil.registerMimeDetector("eu.medsea.mimeutil.detector.MagicMimeMimeDetector");
|
|
|
Collection<?> collection = MimeUtil.getMimeTypes(file);
|
|
|
return collection.toString();
|
|
|
}
|
|
|
|
|
|
public static String getMimeType(byte[] bytes) {
|
|
|
MimeUtil.registerMimeDetector("eu.medsea.mimeutil.detector.MagicMimeMimeDetector");
|
|
|
Collection<?> collection = MimeUtil.getMimeTypes(bytes);
|
|
|
return collection.toString();
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
* 非结构化档案--文件类型map生成
|
|
|
*
|
|
|
* @param map
|
|
|
* @return
|
|
|
*/
|
|
|
public static Map<String, StringBuffer> groupDataMap(Map<String, String> map) {
|
|
|
Map<String, StringBuffer> result = new HashMap<String, StringBuffer>();
|
|
|
Iterator<String> rs = map.keySet().iterator();
|
|
|
while (rs.hasNext()) {
|
|
|
String key = rs.next();
|
|
|
String value = map.get(key);
|
|
|
if (result.containsKey(value)) {
|
|
|
result.get(value).append(",").append(key);
|
|
|
} else {
|
|
|
result.put(value, new StringBuffer(key));
|
|
|
}
|
|
|
}
|
|
|
return result;
|
|
|
}
|
|
|
|
|
|
|
|
|
/**
|
|
|
* 上传文件至Mongodb by GridFS
|
|
|
* @param saveFileName 保存到mongo的文件名
|
|
|
* @param inputStream 文件流
|
|
|
* @param params metaData数据
|
|
|
* @return
|
|
|
*/
|
|
|
public static String uploadFile( InputStream inputStream,String saveFileName,Map<String ,Object> params) {
|
|
|
//metaData参数
|
|
|
Document metaDocument = new Document();
|
|
|
if (params != null && params.size() > 0) {
|
|
|
for (Map.Entry<String, Object> entry : params.entrySet()) {
|
|
|
String key = entry.getKey();
|
|
|
metaDocument.append(key, entry.getValue());
|
|
|
}
|
|
|
}
|
|
|
|
|
|
try {
|
|
|
gridFsOperations.delete(Query.query(GridFsCriteria.where("filename").is(saveFileName)));//删除原来的文件，保证唯一
|
|
|
com.mongodb.gridfs.GridFSFile gridFSFile = gridFsOperations.store(inputStream, saveFileName, "", metaDocument);
|
|
|
if (gridFSFile != null) {
|
|
|
return saveFileName;
|
|
|
}
|
|
|
}catch (Exception e){
|
|
|
e.printStackTrace();
|
|
|
}finally {
|
|
|
if (inputStream != null) {
|
|
|
try {
|
|
|
inputStream.close();
|
|
|
} catch (IOException e) {
|
|
|
e.printStackTrace();
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
return null;
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
* 据文件名返回文件，只返回第一个
|
|
|
*
|
|
|
* @param fileName
|
|
|
* @return
|
|
|
*/
|
|
|
public static boolean readFile(MongoDatabase db, OutputStream os, String fileName) {
|
|
|
try {
|
|
|
GridFSBucket gridFS = GridFSBuckets.create(db);
|
|
|
gridFS.downloadToStreamByName(fileName, os);
|
|
|
os.close();
|
|
|
return true;
|
|
|
} catch (IOException e) {
|
|
|
e.printStackTrace();
|
|
|
return false;
|
|
|
}
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
* 读取文件内容
|
|
|
*
|
|
|
* @param fileName 文件名
|
|
|
* @return
|
|
|
*/
|
|
|
public static String readFileContent( String fileName) {
|
|
|
try {
|
|
|
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
|
|
|
GridFSDBFile gridFSDBFile = gridFsOperations.findOne(Query.query(GridFsCriteria.where("filename").is(fileName)));
|
|
|
gridFSDBFile.writeTo(byteArrayOutputStream);
|
|
|
return byteArrayOutputStream.toString();
|
|
|
} catch (Exception e) {
|
|
|
e.printStackTrace();
|
|
|
return null;
|
|
|
}
|
|
|
}
|
|
|
|
|
|
|
|
|
/**
|
|
|
* 从 mongodb GridFS 下载文件
|
|
|
*
|
|
|
* @param savePath 文件保存路径
|
|
|
* @param fileName GridFS文件名
|
|
|
* @return
|
|
|
*/
|
|
|
public static String downFile( String savePath, String fileName) {
|
|
|
FileOutputStream fileOutputStream = null;
|
|
|
try {
|
|
|
File file = new File(savePath);
|
|
|
fileOutputStream = new FileOutputStream(file);
|
|
|
List<GridFSDBFile> gridFSDBFiles = gridFsOperations.find(Query.query(GridFsCriteria.where("filename").is(fileName)));
|
|
|
if (gridFSDBFiles==null || gridFSDBFiles.isEmpty()){
|
|
|
return null;
|
|
|
}else {
|
|
|
GridFSDBFile gridFSDBFile = gridFSDBFiles.get(0);
|
|
|
gridFSDBFile.writeTo(fileOutputStream);
|
|
|
return savePath;
|
|
|
}
|
|
|
} catch (IOException e) {
|
|
|
e.printStackTrace();
|
|
|
} finally {
|
|
|
if (fileOutputStream != null) {
|
|
|
try {
|
|
|
fileOutputStream.close();
|
|
|
} catch (IOException e) {
|
|
|
e.printStackTrace();
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
return null;
|
|
|
}
|
|
|
|
|
|
public static boolean deleteFile(String fileName){
|
|
|
try {
|
|
|
gridFsOperations.delete(Query.query(GridFsCriteria.where("filename").is(fileName)));//删除原来的文件，保证唯一
|
|
|
return true;
|
|
|
}catch (Exception e){
|
|
|
e.printStackTrace();
|
|
|
return false;
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
airhead