ftp+线程池批量上传文件

FTP服务器(File Transfer Protocol Server)是在互联网上提供文件和访问服务的计算机,它们依照提供服务。FTP是File Transfer Protocol(文件传输协议)。顾名思义,就是专门用来传输文件的协议。简单地说,支持FTP协议的服务器就是FTP服务器

代码：


public class FileBatchUploader implements Closeable {private final String ftpServer;private final String userName;private final String password;private final String targetRemoteDir;private final FTPClient ftp = new FTPClient();private final CompletionService<File> completionService;private final ExecutorService es;private final ExecutorService dispatcher;public FileBatchUploader(String ftpServer, String userName, String password,String targetRemoteDir) {this.ftpServer = ftpServer;this.userName = userName;this.password = password;this.targetRemoteDir = targetRemoteDir;// 使用单工作者线程的线程池this.es = Executors.newSingleThreadExecutor();this.dispatcher = Executors.newSingleThreadExecutor();this.completionService = new ExecutorCompletionService<File>(es);}public void uploadFiles(final Set<File> files) {dispatcher.submit(new Runnable() {@Overridepublic void run() {try {doUploadFiles(files);} catch (InterruptedException ignored) {}}});}private void doUploadFiles(Set<File> files) throws InterruptedException {// 批量提交文件上传任务for (final File file : files) {completionService.submit(new UploadTask(file));}Future<File> future;File md5File;File uploadedFile;Set<File> md5Files = new HashSet<File>();for (File file : files) {try {future = completionService.take();uploadedFile = future.get();// 将上传成功的文件移动到备份目录，并为其生成相应的MD5文件md5File = generateMD5(moveToSuccessDir(uploadedFile));md5Files.add(md5File);} catch (ExecutionException | IOException | NoSuchAlgorithmException e) {e.printStackTrace();moveToDeadDir(file);}}for (File file : md5Files) {// 上传相应的MD5文件completionService.submit(new UploadTask(file));}// 检查md5文件的上传结果int successUploaded = md5Files.size();for (int i = 0; i < successUploaded; i++) {future = completionService.take();try {uploadedFile = future.get();md5Files.remove(uploadedFile);} catch (ExecutionException e) {e.printStackTrace();}}// 将剩余（即未上传成功）的md5文件移动到相应备份目录for (File file : md5Files) {moveToDeadDir(file);}}private File generateMD5(File file) throws IOException, NoSuchAlgorithmException {String md5 = Tools.md5sum(file);File md5File = new File(file.getAbsolutePath() + ".md5");Files.write(Paths.get(md5File.getAbsolutePath()), md5.getBytes("UTF-8"));return md5File;}private static File moveToSuccessDir(File file) {File targetFile = null;try {targetFile = moveFile(file, Paths.get(file.getParent(), "..", "backup", "success"));} catch (IOException e) {e.printStackTrace();}return targetFile;}private static File moveToDeadDir(File file) {File targetFile = null;try {targetFile = moveFile(file, Paths.get(file.getParent(), "..", "backup", "dead"));} catch (IOException e) {e.printStackTrace();}return targetFile;}private static File moveFile(File srcFile, Path destPath) throws IOException {Path sourcePath = Paths.get(srcFile.getAbsolutePath());if (!Files.exists(destPath)) {Files.createDirectories(destPath);}Path destFile = destPath.resolve(srcFile.getName());Files.move(sourcePath, destFile,StandardCopyOption.REPLACE_EXISTING);return destFile.toFile();}class UploadTask implements Callable<File> {private final File file;public UploadTask(File file) {this.file = file;}@Overridepublic File call() throws Exception {Debug.info("uploading %s", file.getCanonicalPath());// 上传指定的文件upload(file);return file;}}// 初始化FTP客户端public void init() throws Exception {FTPClientConfig config = new FTPClientConfig();ftp.configure(config);int reply;ftp.connect(ftpServer);Debug.info("FTP Reply:%s", ftp.getReplyString());reply = ftp.getReplyCode();if (!FTPReply.isPositiveCompletion(reply)) {ftp.disconnect();throw new Exception("FTP server refused connection.");}boolean isOK = ftp.login(userName, password);if (isOK) {Debug.info("FTP Reply:%s", ftp.getReplyString());} else {throw new Exception("Failed to login." + ftp.getReplyString());}reply = ftp.cwd(targetRemoteDir);if (!FTPReply.isPositiveCompletion(reply)) {ftp.disconnect();throw new Exception("Failed to change working directory.reply:"+ reply);} else {Debug.info("FTP Reply:%s", ftp.getReplyString());}ftp.setFileType(FTP.ASCII_FILE_TYPE);}// 将指定的文件上传至FTP服务器protected void upload(File file) throws Exception {boolean isOK;try (InputStream dataIn = new BufferedInputStream(new FileInputStream(file))) {isOK = ftp.storeFile(file.getName(), dataIn);}if (!isOK) {throw new IOException("Failed to upload " + file + ",reply:" + ","+ ftp.getReplyString());}}@Overridepublic void close() throws IOException {dispatcher.shutdown();try {es.awaitTermination(60, TimeUnit.SECONDS);} catch (InterruptedException ignored) {}es.shutdown();try {es.awaitTermination(60, TimeUnit.SECONDS);} catch (InterruptedException ignored) {}Tools.silentClose(new Closeable() {@Overridepublic void close() throws IOException {if (ftp.isConnected()) {ftp.disconnect();}}});}
}

测试上面代码：

public static void main(String[] args) throws Exception {final FileBatchUploader uploader = new FileBatchUploader("localhost", "datacenter", "abc123","/home/datacenter/tmp/") {@Overridepublic void init() throws Exception {Debug.info("init...");super.init();}@Overrideprotected void upload(File file) throws Exception {super.upload(file);}@Overridepublic void close() throws IOException {Debug.info("close...");super.close();}};uploader.init();Set<File> files = new HashSet<File>();files.add(new File("/home/viscent/tmp/incomingX/message1.dat"));files.add(new File("/home/viscent/tmp/incomingX/message2.dat"));files.add(new File("/home/viscent/tmp/incomingX/message3.dat"));files.add(new File("/home/viscent/tmp/incomingX/message4.dat"));files.add(new File("/home/viscent/tmp/incomingX/message5.dat"));uploader.uploadFiles(files);Tools.delayedAction("", new Runnable() {@Overridepublic void run() {Tools.silentClose(uploader);}}, 120);}}

ftp+线程池批量上传文件相关推荐

shell中通过ftp批量上传文件
为了在shell中上传文件,需要避免在控制台中通过交互的方式输入ftp的登录密码,这时要安装一个强大的ftp命令行工具:lftp,通过lftp登录ftp服务器的格式如下: lftp -u userna ...
工作笔记1——利用bat脚本实现批量上传文件到ftp服务器
工作笔记1--利用bat脚本实现批量上传文件到ftp服务器问题概述利用ftp命令实现上传文件注意限时功能将共享目录映射到电脑中的某个盘符实现断点续传问题概述公司中有大概225k个文件需 ...
批量上传文件及进度显示
不带插件 ,自己写js,实现批量上传文件及进度显示今天接受项目中要完成文件批量上传文件而且还要显示上传进度,一开始觉得这个应该不是很麻烦,当我在做的时候遇到了很多问题,很头疼啊. 不过看了别人写的代 ...
python paramiko并发_python paramiko 多线程批量执行指令及批量上传文件和目录
源代码: 环境需求: 1.python3 2.paramiko pip install --upgrade pip apt-get install libssl-dev pip3 install pa ...
微信小程序云开发-批量上传文件到云储存空间
微信小程序开发自己最近在玩微信小程序,准备记录一些自己遇到的,网络上没有对应教程(也许是自己没找到),或者是教程比较少的问题,然后给出自己的解决方案目录微信小程序开发问题一.云储存是什么? ...
vue+elementui 同时有上传文件和批量上传文件功能，上传文件或批量上传文件后必须刷新才能再次上传文件
报错描述: 使用element-ui的上传文件组件写一个批量上传和上传文件,但是发现每次上传文件后或者批量上传文件后,不能再次上传文件或者批量上传文件.只有进入页面第一次点击上传文件或者批量上传文件才 ...
【Fastapi】批量上传文件（文档、图片、视频等）
[Fastapi]批量上传文件项目演示功能说明编程思路重要知识点分析源代码项目演示 [Fastapi]批量上传文件(文档.图片.视频等) https://www.bilibili.com/ ...
使用hbuilder前端工具直接连接服务器FTP/SFTP连接传输上传文件
hbuilder前端工具直接连接FTP/SFTP连接传输上传文件#优点# 1>不需要冗余的下载ftp工具 *第一步* 1>hbuilder插件市场安装SFTP/FTP Sync 2> ...
java 批量上传pdf文件_使用fileinput插件批量上传文件
使用fileinput插件批量上传文件前言最近在做项目的时候涉及文件批量上传,使用bootstrap结合fileinput插件批量上传文件!以下是我在使用fileinput上传文件的过程.项目是基 ...

ftp+线程池批量上传文件

ftp+线程池批量上传文件相关推荐

最新文章

热门文章