这篇文章转发别人的，挺不错的，过段时间把自己的分享出来。
Spring Boot实现SFTP文件上传下载 1.实现背景及现实意义 近期由于系统迁移到docker容器，采用Spring Boot 框架实现微服务治理，在此基础上晚间批量文件服务器也由ftp改成sftp，由于之前ftp的实现是采用公具类的形式，在此基础之上，未屏蔽开发细节和依赖Spring Boot自动装配的特性，进行组件的自动装配和改造，旨在实现简化开发，提高文件传输的安全性和数据交互的可靠性。
2.什么是sftp sftp是SSH File Transfer Protocol的缩写，安全文件传送协议。可以为传输文件提供一种安全的网络的加密方法。sftp 与 ftp 有着几乎一样的语法和功能。SFTP 为 SSH的其中一部分，是一种传输档案至 Blogger 伺服器的安全方式。其实在SSH软件包中，已经包含了一个叫作SFTP(Secure File Transfer Protocol)的安全文件信息传输子系统，SFTP本身没有单独的守护进程，它必须使用sshd守护进程（端口号默认是22）来完成相应的连接和答复操作，所以从某种意义上来说，SFTP并不像一个服务器程序，而更像是一个客户端程序。SFTP同样是使用加密传输认证信息和传输的数据，所以，使用SFTP是非常安全的。但是，由于这种传输方式使用了加密/解密技术，所以传输效率比普通的FTP要低得多。
3.sftp文件传输在java中的实现一 3.1Maven依赖 com.jcraft jsch 0.1.54 3.2sftp相关config封装 /** * @ClassName: SftpConfig * @Description: sftp配置类 * @Author: 尚先生 * @CreateDate: 2019/1/7 * @Version: 1.0 */ public class SftpConfig {
private String hostname; private Integer port; private String username; private String password; private Integer timeout; private Resource privateKey; private String remoteRootPath; private String fileSuffix;
public String getHostname() { return hostname; }
public void setHostname(String hostname) { this.hostname = hostname; }
public Integer getPort() { return port; }
public void setPort(Integer port) { this.port = port; }
public String getUsername() { return username; }
public void setUsername(String username) { this.username = username; }
public String getPassword() { return password; }
public void setPassword(String password) { this.password = password; }
public Integer getTimeout() { return timeout; }
public void setTimeout(Integer timeout) { this.timeout = timeout; }
public Resource getPrivateKey() { return privateKey; }
public void setPrivateKey(Resource privateKey) { this.privateKey = privateKey; }
public String getRemoteRootPath() { return remoteRootPath; }
public void setRemoteRootPath(String remoteRootPath) { this.remoteRootPath = remoteRootPath; }
public String getFileSuffix() { return fileSuffix; }
public void setFileSuffix(String fileSuffix) { this.fileSuffix = fileSuffix; }
public SftpConfig(String hostname, Integer port, String username, String password, Integer timeout, Resource privateKey, String remoteRootPath, String fileSuffix) { this.hostname = hostname; this.port = port; this.username = username; this.password = password; this.timeout = timeout; this.privateKey = privateKey; this.remoteRootPath = remoteRootPath; this.fileSuffix = fileSuffix; } public SftpConfig(String hostname, Integer port, String username, String password, Integer timeout, String remoteRootPath) { this.hostname = hostname; this.port = port; this.username = username; this.password = password; this.timeout = timeout; this.remoteRootPath = remoteRootPath; } public SftpConfig() { } } 3.3sftp工具类实现 /** * @ClassName: SFTP * @Description: sftp上传通用类 * @Author: 尚先生 * @CreateDate: 2019/1/3 * @Version: 1.0 */ public class SFTP {
private long count; /** * 已经连接次数 */ private long count1 = 0;
private long sleepTime;
private static final Logger logger = LoggerFactory.getLogger(SFTP.class);
/** * 连接sftp服务器 * * @return */ public ChannelSftp connect(SftpConfig sftpConfig) { ChannelSftp sftp = null; try { JSch jsch = new JSch(); jsch.getSession(sftpConfig.getUsername(), sftpConfig.getHostname(), sftpConfig.getPort()); Session sshSession = jsch.getSession(sftpConfig.getUsername(), sftpConfig.getHostname(), sftpConfig.getPort()); logger.info("Session created ... UserName=" + sftpConfig.getUsername() + ";host=" + sftpConfig.getHostname() + ";port=" + sftpConfig.getPort()); sshSession.setPassword(sftpConfig.getPassword()); Properties sshConfig = new Properties(); sshConfig.put("StrictHostKeyChecking", "no"); sshSession.setConfig(sshConfig); sshSession.connect(); logger.info("Session connected ..."); logger.info("Opening Channel ..."); Channel channel = sshSession.openChannel("sftp"); channel.connect(); sftp = (ChannelSftp) channel; logger.info("登录成功"); } catch (Exception e) { try { count1 += 1; if (count == count1) { throw new RuntimeException(e); } Thread.sleep(sleepTime); logger.info("重新连接...."); connect(sftpConfig); } catch (InterruptedException e1) { throw new RuntimeException(e1); } } return sftp; }
/** * 上传文件 * * @param directory 上传的目录 * @param uploadFile 要上传的文件 * @param sftpConfig */ public void upload(String directory, String uploadFile, SftpConfig sftpConfig) { ChannelSftp sftp = connect(sftpConfig); try { sftp.cd(directory); } catch (SftpException e) { try { sftp.mkdir(directory); sftp.cd(directory); } catch (SftpException e1) { throw new RuntimeException("ftp创建文件路径失败" + directory); } } File file = new File(uploadFile); InputStream inputStream=null; try { inputStream = new FileInputStream(file); sftp.put(inputStream, file.getName()); } catch (Exception e) { throw new RuntimeException("sftp异常" + e); } finally { disConnect(sftp); closeStream(inputStream,null); } }
/** * 下载文件 * * @param directory 下载目录 * @param downloadFile 下载的文件 * @param saveFile 存在本地的路径 * @param sftpConfig */ public void download(String directory, String downloadFile, String saveFile, SftpConfig sftpConfig) { OutputStream output = null; try { File localDirFile = new File(saveFile); // 判断本地目录是否存在，不存在需要新建各级目录 if (!localDirFile.exists()) { localDirFile.mkdirs(); } if (logger.isInfoEnabled()) { logger.info("开始获取远程文件:[{}]---->[{}]", new Object[]{directory, saveFile}); } ChannelSftp sftp = connect(sftpConfig); sftp.cd(directory); if (logger.isInfoEnabled()) { logger.info("打开远程文件:[{}]", new Object[]{directory}); } output = new FileOutputStream(new File(saveFile.concat(File.separator).concat(downloadFile))); sftp.get(downloadFile, output); if (logger.isInfoEnabled()) { logger.info("文件下载成功"); } disConnect(sftp); } catch (Exception e) { if (logger.isInfoEnabled()) { logger.info("文件下载出现异常，[{}]", e); } throw new RuntimeException("文件下载出现异常，[{}]", e); } finally { closeStream(null,output); } }
/** * 下载远程文件夹下的所有文件 * * @param remoteFilePath * @param localDirPath * @throws Exception */ public void getFileDir(String remoteFilePath, String localDirPath, SftpConfig sftpConfig) throws Exception { File localDirFile = new File(localDirPath); // 判断本地目录是否存在，不存在需要新建各级目录 if (!localDirFile.exists()) { localDirFile.mkdirs(); } if (logger.isInfoEnabled()) { logger.info("sftp文件服务器文件夹[{}],下载到本地目录[{}]", new Object[]{remoteFilePath, localDirFile}); } ChannelSftp channelSftp = connect(sftpConfig); Vector lsEntries = channelSftp.ls(remoteFilePath); if (logger.isInfoEnabled()) { logger.info("远程目录下的文件为[{}]", lsEntries); } for (LsEntry entry : lsEntries) { String fileName = entry.getFilename(); if (checkFileName(fileName)) { continue; } String remoteFileName = getRemoteFilePath(remoteFilePath, fileName); channelSftp.get(remoteFileName, localDirPath); } disConnect(channelSftp); }
/** * 关闭流 * @param outputStream */ private void closeStream(InputStream inputStream,OutputStream outputStream) { if (outputStream != null) { try { outputStream.close(); } catch (IOException e) { e.printStackTrace(); } } if(inputStream != null){ try { inputStream.close(); } catch (IOException e) { e.printStackTrace(); } } }
private boolean checkFileName(String fileName) { if (".".equals(fileName) || "..".equals(fileName)) { return true; } return false; }
private String getRemoteFilePath(String remoteFilePath, String fileName) { if (remoteFilePath.endsWith("/")) { return remoteFilePath.concat(fileName); } else { return remoteFilePath.concat("/").concat(fileName); } }
/** * 删除文件 * * @param directory 要删除文件所在目录 * @param deleteFile 要删除的文件 * @param sftp */ public void delete(String directory, String deleteFile, ChannelSftp sftp) { try { sftp.cd(directory); sftp.rm(deleteFile); } catch (Exception e) { throw new RuntimeException(e); } }
/** * 列出目录下的文件 * * @param directory 要列出的目录 * @param sftpConfig * @return * @throws SftpException */ public List listFiles(String directory, SftpConfig sftpConfig) throws SftpException { ChannelSftp sftp = connect(sftpConfig); List fileNameList = new ArrayList(); try { sftp.cd(directory); } catch (SftpException e) { return fileNameList; } Vector vector = sftp.ls(directory); for (int i = 0; i < vector.size(); i++) { if (vector.get(i) instanceof LsEntry) { LsEntry lsEntry = (LsEntry) vector.get(i); String fileName = lsEntry.getFilename(); if (".".equals(fileName) || "..".equals(fileName)) { continue; } fileNameList.add(fileName); } } disConnect(sftp); return fileNameList; }
/** * 断掉连接 */ public void disConnect(ChannelSftp sftp) { try { sftp.disconnect(); sftp.getSession().disconnect(); } catch (Exception e) { e.printStackTrace(); } }
public SFTP(long count, long sleepTime) { this.count = count; this.sleepTime = sleepTime; }
public SFTP() {
} } 3.4测试类实现 /** * @ClassName: TestSFTPUtils * @Description: SFTP工具类测试类 * @Author: 尚先生 * @CreateDate: 2019/4/25 11:09 * @Version: 1.0 */ public class TestSFTPUtils {
private static final Logger logger = LoggerFactory.getLogger(TestSFTPUtils.class); public static void main(String[] args) { SFTP ftp = new SFTP(3, 6000); SftpConfig sftpConfig = new SftpConfig("10.0.155.55", 22, "test", "test", 1000, "/opt/bdepfile/bdp/tset/20190425"); try { List list = ftp.listFiles("/opt/bdepfile/bdp/pucms/20190108", sftpConfig); logger.info("文件上传下载详情" ,new Object[]{list}); } catch (SftpException e) { logger.error("文件上传下载异常:[{}]" ,new Object[]{e}); } } } 4.sftp文件传输在java中的实现二 4.1Maven依赖 com.jcraft jsch 0.1.54 4.2扩展上述工具类的实现 4.2.1SFTP.java改造 /** * @ClassName: SFTP * @Description: sftp上传通用类 * @Author: 尚先生 * @CreateDate: 2019/1/3 * @Version: 1.0 */ @Component public class SFTP {
private ChannelSftp channelSftp;
@Value("${sftp.remotepath}") private String remotepath;
@Value("${sftp.localpath}") private String localpath;
@Value("${sftp.filenames}") private String filenames;
private static final String COMPLATEG_FILE_FLAG = "over_%s.dat";
private static final Logger logger = LoggerFactory.getLogger(SFTP.class); ... public void setChannelSftp(ChannelSftp channelSftp) { this.channelSftp = channelSftp; }
/** * * @param remotePath * @param remoteFileName * @param localPath * @param localFileName * @return */ public boolean downloadFile(String remotePath, String remoteFileName, String localPath, String localFileName){ logger.info("开始下载文件，远程路径:[{}],本地路径:[{}],文件名称:[{}]",new Object[]{remotePath,localPath,remoteFileName}); FileOutputStream fileOutputStream = null; File file = new File(localPath + localFileName); try { fileOutputStream = new FileOutputStream(file); channelSftp.get(remotePath + remoteFileName,fileOutputStream); return true; } catch (Exception e) { logger.error("sftp下载文件失败:[{}]",new Object[]{e}); return false; } }
/** * 单个ok文件下载 * @param trandate * @return */ public boolean downloadOKFile(String trandate){ trandate = trandate.replace("-", ""); String localDirPath = localpath.concat("/").concat(trandate); File localDirFile = new File(localDirPath); if (!localDirFile.exists()){ localDirFile.mkdirs(); }else { logger.info("文件[{}]已存在",new Object[]{localDirPath}); if (!localDirFile.isDirectory()){ logger.error("文件[{}]已存在，但不是目录，文件下载失败",new Object[]{localDirPath}); throw new RuntimeException(String.format("本地文件[{%s}]已存在，但不是目录，不能创建文件",localDirPath)); } } String filename = String.format(COMPLATEG_FILE_FLAG, trandate); String remoteFilePath = remotepath.concat("/").concat(trandate).concat("/"); String localFilePath = localDirPath.concat("/"); boolean flag = downloadFile(remoteFilePath, filename, localFilePath, filename); return flag; }
/** * 多个文件下载 * @param trandate * @return */ public boolean downloadCoreFilesToLocal(String trandate){ boolean flag = false; trandate = trandate.replace("-", ""); String localDirPath = localpath.concat("/").concat(trandate).concat("/"); String remoteDirPath = remotepath.concat("/").concat(trandate).concat("/"); for (String coreFileName : filenames.split(",")){ //文件名称截取 String coreFilaName = String.format(coreFileName.trim(), trandate); flag = downloadFile(remoteDirPath, coreFileName, localDirPath, coreFileName); } return flag; }
} 4.2.2自动装配Sftp连接器 /** * @ClassName: SftpClientConfigure * @Description: 自动装配Sftp连接器 * @Author: 尚先生 * @CreateDate: 2019/4/25 * @Version: 1.0 */ @Configuration @ConfigurationProperties(prefix = "sftp") public class SftpClientConfigure {
private String hostname; private Integer port; private String username; private String password; private Integer timeout; private String privateKey; private String remoteRootPath; private String fileSuffix;
// 通道类型 private static final String CHANNEL_TYPE = "sftp";
private static final Logger logger = LoggerFactory.getLogger(SftpClientConfigure.class);
public String getHostname() { return hostname; }
public void setHostname(String hostname) { this.hostname = hostname; }
public Integer getPort() { return port; }
public void setPort(Integer port) { this.port = port; }
public String getUsername() { return username; }
public void setUsername(String username) { this.username = username; }
public String getPassword() { return password; }
public void setPassword(String password) { this.password = password; }
public String getPrivateKey() { return privateKey; }
public void setPrivateKey(String privateKey) { this.privateKey = privateKey; }
public String getRemoteRootPath() { return remoteRootPath; }
public void setRemoteRootPath(String remoteRootPath) { this.remoteRootPath = remoteRootPath; }
public String getFileSuffix() { return fileSuffix; }
public void setFileSuffix(String fileSuffix) { this.fileSuffix = fileSuffix; }
public Integer getTimeout() { return timeout; }
public void setTimeout(Integer timeout) { this.timeout = timeout; }
@Bean("sshSession") @Lazy @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) public Session session() throws JSchException { if (logger.isInfoEnabled()) { logger.info("获取session，设置的超时时间为[{}]毫秒", timeout); } JSch jsch = new JSch(); Session session = jsch.getSession(username, hostname, port); // 设置秘钥 // jsch.addIdentity(privateKey); session.setPassword(password); Properties config = new Properties(); config.put("StrictHostKeyChecking", "no"); session.setConfig(config); //为Session对象设置properties session.setTimeout(timeout); //设置timeout时间 session.connect(); //通过Session建立链接 return session; }
@Bean("coreSftpChannel") @Lazy @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) public ChannelSftp channel(Session session) throws JSchException { if (logger.isInfoEnabled()) { logger.info("初始化sftp连接"); } Channel channel = session.openChannel(CHANNEL_TYPE); //打开SFTP通道 channel.connect(); //建立SFTP通道的连接 return (ChannelSftp) channel; }
} 4.2.3配置文件 # sftp配置 sftp.hostname=10.0.155.55
sftp.port=22
sftp.username=test
sftp.password=test
sftp.timeout=6000
sftp.privateKey=
sftp.remotepath=/opt/bdepfile/bdp/tset
sftp.localpath=D:/core
sftp.filenames=duebillInfo_%s.dat,repayInfo_%s.dat 4.2.4测试类实现 /** * @ClassName: TestAutoConfigurationSFTP * @Description: SftpClientConfigure测试类 * @Author: 尚先生 * @CreateDate: 2019/4/25 * @Version: 1.0 */ @RunWith(SpringRunner.class) @SpringBootTest public class TestAutoConfigurationSFTP {
private static final Logger logger = LoggerFactory.getLogger(TestAutoConfigurationSFTP.class);
@Autowired @Qualifier("coreSftpChannel") private ChannelSftp channelSftp;
@Autowired private SFTP sftp;
@Test public void testAotuDownload(){ String trandate = "2019-04-25"; boolean flag = false; sftp.setChannelSftp(channelSftp); flag = sftp.downloadOKFile(trandate); flag = sftp.downloadCoreFilesToLocal(trandate); logger.error("下载文件结果：[{}]",new Object[]{flag}); } } 4.2.5测试结果 打开 Git Bash Here cd D:\core ll 20190425 cd 20190425 ll -als ./ ../ duebillInfo_20190425.dat repayInfo_20190425.dat over_20190425.dat 4.3sftp扩展实现 由于当前Spring Boot环境实现，采用的是Spring Boot"自动装配"实现的，自动注入并调用实现从核心拉取文件的功能。
在项目中取固定文件时，只需动态追加或者替换下面的配置项sftp.filenames=duebillInfo_%s.dat,repayInfo_%s.dat
如果是新增sftp连接时可以手动创建
SftpConfig config = new SftpConfig();
SFTP sftp = new SFTP();
sftp.download(directory, downloadFile, saveFile, sftpConfig)
至此，多种方式实现sftp文件传输可以共存，而且在系统中可以实现"自动装配"。
完整代码和相关依赖请见GitHub https://github.com/dwyanewede/project-learn/tree/master/src/main/java/com/learn/demo/sftp
