java操作hdfs文件、文件夹

工具类HdfsUtils.java，及测试用例代码如下：

HdfsUtils.java

package com.xy6.demo.utils;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.zookeeper.common.IOUtils;
/**
* operate hdfs file or directory util class
*
* @author zhang
* @since 2016-09-26
*
*/
public class HdfsUtils {
public static String uri = "hdfs://192.168.1.118:9000";
/**
* make a new dir in the hdfs
*
* @param dir the dir may like '/tmp/testdir'
* @return boolean true-success, false-failed
* @exception IOException something wrong happends when operating files
*/
public static boolean mkdir(String dir) throws IOException {
if (StringUtils.isBlank(dir)) {
return false;
}
dir = uri + dir;
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(URI.create(dir), conf);
if (!fs.exists(new Path(dir))) {
fs.mkdirs(new Path(dir));
}
fs.close();
return true;
}
/**
* delete a dir in the hdfs.
* if dir not exists, it will throw FileNotFoundException
*
* @param dir the dir may like '/tmp/testdir'
* @return boolean true-success, false-failed
* @exception IOException something wrong happends when operating files
*
*/
public static boolean deleteDir(String dir) throws IOException {
if (StringUtils.isBlank(dir)) {
return false;
}
dir = uri + dir;
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(URI.create(dir), conf);
fs.delete(new Path(dir), true);
fs.close();
return true;
}
/**
* list files/directories/links names under a directory, not include embed
* objects
*
* @param dir a folder path may like '/tmp/testdir'
* @return List<String> list of file names
* @throws IOException file io exception
*/
public static List<String> listAll(String dir) throws IOException {
if (StringUtils.isBlank(dir)) {
return new ArrayList<String>();
}
dir = uri + dir;
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(URI.create(dir), conf);
FileStatus[] stats = fs.listStatus(new Path(dir));
List<String> names = new ArrayList<String>();
for (int i = 0; i < stats.length; ++i) {
if (stats[i].isFile()) {
// regular file
names.add(stats[i].getPath().toString());
} else if (stats[i].isDirectory()) {
// dir
names.add(stats[i].getPath().toString());
} else if (stats[i].isSymlink()) {
// is s symlink in linux
names.add(stats[i].getPath().toString());
}
}
fs.close();
return names;
}
/*
* upload the local file to the hds,
* notice that the path is full like /tmp/test.txt
* if local file not exists, it will throw a FileNotFoundException
*
* @param localFile local file path, may like F:/test.txt or /usr/local/test.txt
*
* @param hdfsFile hdfs file path, may like /tmp/dir
* @return boolean true-success, false-failed
*
* @throws IOException file io exception
*/
public static boolean uploadLocalFile2HDFS(String localFile, String hdfsFile) throws IOException {
if (StringUtils.isBlank(localFile) || StringUtils.isBlank(hdfsFile)) {
return false;
}
hdfsFile = uri + hdfsFile;
Configuration config = new Configuration();
FileSystem hdfs = FileSystem.get(URI.create(uri), config);
Path src = new Path(localFile);
Path dst = new Path(hdfsFile);
hdfs.copyFromLocalFile(src, dst);
hdfs.close();
return true;
}
/*
* create a new file in the hdfs.
*
* notice that the toCreateFilePath is the full path
*
* and write the content to the hdfs file.
*/
/**
* create a new file in the hdfs.
* if dir not exists, it will create one
*
* @param newFile new file path, a full path name, may like '/tmp/test.txt'
* @param content file content
* @return boolean true-success, false-failed
* @throws IOException file io exception
*/
public static boolean createNewHDFSFile(String newFile, String content) throws IOException {
if (StringUtils.isBlank(newFile) || null == content) {
return false;
}
newFile = uri + newFile;
Configuration config = new Configuration();
FileSystem hdfs = FileSystem.get(URI.create(newFile), config);
FSDataOutputStream os = hdfs.create(new Path(newFile));
os.write(content.getBytes("UTF-8"));
os.close();
hdfs.close();
return true;
}
/**
* delete the hdfs file
*
* @param hdfsFile a full path name, may like '/tmp/test.txt'
* @return boolean true-success, false-failed
* @throws IOException file io exception
*/
public static boolean deleteHDFSFile(String hdfsFile) throws IOException {
if (StringUtils.isBlank(hdfsFile)) {
return false;
}
hdfsFile = uri + hdfsFile;
Configuration config = new Configuration();
FileSystem hdfs = FileSystem.get(URI.create(hdfsFile), config);
Path path = new Path(hdfsFile);
boolean isDeleted = hdfs.delete(path, true);
hdfs.close();
return isDeleted;
}
/**
* read the hdfs file content
*
* @param hdfsFile a full path name, may like '/tmp/test.txt'
* @return byte[] file content
* @throws IOException file io exception
*/
public static byte[] readHDFSFile(String hdfsFile) throws Exception {
if (StringUtils.isBlank(hdfsFile)) {
return null;
}
hdfsFile = uri + hdfsFile;
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(URI.create(hdfsFile), conf);
// check if the file exists
Path path = new Path(hdfsFile);
if (fs.exists(path)) {
FSDataInputStream is = fs.open(path);
// get the file info to create the buffer
FileStatus stat = fs.getFileStatus(path);
// create the buffer
byte[] buffer = new byte[Integer.parseInt(String.valueOf(stat.getLen()))];
is.readFully(0, buffer);
is.close();
fs.close();
return buffer;
} else {
throw new Exception("the file is not found .");
}
}
/**
* append something to file dst
*
* @param hdfsFile a full path name, may like '/tmp/test.txt'
* @param content string
* @return boolean true-success, false-failed
* @throws Exception something wrong
*/
public static boolean append(String hdfsFile, String content) throws Exception {
if (StringUtils.isBlank(hdfsFile)) {
return false;
}
if(StringUtils.isEmpty(content)){
return true;
}
hdfsFile = uri + hdfsFile;
Configuration conf = new Configuration();
// solve the problem when appending at single datanode hadoop env
conf.set("dfs.client.block.write.replace-datanode-on-failure.policy", "NEVER");
conf.set("dfs.client.block.write.replace-datanode-on-failure.enable", "true");
FileSystem fs = FileSystem.get(URI.create(hdfsFile), conf);
// check if the file exists
Path path = new Path(hdfsFile);
if (fs.exists(path)) {
try {
InputStream in = new ByteArrayInputStream(content.getBytes());
OutputStream out = fs.append(new Path(hdfsFile));
IOUtils.copyBytes(in, out, 4096, true);
out.close();
in.close();
fs.close();
} catch (Exception ex) {
fs.close();
throw ex;
}
} else {
HdfsUtils.createNewHDFSFile(hdfsFile, content);
}
return true;
}
}

HdfsUtilsTest .java

package test.com.xy6.demo.utils;
import static org.junit.Assert.assertEquals;
import java.util.ArrayList;
import java.util.List;
import org.junit.Test;
import com.xy6.demo.utils.HdfsUtils;
public class HdfsUtilsTest {
public static String uri = "hdfs://192.168.1.118:9000";
public String dir = "/user/output1";
public String parentDir = "/user";
@Test
public void testMkdirNull1() {
try{
assertEquals(false, HdfsUtils.mkdir(null));
assertEquals(false, HdfsUtils.mkdir(" "));
assertEquals(false, HdfsUtils.mkdir(""));
} catch(Exception ex){
assertEquals(true, false);
}
}
@Test
public void testMkdirNormal1() {
try{
HdfsUtils.deleteDir(dir);
boolean result = HdfsUtils.mkdir(dir);
assertEquals(true, result);
List<String> listFile = HdfsUtils.listAll(parentDir);
boolean existFile = false;
for(String elem : listFile){
if(elem.equals(uri + dir)){
existFile = true;
break;
}
}
assertEquals(true, existFile);
} catch(Exception ex){
ex.printStackTrace();
assertEquals(true, false);
}
}
@Test
public void testDeleteDirNull1() {
try{
assertEquals(false, HdfsUtils.deleteDir(null));
assertEquals(false, HdfsUtils.deleteDir(""));
assertEquals(false, HdfsUtils.deleteDir(" "));
} catch(Exception ex){
assertEquals(true, false);
}
}
@Test
public void testDeleteDir() {
try{
assertEquals(true, HdfsUtils.mkdir(dir));
assertEquals(true, HdfsUtils.deleteDir(dir));
List<String> listFile = HdfsUtils.listAll(parentDir);
boolean existFile = false;
for(String elem : listFile){
if(uri + dir == elem){
existFile = true;
break;
}
}
assertEquals(false, existFile);
} catch(Exception ex){
ex.printStackTrace();
assertEquals(true, false);
}
}
@Test
public void testListAllNull1() {
try{
List<String> listFile = new ArrayList<String>();
assertEquals(listFile.toString(), HdfsUtils.listAll(null).toString());
assertEquals(listFile.toString(), HdfsUtils.listAll(" ").toString());
assertEquals(listFile.toString(), HdfsUtils.listAll("").toString());
} catch(Exception ex){
assertEquals(true, false);
}
}
@Test
public void testListAllEmptyFolder() {
try{
HdfsUtils.deleteDir(dir);
assertEquals(true, HdfsUtils.mkdir(dir));
List<String> listFile = HdfsUtils.listAll(dir);
assertEquals(0, listFile.size());
} catch(Exception ex){
ex.printStackTrace();
assertEquals(true, false);
}
}
@Test
public void testListAllNotExistFolder() {
try{
HdfsUtils.deleteDir(dir);
List<String> listFile = HdfsUtils.listAll(dir);
assertEquals(0, listFile.size());
} catch(Exception ex){
assertEquals(true, true);
}
}
@Test
public void testUploadLocalFile2HDFSNull1() {
try{
assertEquals(false, HdfsUtils.uploadLocalFile2HDFS(null, null));
assertEquals(false, HdfsUtils.uploadLocalFile2HDFS("", ""));
assertEquals(false, HdfsUtils.uploadLocalFile2HDFS(" ", " "));
} catch(Exception ex){
assertEquals(true, false);
}
}
@Test
public void testUploadLocalFile2HDFS() {
String localFile = "F:/Program Files/eclipse/eclipse.ini";
String remoteFile = dir + "/eclipse.ini";
try{
HdfsUtils.mkdir(dir);
HdfsUtils.deleteHDFSFile(remoteFile);
assertEquals(true, HdfsUtils.uploadLocalFile2HDFS(localFile, remoteFile));
} catch(Exception ex){
ex.printStackTrace();
assertEquals(true, false);
}
}
@Test
public void testUploadLocalFile2HDFSLocalNotExist() {
String localFile = "F:/Program Files/eclipse/eclipse2.ini";
String remoteFile = dir + "/eclipse.ini";
try{
assertEquals(true, HdfsUtils.mkdir(dir));
HdfsUtils.deleteHDFSFile(remoteFile);
HdfsUtils.uploadLocalFile2HDFS(localFile, remoteFile);
} catch(Exception ex){
assertEquals(true, true);
}
}
@Test
public void testCreateNewHDFSFileNull1() {
try{
assertEquals(false, HdfsUtils.createNewHDFSFile(null, null));
assertEquals(false, HdfsUtils.createNewHDFSFile(" ", " "));
assertEquals(false, HdfsUtils.createNewHDFSFile("", ""));
} catch(Exception ex){
assertEquals(true, false);
}
}
@Test
public void testCreateNewHDFSFileNormal1() {
try{
String newFile = dir + "/file1.txt";
String content = "hello file1";
HdfsUtils.deleteHDFSFile(newFile);
assertEquals(true, HdfsUtils.createNewHDFSFile(newFile, content));
String result = new String(HdfsUtils.readHDFSFile(newFile));
assertEquals(content, result);
} catch(Exception ex){
ex.printStackTrace();
assertEquals(true, false);
}
}
@Test
public void testCreateNewHDFSFileFoldNotexist1() {
try{
String newFile = dir + "/file1.txt";
String content = "hello file1";
assertEquals(true, HdfsUtils.deleteDir(dir));
assertEquals(true, HdfsUtils.createNewHDFSFile(newFile, content));
} catch(Exception ex){
ex.printStackTrace();
assertEquals(true, false);
}
}
@Test
public void testDeleteHDFSFileNull1() {
try{
assertEquals(false, HdfsUtils.deleteHDFSFile(null));
assertEquals(false, HdfsUtils.deleteHDFSFile(" "));
assertEquals(false, HdfsUtils.deleteHDFSFile(""));
} catch(Exception ex){
assertEquals(true, false);
}
}
@Test
public void testDeleteHDFSFile() {
this.testUploadLocalFile2HDFS();
try{
String remoteFile = dir + "/eclipse.ini";
assertEquals(true, HdfsUtils.deleteHDFSFile(remoteFile));
} catch(Exception ex){
assertEquals(true, false);
}
}
@Test
public void testDeleteHDFSFileNotexist1() {
try{
String remoteFile = dir + "/eclipse2.ini";
assertEquals(false, HdfsUtils.deleteHDFSFile(remoteFile));
} catch(Exception ex){
assertEquals(true, false);
}
}
@Test
public void testReadHDFSFileNull1() {
try{
assertEquals(null, HdfsUtils.readHDFSFile(null));
assertEquals(null, HdfsUtils.readHDFSFile(" "));
assertEquals(null, HdfsUtils.readHDFSFile(""));
} catch(Exception ex){
assertEquals(true, false);
}
}
@Test
public void testReadHDFSFile() {
this.testUploadLocalFile2HDFS();
try{
String remoteFile = dir + "/eclipse.ini";
String result = new String(HdfsUtils.readHDFSFile(remoteFile));
assertEquals(true, result.length() > 0);
} catch(Exception ex){
ex.printStackTrace();
assertEquals(true, false);
}
}
@Test
public void testAppendNull1() {
try{
assertEquals(false, HdfsUtils.append(null, null));
assertEquals(false, HdfsUtils.append(" ", " "));
assertEquals(false, HdfsUtils.append("", ""));
} catch(Exception ex){
assertEquals(true, false);
}
}
@Test
public void testAppend() {
try{
String newFile = dir + "/file1.txt";
String content1 = "hello append1\r\n";
String content2 = "hello append2\r\n";
HdfsUtils.deleteHDFSFile(newFile);
assertEquals(true, HdfsUtils.createNewHDFSFile(newFile, ""));
assertEquals(true, HdfsUtils.append(newFile, content1));
assertEquals(content1, new String(HdfsUtils.readHDFSFile(newFile)));
assertEquals(true, HdfsUtils.append(newFile, content2));
assertEquals(content1 + content2, new String(HdfsUtils.readHDFSFile(newFile)));
} catch(Exception ex){
ex.printStackTrace();
assertEquals(true, false);
}
}
}

java操作hdfs文件、文件夹相关推荐

hadoop java操作hdfs
hfds 是一种文件系统,用于存储hadoop将要处理的数据.适用于大规模分布式数据处理,是一个可扩展行的文件分布式系统: 优点 1.如果出现节点宕机,hdfs,可以持续监视,错误检查,容错处理,文档 ...
Java 操作 HDFS
HDFS 作为开源界比较成熟的分布式文件存储系统,适用于海量文件存储,本文介绍了如何使用 Java 操作 HDFS,采用 Maven 管理包. pom.xml <dependency>&l ...
Java操作HDFS文件
1.读取单个文件 [java] view plain copy Date date = DateUtil.getSpecifiedDayBefore(); String yesterday = Dat ...
Java操作word模板文件
关于导出word文档,之前想过用ireport但模板文件比较复杂不容易画.所以采取了Java操作word文件,替换word中的元素方法模板文件如下单位名称:$ACCTNAME$ NO: $SN$ ...
Java操作HDFS文件系统
对于操作HDFS文件系统,需要有一个入口,对于Hadoop来说,编程入口就是FileSystem. 例如我们要使用API创建一个文件夹: /*** @author vincent* @time 201 ...
Java读取Hdfs的文件数据出现乱码的解决方案和办法
使用JAVA api读取HDFS文件乱码踩坑想写一个读取HFDS上的部分文件数据做预览的接口,根据网上的博客实现后,发现有时读取信息会出现乱码,例如读取一个csv时,字符串之间被逗号分割英文字符串 ...
Java操作FastDFS实现文件上传和下载
前言:在GItHub上(地址:https://github.com/happyfish100),作者提供了fastdfs-client-java源码,源码里面提供了操作FastDFS的工具类,所以我们 ...
Chimm.Excel —— 使用Java 操作 excel 模板文件生成 excel 文档
内容已不在此处更新,请移步https://blog.csdn.net/chimmhuang/article/details/111251115 1. 项目介绍 Chimm.Excel 是什么? 该程序 ...
java操作LINUX上文件，设置权限问题
可以使用chmod u+x fileName;这样的话,可以使用Runtime类的方法执行一下命令行.进行权限赋值. ===================== File file = new Fil ...

java操作hdfs文件、文件夹

java操作hdfs文件、文件夹相关推荐

最新文章

热门文章