Java往HDFS上读写文件demo

项目中用到需要往hdfs文件系统上读写数据，使用到相关技术，此处做个整理，以便日后查阅。项目环境：Springboot+Maven+Windows7

1.我是在windows7下使用，会用到一个软件，hadoop-common-2.2.0-bin-master.zip，这个是hadoop在windows环境下依赖的一个进程。点击下载 hadoop-common-2.2.0-bin-master

2.添加Maven依赖，刚开始并没有这么多依赖，但是会报错，查找报错信息，是需要某些jar包，所以就把相关依赖都加进来了。

<dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client</artifactId> <version>2.7.2</version> </dependency> <dependency> <groupId>xerces</groupId> <artifactId>xercesImpl</artifactId> <version>2.11.0</version> </dependency> <dependency> <groupId>xml-apis</groupId> <artifactId>xml-apis</artifactId> <version>1.4.01</version> </dependency>

<dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>2.7.2</version> </dependency>

3还需要从hadoop上面获取以下2个配置文件。

4.具体代码如下：

public class HadoopTest { public static FileSystem fileSystem; static String PATH = "/tmp/wlltest/"; public static Configuration conf; static { System.setProperty("hadoop.home.dir", "E:\\hadoop-common-2.2.0-bin-master");//用到的包在Resources里面，不用会有错误，不影响使用 conf = new Configuration(); try { fileSystem = FileSystem.get(conf); } catch (IOException e) { e.printStackTrace(); } } /** * 创建文件夹 * @param dir */ @RequestMapping("/makedir") @ResponseBody public void makedir(String dir) { Path newdir = new Path(dir); try { fileSystem.mkdirs(newdir); } catch (IOException e) { e.printStackTrace(); } } /** * 上传新文件 * @throws IOException */ @RequestMapping("/upload") @ResponseBody public void upNewFile() throws IOException { long start = System.currentTimeMillis(); String newfile = "D://jdk1.7.0_888.zip"; String fileName = newfile.substring(newfile.lastIndexOf("/")); Path srcPath = new Path(newfile); //本地上传文件路径 File locFile = new File("D://jdk1.7.0_888.zip"); Path dstPath = new Path(PATH); //hdfs目标路径 //调用文件系统的文件复制函数,前面参数是指是否删除原文件，true为删除，默认为false fileSystem.copyFromLocalFile(false, srcPath, dstPath); //打印文件路径 System.out.println("Upload to " + conf.get("fs.default.name")); System.out.println("------------list files------------" + "\n"); FileStatus[] fileStatus = fileSystem.listStatus(dstPath); for (FileStatus file : fileStatus) { String path = file.getPath().toString(); path = path.substring(path.lastIndexOf("/")); if (path.equals(fileName)) { if (file.getLen() == locFile.length()) { //校验上传的文件和本地文件完整性 System.out.println("文件上传完成"); } else { System.out.println("文件上传不完整---"); } } } long end = System.currentTimeMillis(); System.out.println("---耗时:" + (end - start) / 1000 + "S"); // fileSystem.close(); } /** * 删除文件 * @throws IOException */ @RequestMapping("/delete") @ResponseBody public void deleteFile() throws IOException { String file = "/tmp/wlltest/123.txt"; fileSystem.deleteOnExit(new Path(file)); System.out.println("---delete "); } /** * 获取文件夹文件目录 * @throws IOException */ @RequestMapping("list") @ResponseBody public void getAllList() throws IOException { FileStatus fileList[] = fileSystem.listStatus(new Path(PATH)); if (fileList != null) { for (int i = 0; i < fileList.length; i++) { System.out.println("name:" + fileList[i].getPath().getName() + "/t/tsize:" + fileList[i].getLen()); } } }