文件分割与合并是一个常见需求，比如：上传大文件时，可以先分割成小块，传到服务器后，再进行合并。很多高大上的分布式文件系统（比如：google的GFS、taobao的TFS）里，也是按block为单位，对文件进行分割或合并。
看下基本思路：
如果有一个大文件，指定分割大小后（比如：按1M切割）
step 1：
先根据原始文件大小、分割大小，算出最终分割的小文件数N
step 2:
在磁盘上创建这N个小文件
step 3:
开多个线程（线程数=分割文件数），每个线程里，利用RandomAccessFile的seek功能，将读取指针定位到原文件里每一段的段首位置，然后向后读取指定大小（即：分割块大小），最终写入对应的分割文件，因为多线程并行处理，各写各的小文件，速度相对还是比较快的。
合并时，把上面的思路逆向处理即可。
核心代码：
分割处理：
1 /**
2 * 拆分文件
3 * @param fileName 待拆分的完整文件名
4 * @param byteSize 按多少字节大小拆分
5 * @return 拆分后的文件名列表
6 * @throws IOException
7 */
8 public List<String> splitBySize(String fileName, int byteSize)
9 throws IOException {
10 List<String> parts = new ArrayList<String>();
11 File file = new File(fileName);
12 int count = (int) Math.ceil(file.length() / (double) byteSize);
13 int countLen = (count + "").length();
14 ThreadPoolExecutor threadPool = new ThreadPoolExecutor(count,
15 count * 3, 1, TimeUnit.SECONDS,
16 new ArrayBlockingQueue<Runnable>(count * 2));
17
18 for (int i = 0; i < count; i++) {
19 String partFileName = file.getName() + "."
20 + leftPad((i + 1) + "", countLen, '0') + ".part";
21 threadPool.execute(new SplitRunnable(byteSize, i * byteSize,
22 partFileName, file));
23 parts.add(partFileName);
24 }
25 return parts;
26 }
View Code
1 private class SplitRunnable implements Runnable {
2 int byteSize;
3 String partFileName;
4 File originFile;
5 int startPos;
6
7 public SplitRunnable(int byteSize, int startPos, String partFileName,
8 File originFile) {
9 this.startPos = startPos;
10 this.byteSize = byteSize;
11 this.partFileName = partFileName;
12 this.originFile = originFile;
13 }
14
15 public void run() {
16 RandomAccessFile rFile;
17 OutputStream os;
18 try {
19 rFile = new RandomAccessFile(originFile, "r");
20 byte[] b = new byte[byteSize];
21 rFile.seek(startPos);// 移动指针到每“段”开头
22 int s = rFile.read(b);
23 os = new FileOutputStream(partFileName);
24 os.write(b, 0, s);
25 os.flush();
26 os.close();
27 } catch (IOException e) {
28 e.printStackTrace();
29 }
30 }
31 }
View Code
合并处理：
1 /**
2 * 合并文件
3 *
4 * @param dirPath 拆分文件所在目录名
5 * @param partFileSuffix 拆分文件后缀名
6 * @param partFileSize 拆分文件的字节数大小
7 * @param mergeFileName 合并后的文件名
8 * @throws IOException
9 */
10 public void mergePartFiles(String dirPath, String partFileSuffix,
11 int partFileSize, String mergeFileName) throws IOException {
12 ArrayList<File> partFiles = FileUtil.getDirFiles(dirPath,
13 partFileSuffix);
14 Collections.sort(partFiles, new FileComparator());
15
16 RandomAccessFile randomAccessFile = new RandomAccessFile(mergeFileName,
17 "rw");
18 randomAccessFile.setLength(partFileSize * (partFiles.size() - 1)
19 + partFiles.get(partFiles.size() - 1).length());
20 randomAccessFile.close();
21
22 ThreadPoolExecutor threadPool = new ThreadPoolExecutor(
23 partFiles.size(), partFiles.size() * 3, 1, TimeUnit.SECONDS,
24 new ArrayBlockingQueue<Runnable>(partFiles.size() * 2));
25
26 for (int i = 0; i < partFiles.size(); i++) {
27 threadPool.execute(new MergeRunnable(i * partFileSize,
28 mergeFileName, partFiles.get(i)));
29 }
30
31 }
View Code
1 private class MergeRunnable implements Runnable {
2 long startPos;
3 String mergeFileName;
4 File partFile;
5
6 public MergeRunnable(long startPos, String mergeFileName, File partFile) {
7 this.startPos = startPos;
8 this.mergeFileName = mergeFileName;
9 this.partFile = partFile;
10 }
11
12 public void run() {
13 RandomAccessFile rFile;
14 try {
15 rFile = new RandomAccessFile(mergeFileName, "rw");
16 rFile.seek(startPos);
17 FileInputStream fs = new FileInputStream(partFile);
18 byte[] b = new byte[fs.available()];
19 fs.read(b);
20 fs.close();
21 rFile.write(b);
22 rFile.close();
23 } catch (IOException e) {
24 e.printStackTrace();
25 }
26 }
27 }
View Code
为了方便文件操作，把关于文件读写的功能，全封装到FileUtil类：
1 package com.cnblogs.yjmyzz;
2
3 import java.io.*;
4 import java.util.*;
5 import java.util.concurrent.*;
6
7 /**
8 * 文件处理辅助类
9 *
10 * @author yjmyzz@126.com
11 * @version 0.2
12 * @since 2014-11-17
13 *
14 */
15 public class FileUtil {
16
17 /**
18 * 当前目录路径
19 */
20 public static String currentWorkDir = System.getProperty("user.dir") + "\\";
21
22 /**
23 * 左填充
24 *
25 * @param str
26 * @param length
27 * @param ch
28 * @return
29 */
30 public static String leftPad(String str, int length, char ch) {
31 if (str.length() >= length) {
32 return str;
33 }
34 char[] chs = new char[length];
35 Arrays.fill(chs, ch);
36 char[] src = str.toCharArray();
37 System.arraycopy(src, 0, chs, length - src.length, src.length);
38 return new String(chs);
39
40 }
41
42 /**
43 * 删除文件
44 *
45 * @param fileName
46 * 待删除的完整文件名
47 * @return
48 */
49 public static boolean delete(String fileName) {
50 boolean result = false;
51 File f = new File(fileName);
52 if (f.exists()) {
53 result = f.delete();
54
55 } else {
56 result = true;
57 }
58 return result;
59 }
60
61 /***
62 * 递归获取指定目录下的所有的文件（不包括文件夹）
63 *
64 * @param obj
65 * @return
66 */
67 public static ArrayList<File> getAllFiles(String dirPath) {
68 File dir = new File(dirPath);
69
70 ArrayList<File> files = new ArrayList<File>();
71
72 if (dir.isDirectory()) {
73 File[] fileArr = dir.listFiles();
74 for (int i = 0; i < fileArr.length; i++) {
75 File f = fileArr[i];
76 if (f.isFile()) {
77 files.add(f);
78 } else {
79 files.addAll(getAllFiles(f.getPath()));
80 }
81 }
82 }
83 return files;
84 }
85
86 /**
87 * 获取指定目录下的所有文件(不包括子文件夹)
88 *
89 * @param dirPath
90 * @return
91 */
92 public static ArrayList<File> getDirFiles(String dirPath) {
93 File path = new File(dirPath);
94 File[] fileArr = path.listFiles();
95 ArrayList<File> files = new ArrayList<File>();
96
97 for (File f : fileArr) {
98 if (f.isFile()) {
99 files.add(f);
100 }
101 }
102 return files;
103 }
104
105 /**
106 * 获取指定目录下特定文件后缀名的文件列表(不包括子文件夹)
107 *
108 * @param dirPath
109 * 目录路径
110 * @param suffix
111 * 文件后缀
112 * @return
113 */
114 public static ArrayList<File> getDirFiles(String dirPath,
115 final String suffix) {
116 File path = new File(dirPath);
117 File[] fileArr = path.listFiles(new FilenameFilter() {
118 public boolean accept(File dir, String name) {
119 String lowerName = name.toLowerCase();
120 String lowerSuffix = suffix.toLowerCase();
121 if (lowerName.endsWith(lowerSuffix)) {
122 return true;
123 }
124 return false;
125 }
126
127 });
128 ArrayList<File> files = new ArrayList<File>();
129
130 for (File f : fileArr) {
131 if (f.isFile()) {
132 files.add(f);
133 }
134 }
135 return files;
136 }
137
138 /**
139 * 读取文件内容
140 *
141 * @param fileName
142 * 待读取的完整文件名
143 * @return 文件内容
144 * @throws IOException
145 */
146 public static String read(String fileName) throws IOException {
147 File f = new File(fileName);
148 FileInputStream fs = new FileInputStream(f);
149 String result = null;
150 byte[] b = new byte[fs.available()];
151 fs.read(b);
152 fs.close();
153 result = new String(b);
154 return result;
155 }
156
157 /**
158 * 写文件
159 *
160 * @param fileName
161 * 目标文件名
162 * @param fileContent
163 * 写入的内容
164 * @return
165 * @throws IOException
166 */
167 public static boolean write(String fileName, String fileContent)
168 throws IOException {
169 boolean result = false;
170 File f = new File(fileName);
171 FileOutputStream fs = new FileOutputStream(f);
172 byte[] b = fileContent.getBytes();
173 fs.write(b);
174 fs.flush();
175 fs.close();
176 result = true;
177 return result;
178 }
179
180 /**
181 * 追加内容到指定文件
182 *
183 * @param fileName
184 * @param fileContent
185 * @return
186 * @throws IOException
187 */
188 public static boolean append(String fileName, String fileContent)
189 throws IOException {
190 boolean result = false;
191 File f = new File(fileName);
192 if (f.exists()) {
193 RandomAccessFile rFile = new RandomAccessFile(f, "rw");
194 byte[] b = fileContent.getBytes();
195 long originLen = f.length();
196 rFile.setLength(originLen + b.length);
197 rFile.seek(originLen);
198 rFile.write(b);
199 rFile.close();
200 }
201 result = true;
202 return result;
203 }
204
205 /**
206 * 拆分文件
207 *
208 * @param fileName
209 * 待拆分的完整文件名
210 * @param byteSize
211 * 按多少字节大小拆分
212 * @return 拆分后的文件名列表
213 * @throws IOException
214 */
215 public List<String> splitBySize(String fileName, int byteSize)
216 throws IOException {
217 List<String> parts = new ArrayList<String>();
218 File file = new File(fileName);
219 int count = (int) Math.ceil(file.length() / (double) byteSize);
220 int countLen = (count + "").length();
221 ThreadPoolExecutor threadPool = new ThreadPoolExecutor(count,
222 count * 3, 1, TimeUnit.SECONDS,
223 new ArrayBlockingQueue<Runnable>(count * 2));
224
225 for (int i = 0; i < count; i++) {
226 String partFileName = file.getName() + "."
227 + leftPad((i + 1) + "", countLen, '0') + ".part";
228 threadPool.execute(new SplitRunnable(byteSize, i * byteSize,
229 partFileName, file));
230 parts.add(partFileName);
231 }
232 return parts;
233 }
234
235 /**
236 * 合并文件
237 *
238 * @param dirPath
239 * 拆分文件所在目录名
240 * @param partFileSuffix
241 * 拆分文件后缀名
242 * @param partFileSize
243 * 拆分文件的字节数大小
244 * @param mergeFileName
245 * 合并后的文件名
246 * @throws IOException
247 */
248 public void mergePartFiles(String dirPath, String partFileSuffix,
249 int partFileSize, String mergeFileName) throws IOException {
250 ArrayList<File> partFiles = FileUtil.getDirFiles(dirPath,
251 partFileSuffix);
252 Collections.sort(partFiles, new FileComparator());
253
254 RandomAccessFile randomAccessFile = new RandomAccessFile(mergeFileName,
255 "rw");
256 randomAccessFile.setLength(partFileSize * (partFiles.size() - 1)
257 + partFiles.get(partFiles.size() - 1).length());
258 randomAccessFile.close();
259
260 ThreadPoolExecutor threadPool = new ThreadPoolExecutor(
261 partFiles.size(), partFiles.size() * 3, 1, TimeUnit.SECONDS,
262 new ArrayBlockingQueue<Runnable>(partFiles.size() * 2));
263
264 for (int i = 0; i < partFiles.size(); i++) {
265 threadPool.execute(new MergeRunnable(i * partFileSize,
266 mergeFileName, partFiles.get(i)));
267 }
268
269 }
270
271 /**
272 * 根据文件名，比较文件
273 *
274 * @author yjmyzz@126.com
275 *
276 */
277 private class FileComparator implements Comparator<File> {
278 public int compare(File o1, File o2) {
279 return o1.getName().compareToIgnoreCase(o2.getName());
280 }
281 }
282
283 /**
284 * 分割处理Runnable
285 *
286 * @author yjmyzz@126.com
287 *
288 */
289 private class SplitRunnable implements Runnable {
290 int byteSize;
291 String partFileName;
292 File originFile;
293 int startPos;
294
295 public SplitRunnable(int byteSize, int startPos, String partFileName,
296 File originFile) {
297 this.startPos = startPos;
298 this.byteSize = byteSize;
299 this.partFileName = partFileName;
300 this.originFile = originFile;
301 }
302
303 public void run() {
304 RandomAccessFile rFile;
305 OutputStream os;
306 try {
307 rFile = new RandomAccessFile(originFile, "r");
308 byte[] b = new byte[byteSize];
309 rFile.seek(startPos);// 移动指针到每“段”开头
310 int s = rFile.read(b);
311 os = new FileOutputStream(partFileName);
312 os.write(b, 0, s);
313 os.flush();
314 os.close();
315 } catch (IOException e) {
316 e.printStackTrace();
317 }
318 }
319 }
320
321 /**
322 * 合并处理Runnable
323 *
324 * @author yjmyzz@126.com
325 *
326 */
327 private class MergeRunnable implements Runnable {
328 long startPos;
329 String mergeFileName;
330 File partFile;
331
332 public MergeRunnable(long startPos, String mergeFileName, File partFile) {
333 this.startPos = startPos;
334 this.mergeFileName = mergeFileName;
335 this.partFile = partFile;
336 }
337
338 public void run() {
339 RandomAccessFile rFile;
340 try {
341 rFile = new RandomAccessFile(mergeFileName, "rw");
342 rFile.seek(startPos);
343 FileInputStream fs = new FileInputStream(partFile);
344 byte[] b = new byte[fs.available()];
345 fs.read(b);
346 fs.close();
347 rFile.write(b);
348 rFile.close();
349 } catch (IOException e) {
350 e.printStackTrace();
351 }
352 }
353 }
354
355 }
View Code
单元测试：
1 package com.cnblogs.yjmyzz;
2
3 import java.io.IOException;
4
5 import org.junit.Test;
6
7 public class FileTest {
8
9 @Test
10 public void writeFile() throws IOException, InterruptedException {
11
12 System.out.println(FileUtil.currentWorkDir);
13
14 StringBuilder sb = new StringBuilder();
15
16 long originFileSize = 1024 * 1024 * 100;// 100M
17 int blockFileSize = 1024 * 1024 * 15;// 15M
18
19 // 生成一个大文件
20 for (int i = 0; i < originFileSize; i++) {
21 sb.append("A");
22 }
23
24 String fileName = FileUtil.currentWorkDir + "origin.myfile";
25 System.out.println(fileName);
26 System.out.println(FileUtil.write(fileName, sb.toString()));
27
28 // 追加内容
29 sb.setLength(0);
30 sb.append("0123456789");
31 FileUtil.append(fileName, sb.toString());
32
33 FileUtil fileUtil = new FileUtil();
34
35 // 将origin.myfile拆分
36 fileUtil.splitBySize(fileName, blockFileSize);
37
38 Thread.sleep(10000);// 稍等10秒，等前面的小文件全都写完
39
40 // 合并成新文件
41 fileUtil.mergePartFiles(FileUtil.currentWorkDir, ".part",
42 blockFileSize, FileUtil.currentWorkDir + "new.myfile");
43
44 }
45 }
View Code
时间： 2024-10-01 17:21:07