8000 HDFS-17376. Distcp creates Factor 1 replication file on target if Source is EC. by sadanand48 · Pull Request #6540 · apache/hadoop · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

HDFS-17376. Distcp creates Factor 1 replication file on target if Source is EC. #6540

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. 8000 We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Feb 9, 2024
Merged

HDFS-17376. Distcp creates Factor 1 replication file on target if Source is EC. #6540

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1cbb07e
HDFS-17376. Distcp creates Factor 1 replication file on target if Sou…
Feb 8, 2024
e343c4c
fix checkstyle
Feb 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .../hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -380,6 +380,9 @@ private static ThrottledInputStream getInputStream(Path path,
private static short getReplicationFactor(
EnumSet<FileAttribute> fileAttributes, CopyListingFileStatus source,
FileSystem targetFS, Path tmpTargetPath) {
if (source.isErasureCoded()) {
return targetFS.getDefaultReplication(tmpTargetPath);
}
return fileAttributes.contains(FileAttribute.REPLICATION)
? source.getReplication()
: targetFS.getDefaultReplication(tmpTargetPath);
Expand Down
24 changes: 21 additions & 3 deletions ...op-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpWithRawXAttrs.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ public class TestDistCpWithRawXAttrs {
private static final Path dir1 = new Path("/src/dir1");
private static final Path subDir1 = new Path(dir1, "subdir1");
private static final Path file1 = new Path("/src/file1");
private static final Path FILE_2 = new Path("/src/dir1/file2");
private static final String rawRootName = "/.reserved/raw";
private static final String rootedDestName = "/dest";
private static final String rootedSrcName = "/src";
Expand All @@ -73,7 +74,7 @@ public static void init() throws Exception {
conf = new Configuration();
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY, true);
conf.setInt(DFSConfigKeys.DFS_LIST_LIMIT, 2);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).format(true)
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).format(true)
.build();
cluster.waitActive();
fs = cluster.getFileSystem();
Expand Down Expand Up @@ -178,7 +179,7 @@ private void doTestPreserveRawXAttrs(String src, String dest,
}

@Test
public void testPreserveEC() throws Exception {
public void testPreserveAndNoPreserveEC() throws Exception {
final String src = "/src";
final String dest = "/dest";

Expand All @@ -190,9 +191,11 @@ public void testPreserveEC() throws Exception {

fs.delete(new Path("/dest"), true);
fs.mkdirs(subDir1);
fs.create(file1).close();
DistributedFileSystem dfs = (DistributedFileSystem) fs;
dfs.enableErasureCodingPolicy("XOR-2-1-1024k");
dfs.setErasureCodingPolicy(dir1, "XOR-2-1-1024k");
fs.create(file1).close();
fs.create(FILE_2).close();
int res = ToolRunner.run(conf, new ECAdmin(conf), args);
assertEquals("Unable to set EC policy on " + subDir1.toString(), res, 0);

Expand All @@ -203,6 +206,7 @@ public void testPreserveEC() throws Exception {
FileStatus srcStatus = fs.getFileStatus(new Path(src));
FileStatus srcDir1Status = fs.getFileStatus(dir1);
FileStatus srcSubDir1Status = fs.getFileStatus(subDir1);
FileStatus srcFile2Status = fs.getFileStatus(FILE_2);

FileStatus destStatus = fs.getFileStatus(new Path(dest));
FileStatus destDir1Status = fs.getFileStatus(destDir1);
Expand All @@ -214,12 +218,26 @@ public void testPreserveEC() throws Exception {
destStatus.isErasureCoded());
assertTrue("/src/dir1 is not erasure coded!",
srcDir1Status.isErasureCoded());
assertTrue("/src/dir1/file2 is not erasure coded",
srcFile2Status.isErasureCoded());
assertTrue("/dest/dir1 is not erasure coded!",
destDir1Status.isErasureCoded());
assertTrue("/src/dir1/subdir1 is not erasure coded!",
srcSubDir1Status.isErasureCoded());
assertTrue("/dest/dir1/subdir1 is not erasure coded!",
destSubDir1Status.isErasureCoded());

// test without -p to check if src is EC then target FS default replication
// is obeyed on the target file.

fs.delete(new Path(dest), true);
DistCpTestUtils.assertRunDistCp(DistCpConstants.SUCCESS, src, dest, null,
conf);
FileStatus destFileStatus = fs.getFileStatus(new Path(destDir1, "file2"));
assertFalse(destFileStatus.isErasureCoded());
assertEquals(fs.getDefaultReplication(new Path(destDir1, "file2")),
destFileStatus.getReplication());
dfs.unsetErasureCodingPolicy(dir1);
}

@Test
Expand Down
0