8000 HDFS-17781.Fix webImageViewer getContentSummary return wrong result when inode is a ec file by LiuGuH · Pull Request #7676 · apache/hadoop · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

HDFS-17781.Fix webImageViewer getContentSummary return wrong result when inode is a ec file #7676

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: trunk
Choose a base branch
from
+93 −9
Open

HDFS-17781.Fix webImageViewer getContentSummary return wrong result when inode is a ec file #7676

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
32edb1f
HDFS-17781.WebImageViewer supports ec files getcontentsummary return …
LiuGuH May 7, 2025
48fd51f
init ErasureCodingPolicyManager
LiuGuH May 8, 2025
01aec38
triggle compile
LiuGuH May 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 44 additions & 2 deletions ...oop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,16 +37,22 @@
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.XAttrHelper;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
import org.apache.hadoop.hdfs.protocol.XAttrNotFoundException;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
import org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode;
import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf;
import org.apache.hadoop.hdfs.server.namenode.FSImageUtil;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.ErasureCodingSection;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.INode;
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.hdfs.server.namenode.INodeId;
import org.apache.hadoop.hdfs.server.namenode.SerialNumberManager;
import org.apache.hadoop.hdfs.util.StripedBlockUtil;
import org.apache.hadoop.hdfs.web.JsonUtil;
import org.apache.hadoop.hdfs.web.resources.XAttrEncodingParam;
import org.apache.hadoop.io.IOUtils;
Expand Down Expand Up @@ -232,6 +238,22 @@ static ImmutableList<Long> loadINodeReferenceSection(InputStream in)
return builder.build();
}

static ErasureCodingPolicyManager loadErasureCodingSection(InputStream in) throws IOException {
ErasureCodingSection s = ErasureCodingSection.parseDelimitedFrom(in);
List<ErasureCodingPolicyInfo> ecPolicies =
org.apache.hadoop.thirdparty.com.google.common.collect.Lists.newArrayListWithCapacity(
s.getPoliciesCount());
for (int i = 0; i < s.getPoliciesCount(); ++i) {
ecPolicies.add(PBHelperClient.convertErasureCodingPolicyInfo(s.getPolicies(i)));
}
Configuration conf = new Configuration();
ErasureCodingPolicyManager erasureCodingPolicyManager =
ErasureCodingPolicyManager.getInstance();
erasureCodingPolicyManager.init(conf);
erasureCodingPolicyManager.loadPolicies(ecPolicies, conf);
return erasureCodingPolicyManager;
}

private static byte[][] loadINodeSection(InputStream in)
throws IOException {
FsImageProto.INodeSection s = FsImageProto.INodeSection
Expand Down Expand Up @@ -344,7 +366,7 @@ private Map<String, Object> getContentSummaryMap(String path)
data[1] = 1;
data[2] = getFileSize(f);
nsQuota = -1;
data[3] = data[2] * f.getReplication();
data[3] = getStorageSpaceConsumed(f);
spaceQuota = -1;
return fillSummaryMap(spaceQuota, nsQuota, data);
case DIRECTORY:
Expand Down Expand Up @@ -396,7 +418,7 @@ private void fillDirSummary(long id, long[] data) throws IOException {
long curLength = getFileSize(f);
data[1]++;
data[2] += curLength;
data[3] += (curLength) * (f.getReplication());
data[3] += getStorageSpaceConsumed(f);
break;
case SYMLINK:
data[1]++;
Expand Down Expand Up @@ -662,6 +684,26 @@ static long getFileSize(FsImageProto.INodeSection.INodeFile f) {
return size;
}

static long getStorageSpaceConsumed(FsImageProto.INodeSection.INodeFile f) {
long size = 0;
boolean isStriped = f.hasErasureCodingPolicyID();
ErasureCodingPolicy ecPolicy = null;
if (isStriped) {
ecPolicy =
ErasureCodingPolicyManager.getInstance().getByID((byte) f.getErasureCodingPolicyID());
}

for (HdfsProtos.BlockProto p : f.getBlocksList()) {
if (isStriped) {
size += StripedBlockUtil.spaceConsumedByStripedBlock(p.getNumBytes(),
ecPolicy.getNumDataUnits(), ecPolicy.getNumParityUnits(), ecPolicy.getCellSize());
} else {
size += p.getNumBytes() * f.getReplication();
}
}
return size;
}

private String toString(FsPermission permission) {
return String.format("%o", permission.toShort());
}
Expand Down
4 changes: 4 additions & 0 deletions ...hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -643,6 +643,10 @@ public int compare(FsImageProto.FileSummary.Section s1,
LOG.info("Loading inode references");
refIdList = FSImageLoader.loadINodeReferenceSection(is);
break;
case ERASURE_CODING:
LOG.info("Loading erasure coding");
FSImageLoader.loadErasureCodingSection(is);
break;
default:
break;
}
Expand Down
52 changes: 45 additions & 7 deletions .../apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java
View file
Open in desktop
E61E
Original file line number Diff line number Diff line change
Expand Up @@ -22,24 +22,31 @@

import java.io.File;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.SafeModeAction;
import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.StripedFileTestUtil;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil;
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
import org.apache.hadoop.net.NetUtils;

import org.junit.After;
import org.junit.Before;
import org.junit.Test;
Expand Down Expand Up @@ -98,18 +105,18 @@ public void testFileHavingMultipleBlocks() throws Exception {
}

@Test(timeout = 60000)
public void testFileLargerThanABlockGroup1() throws IOException {
public void testFileLargerThanABlockGroup1() throws Exception {
testFileSize(blockSize * dataBlocks + cellSize + 123);
}

@Test(timeout = 60000)
public void testFileLargerThanABlockGroup2() throws IOException {
public void testFileLargerThanABlockGroup2() throws Exception {
testFileSize(blockSize * dataBlocks * 3 + cellSize * dataBlocks + cellSize
+ 123);
}

@Test(timeout = 60000)
public void testFileFullBlockGroup() throws IOException {
public void testFileFullBlockGroup() throws Exception {
testFileSize(blockSize * dataBlocks);
}

Expand All @@ -119,8 +126,7 @@ public void testFileMoreThanOneStripe() throws Exception {
testFileSize(numBytes);
}

private void testFileSize(int numBytes) throws IOException,
UnresolvedLinkException, SnapshotAccessControlException {
private void testFileSize(int numBytes) throws Exception{
fs.setSafeMode(SafeModeAction.LEAVE);
File orgFsimage = null;
Path file = new Path("/eczone/striped");
Expand Down Expand Up @@ -166,5 +172,37 @@ private void testFileSize(int numBytes) throws IOException,
"Wrongly computed file size contains striped blocks, file status:"
+ fileStatus + ". Expected file size is : " + EXPECTED_FILE_SIZE,
fileStatus.contains(EXPECTED_FILE_SIZE));

testGetContentSummaryForFile(orgFsimage, file);
}

public void testGetContentSummaryForFile(File orgFsimage, Path file)
throws IOException, InterruptedException, URISyntaxException {
ContentSummary contentSummary = fs.getContentSummary(file);
try (WebImageViewer viewer = new WebImageViewer(NetUtils.createSocketAddr("localhost:0"))) {
viewer.initServer(orgFsimage.getAbsolutePath());
int port = viewer.getPort();
URL url =
new URL("http://localhost:" + port + "/webhdfs/v1" + file + "?op=GETCONTENTSUMMARY");
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setRequestMethod("GET");
connection.connect();
assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode());
// create a WebHdfsFileSystem instance
URI uri = new URI("webhdfs://localhost:" + port);
Configuration conf = new Configuration();
WebHdfsFileSystem webfs = (WebHdfsFileSystem) FileSystem.get(uri, conf);
ContentSummary summary = webfs.getContentSummary(file);
verifyContentSummary(contentSummary, summary);
}
}

private void verifyContentSummary(ContentSummary expected, ContentSummary actual) {
assertEquals(expected.getDirectoryCount(), actual.getDirectoryCount());
assertEquals(expected.getFileCount(), actual.getFileCount());
assertEquals(expected.getLength(), actual.getLength());
assertEquals(expected.getSpaceConsumed(), actual.getSpaceConsumed());
assertEquals(expected.getQuota(), actual.getQuota());
assertEquals(expected.getSpaceQuota(), actual.getSpaceQuota());
}
}
0