10000 HADOOP-12774 use UGI.currentUser for user and group of s3a objects by steveloughran · Pull Request #136 · apache/hadoop · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

HADOOP-12774 use UGI.currentUser for user and group of s3a objects #136

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 4 commits into from
Closed

HADOOP-12774 use UGI.currentUser for user and group of s3a objects #136

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ba8ed5b
HADOOP-12774 use UGI.currentUser for user and group of s3a objects
steveloughran Oct 7, 2016
3a92100
Patch 002: create a fake user, create an FS and verify that the user …
steveloughran Oct 10, 2016
f9dae32
HADOOP-12774: Address Chris's comments about using parent class field…
steveloughran Oct 12, 2016
fae8ee7
HADOOP-12774: Address checkstyle complaints
steveloughran Oct 13, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -298,7 +298,7 @@ private boolean buildNextStatusBatch(ObjectListing objects) {
// Skip over keys that are ourselves and old S3N _$folder$ files
if (acceptor.accept(keyPath, summary) && filter.accept(keyPath)) {
FileStatus status = createFileStatus(keyPath, summary,
owner.getDefaultBlockSize(keyPath));
owner.getDefaultBlockSize(keyPath), owner.getUsername());
LOG.debug("Adding: {}", status);
stats.add(status);
added++;
Expand All @@ -312,7 +312,8 @@ private boolean buildNextStatusBatch(ObjectListing objects) {
for (String prefix : objects.getCommonPrefixes()) {
Path keyPath = owner.keyToQualifiedPath(prefix);
if (acceptor.accept(keyPath, prefix) && filter.accept(keyPath)) {
FileStatus status = new S3AFileStatus(true, false, keyPath);
FileStatus status = new S3AFileStatus(false, keyPath,
owner.getUsername());
LOG.debug("Adding directory: {}", status);
added++;
stats.add(status);
Expand Down
33 changes: 23 additions & 10 deletions hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileStatus.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,28 +33,41 @@
public class S3AFileStatus extends FileStatus {
private boolean isEmptyDirectory;

// Directories
public S3AFileStatus(boolean isdir, boolean isemptydir, Path path) {
super(0, isdir, 1, 0, 0, path);
/**
* Create a directory status.
* @param isemptydir is this an empty directory?
* @param path the path
* @param owner the owner
*/
public S3AFileStatus(boolean isemptydir,
Path path,
String owner) {
super(0, true, 1, 0, 0, path);
isEmptyDirectory = isemptydir;
setOwner(owner);
setGroup(owner);
}

// Files
/**
* A simple file.
* @param length file length
* @param modification_time mod time
* @param path path
* @param blockSize block size
* @param owner owner
*/
public S3AFileStatus(long length, long modification_time, Path path,
long blockSize) {
long blockSize, String owner) {
super(length, false, 1, blockSize, modification_time, path);
isEmptyDirectory = false;
setOwner(owner);
setGroup(owner);
}

public boolean isEmptyDirectory() {
return isEmptyDirectory;
}

@Override
public String getOwner() {
return System.getProperty("user.name");
}

/** Compare if this object is equal to another object.
* @param o the object to be compared.
* @return true if two file status has the same path name; false if not.
Expand Down
29 changes: 21 additions & 8 deletions hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@
import org.apache.hadoop.fs.StorageStatistics;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.s3native.S3xLoginHelper;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.ReflectionUtils;

Expand Down Expand Up @@ -119,6 +120,7 @@ public class S3AFileSystem extends FileSystem {
public static final int DEFAULT_BLOCKSIZE = 32 * 1024 * 1024;
private URI uri;
private Path workingDir;
private String username;
private AmazonS3 s3;
private String bucket;
private int maxKeys;
Expand Down Expand Up @@ -159,7 +161,9 @@ public void initialize(URI name, Configuration conf) throws IOException {
instrumentation = new S3AInstrumentation(name);

uri = S3xLoginHelper.buildFSURI(name);
workingDir = new Path("/user", System.getProperty("user.name"))
// Username is the current user at the time the FS was instantiated.
username = UserGroupInformation.getCurrentUser().getShortUserName();
workingDir = new Path("/user", username)
.makeQualified(this.uri, this.getWorkingDirectory());

bucket = name.getHost();
Expand Down Expand Up @@ -1386,6 +1390,14 @@ public Path getWorkingDirectory() {
return workingDir;
}

/**
* Get the username of the FS.
* @return the short name of the user who instantiated the FS
*/
public String getUsername() {
return username;
}

/**
*
* Make the given path and all non-existent parents into
Expand Down Expand Up @@ -1477,14 +1489,14 @@ public S3AFileStatus getFileStatus(final Path f) throws IOException {

if (objectRepresentsDirectory(key, meta.getContentLength())) {
LOG.debug("Found exact file: fake directory");
return new S3AFileStatus(true, true,
path);
return new S3AFileStatus(true, path, username);
} else {
LOG.debug("Found exact file: normal file");
return new S3AFileStatus(meta.getContentLength(),
dateToLong(meta.getLastModified()),
path,
getDefaultBlockSize(path));
getDefaultBlockSize(path),
username);
}
} catch (AmazonServiceException e) {
if (e.getStatusCode() != 404) {
Expand All @@ -1502,15 +1514,16 @@ public S3AFileStatus getFileStatus(final Path f) throws IOException {

if (objectRepresentsDirectory(newKey, meta.getContentLength())) {
LOG.debug("Found file (with /): fake directory");
return new S3AFileStatus(true, true, path);
return new S3AFileStatus(true, path, username);
} else {
LOG.warn("Found file (with /): real file? should not happen: {}",
key);

return new S3AFileStatus(meta.getContentLength(),
dateToLong(meta.getLastModified()),
path,
getDefaultBlockSize(path));
getDefaultBlockSize(path),
username);
}
} catch (AmazonServiceException e) {
if (e.getStatusCode() != 404) {
Expand Down Expand Up @@ -1547,10 +1560,10 @@ public S3AFileStatus getFileStatus(final Path f) throws IOException {
}
}

return new S3AFileStatus(true, false, path);
return new S3AFileStatus(false, path, username);
} else if (key.isEmpty()) {
LOG.debug("Found root directory");
return new S3AFileStatus(true, true, path);
return new S3AFileStatus(true, path, username);
}
} catch (AmazonServiceException e) {
if (e.getStatusCode() != 404) {
Expand Down
8 changes: 5 additions & 3 deletions hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -236,17 +236,19 @@ public static String stringify(AmazonS3Exception e) {
* @param keyPath path to entry
* @param summary summary from AWS
* @param blockSize block size to declare.
* @param owner owner of the file
* @return a status entry
*/
public static S3AFileStatus createFileStatus(Path keyPath,
S3ObjectSummary summary,
long blockSize) {
long blockSize,
String owner) {
if (objectRepresentsDirectory(summary.getKey(), summary.getSize())) {
return new S3AFileStatus(true, true, keyPath);
return new S3AFileStatus(true, keyPath, owner);
} else {
return new S3AFileStatus(summary.getSize(),
dateToLong(summary.getLastModified()), keyPath,
blockSize);
blockSize, owner);
}
}

Expand Down
23 changes: 22 additions & 1 deletion hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,10 @@

import java.io.File;
import java.net.URI;
import java.security.PrivilegedExceptionAction;

import org.apache.hadoop.security.ProviderUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.hadoop.util.VersionInfo;
Expand Down Expand Up @@ -436,7 +438,7 @@ public void testDirectoryAllocatorRR() throws Throwable {
dir1.mkdirs();
dir2.mkdirs();
conf = new Configuration();
conf.set(Constants.BUFFER_DIR, dir1 +", " + dir2);
conf.set(Constants.BUFFER_DIR, dir1 + ", " + dir2);
fs = S3ATestUtils.createTestFileSystem(conf);
File tmp1 = fs.createTmpFileForWrite("out-", 1024, conf);
tmp1.delete();
Expand All @@ -446,6 +448,25 @@ public void testDirectoryAllocatorRR() throws Throwable {
tmp1.getParent(), tmp2.getParent());
}

@Test
public void testUsernameFromUGI() throws Throwable {
final String alice = "alice";
UserGroupInformation fakeUser =
UserGroupInformation.createUserForTesting(alice,
new String[]{"users", "administrators"});
conf = new Configuration();
fs = fakeUser.doAs(new PrivilegedExceptionAction<S3AFileSystem>() {
@Override
public S3AFileSystem run() throws Exception{
return S3ATestUtils.createTestFileSystem(conf);
}
});
assertEquals("username", alice, fs.getUsername());
S3AFileStatus status = fs.getFileStatus(new Path("/"));
assertEquals("owner in " + status, alice, status.getOwner());
assertEquals("group in " + status, alice, status.getGroup());
}

/**
* Reads and returns a field from an object using reflection. If the field
* cannot be found, is null, or is not the expected type, then this method
Expand Down
0