apache · jmahonin · Jun 22, 2022 · Aug 19, 2022 · Aug 24, 2022 · Aug 24, 2022
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -79,6 +79,14 @@ private Constants() {
   public static final String ASSUMED_ROLE_ARN =
       "fs.s3a.assumed.role.arn";
 
+
+  /**
+   * External ID for the assumed role request, must be valid characters according
+   * to the AWS APIs: {@value}.
+   */
+  public static final String ASSUMED_ROLE_EXTERNAL_ID =
+      "fs.s3a.assumed.role.externalid";
+
   /**
    * Session name for the assumed role, must be valid characters according
    * to the AWS APIs: {@value}.

diff --git a/...hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java b/...hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java
@@ -82,6 +82,8 @@ public class AssumedRoleCredentialProvider implements AWSCredentialsProvider,
 
   private final String arn;
 
+  private final String externalId;
+
   private final AWSCredentialProviderList credentialsToSTS;
 
   private final Invoker invoker;
@@ -118,16 +120,21 @@ public AssumedRoleCredentialProvider(@Nullable URI fsUri, Configuration conf)
         buildSessionName());
     duration = conf.getTimeDuration(ASSUMED_ROLE_SESSION_DURATION,
         ASSUMED_ROLE_SESSION_DURATION_DEFAULT, TimeUnit.SECONDS);
+    externalId = conf.getTrimmed(ASSUMED_ROLE_EXTERNAL_ID);
     String policy = conf.getTrimmed(ASSUMED_ROLE_POLICY, "");
-
     LOG.debug("{}", this);
     STSAssumeRoleSessionCredentialsProvider.Builder builder
         = new STSAssumeRoleSessionCredentialsProvider.Builder(arn, sessionName);
     builder.withRoleSessionDurationSeconds((int) duration);
+    if (StringUtils.isNotEmpty(externalId)) {
+      LOG.debug("External Id {}", externalId);
+      builder.withExternalId(externalId);
+    }
     if (StringUtils.isNotEmpty(policy)) {
       LOG.debug("Scope down policy {}", policy);
       builder.withScopeDownPolicy(policy);
     }
+
     String endpoint = conf.getTrimmed(ASSUMED_ROLE_STS_ENDPOINT, "");
     String region = conf.getTrimmed(ASSUMED_ROLE_STS_ENDPOINT_REGION,
         ASSUMED_ROLE_STS_ENDPOINT_REGION_DEFAULT);
@@ -199,7 +206,8 @@ public String toString() {
     final StringBuilder sb = new StringBuilder(
         "AssumedRoleCredentialProvider{");
     sb.append("role='").append(arn).append('\'');
-    sb.append(", session'").append(sessionName).append('\'');
+    sb.append(", session='").append(sessionName).append('\'');
+    sb.append(", externalId='").append(externalId).append('\'');
     sb.append(", duration=").append(duration);
     sb.append('}');
     return sb.toString();

diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md
@@ -142,6 +142,14 @@ Here are the full set of configuration options.
   </description>
 </property>
 
+<property>
+  <name>fs.s3a.assumed.role.externalid</name>
+  <value />
+  <description>
+    Optional externalId to specify when assuming a role
+  </description>
+</property>
+
 <property>
   <name>fs.s3a.assumed.role.session.name</name>
   <value />
@@ -647,7 +655,7 @@ The security token included in the request is invalid.
   ... 25 more
 ```
 
-### <a name="invalid_session"></a> `AWSSecurityTokenServiceExceptiond`: "Member must satisfy regular expression pattern: `[\w+=,.@-]*`"
+### <a name="invalid_session"></a> `AWSSecurityTokenServiceException`: "Member must satisfy regular expression pattern: `[\w+=,.@-]*`"
 
 
 The session name, as set in `fs.s3a.assumed.role.session.name` must match the wildcard `[\w+=,.@-]*`.
@@ -685,6 +693,34 @@ Caused by: com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceExc
   at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1303)
 ```
 
+Similarly, if `fs.s3a.assumed.role.externalid` is specified, it must match the same wildcard `[\w+=,.@-]*`.
+
+```
+org.apache.hadoop.fs.s3a.AWSBadRequestException:
+ Instantiate org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider:
+   com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException:
+   1 validation error detected: Value 'invalid external id' at 'externalId' failed to satisfy constraint:
+   Member must satisfy regular expression pattern: [\w+=,.@:\/-]*
+   (Service: AWSSecurityTokenService; Status Code: 400; Error Code: ValidationError;
+  at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:241)
+  at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProvider(S3AUtils.java:730)
+  at org.apache.hadoop.fs.s3a.S3AUtils.buildAWSProviderList(S3AUtils.java:644)
+  at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProviderSet(S3AUtils.java:577)
+  at org.apache.hadoop.fs.s3a.S3AFileSystem.bindAWSClient(S3AFileSystem.java:878)
+  at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:523)
+  at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3563)
+  at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:553)
+  at org.apache.hadoop.fs.Path.getFileSystem(Path.java:366)
+
+Caused by: com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException:
+    1 validation error detected: Value 'invalid external id' at 'externalId'
+    failed to satisfy constraint:
+    Member must satisfy regular expression pattern: [\w+=,.@:\/-]*
+    (Service: AWSSecurityTokenService; Status Code: 400; Error Code: ValidationError; Request ID: 2f53f2c9-ef6a-4561-ba43-bdec489136ae; Proxy: null)
+  at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1879)
+  at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1418)
+  at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1387)
+```
 
 ### <a name="access_denied"></a> `java.nio.file.AccessDeniedException` within a FileSystem API call
 

diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java
@@ -191,6 +191,14 @@ public void testAssumedInvalidRole() throws Throwable {
         () -> new AssumedRoleCredentialProvider(uri, conf));
   }
 
+  @Test
+  public void testAssumedRoleBadExternalId() throws Throwable {
+    describe("Attempt to create the FS with an invalid external id");
+    Configuration conf = createAssumedRoleConfig();
+    conf.set(ASSUMED_ROLE_EXTERNAL_ID, "invalid_external_id!");
+    expectFileSystemCreateFailure(conf, AWSBadRequestException.class, "");
+  }
+
   @Test
   public void testAssumeRoleFSBadARN() throws Exception {
     describe("Attemnpt to create the FS with an invalid ARN");

diff --git a/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml b/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml
@@ -34,10 +34,15 @@
 
   <property>
     <name>fs.s3a.bucket.landsat-pds.endpoint</name>
-    <value>${central.endpoint}</value>
+    <value>${oregon.endpoint}</value>
     <description>The endpoint for s3a://landsat-pds URLs</description>
   </property>
 
+  <property>
+    <name>fs.s3a.bucket.landsat-pds.endpoint.region</name>
+    <value>us-west-2</value>
+  </property>
+
   <property>
     <name>fs.s3a.bucket.landsat-pds.multipart.purge</name>
     <value>false</value>
@@ -61,7 +66,12 @@
 
   <property>
     <name>fs.s3a.bucket.usgs-landsat.endpoint</name>
-    <value>${central.endpoint}</value>
+    <value>${oregon.endpoint}</value>
+  </property>
+
+  <property>
+    <name>fs.s3a.bucket.usgs-landsat.endpoint.region</name>
+    <value>us-west-2</value>
   </property>
 
   <property>