Skip to content

Commit 3b20f6b

Browse files
author
vnarayanan
committed
Address more review comments
1 parent 52c722f commit 3b20f6b

File tree

2 files changed

+68
-64
lines changed

2 files changed

+68
-64
lines changed

‎hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/ProfileAWSCredentialsProvider.java

Lines changed: 59 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -36,74 +36,74 @@
3636
@InterfaceAudience.Public
3737
@InterfaceStability.Evolving
3838
public class ProfileAWSCredentialsProvider extends AbstractAWSCredentialProvider {
39-
private static final Logger LOG = LoggerFactory.getLogger(ProfileAWSCredentialsProvider.class);
39+
private static final Logger LOG = LoggerFactory.getLogger(ProfileAWSCredentialsProvider.class);
4040

41-
public static final String NAME
42-
= "org.apache.hadoop.fs.s3a.auth.ProfileAWSCredentialsProvider";
41+
public static final String NAME
42+
= "org.apache.hadoop.fs.s3a.auth.ProfileAWSCredentialsProvider";
4343

44-
/** Conf setting for credentials file path*/
45-
public static final String PROFILE_FILE = "fs.s3a.auth.profile.file";
44+
/** Conf setting for credentials file path*/
45+
public static final String PROFILE_FILE = "fs.s3a.auth.profile.file";
4646

47-
/** Conf setting for profile name*/
48-
public static final String PROFILE_NAME = "fs.s3a.auth.profile.name";
47+
/** Conf setting for profile name*/
48+
public static final String PROFILE_NAME = "fs.s3a.auth.profile.name";
4949

50-
/** Environment variable for credentials file path*/
51-
public static final String CREDENTIALS_FILE_ENV = "AWS_SHARED_CREDENTIALS_FILE";
52-
/** Environment variable for profile name*/
53-
public static final String PROFILE_ENV = "AWS_PROFILE";
50+
/** Environment variable for credentials file path*/
51+
public static final String CREDENTIALS_FILE_ENV = "AWS_SHARED_CREDENTIALS_FILE";
52+
/** Environment variable for profile name*/
53+
public static final String PROFILE_ENV = "AWS_PROFILE";
5454

55-
private final ProfileCredentialsProvider pcp;
55+
private final ProfileCredentialsProvider pcp;
5656

57-
private static Path getCredentialsPath(Configuration conf) {
58-
String credentialsFile = conf.get(PROFILE_FILE, null);
59-
if (credentialsFile == null) {
60-
credentialsFile = SystemUtils.getEnvironmentVariable(CREDENTIALS_FILE_ENV, null);
61-
if (credentialsFile != null) {
62-
LOG.debug("Fetched credentials file path from environment variable");
63-
}
64-
}
65-
else {
66-
LOG.debug("Fetched credentials file path from conf");
67-
}
68-
if (credentialsFile == null) {
69-
LOG.debug("Using default credentials file path");
70-
return FileSystems.getDefault().getPath(SystemUtils.getUserHome().getPath(),".aws", "credentials");
71-
}
72-
else {
73-
return FileSystems.getDefault().getPath(credentialsFile);
74-
}
57+
private static Path getCredentialsPath(Configuration conf) {
58+
String credentialsFile = conf.get(PROFILE_FILE, null);
59+
if (credentialsFile == null) {
60+
credentialsFile = SystemUtils.getEnvironmentVariable(CREDENTIALS_FILE_ENV, null);
61+
if (credentialsFile != null) {
62+
LOG.debug("Fetched credentials file path from environment variable");
63+
}
7564
}
76-
77-
private static String getCredentialsName(Configuration conf) {
78-
String profileName = conf.get(PROFILE_NAME, null);
79-
if (profileName == null) {
80-
profileName = SystemUtils.getEnvironmentVariable(PROFILE_ENV, null);
81-
if (profileName == null) {
82-
profileName = "default";
83-
LOG.debug("Using default profile name");
84-
}
85-
else {
86-
LOG.debug("Fetched profile name from environment variable");
87-
}
88-
}
89-
else {
90-
LOG.debug("Fetched profile name from conf");
91-
}
92-
return profileName;
65+
else {
66+
LOG.debug("Fetched credentials file path from conf");
9367
}
94-
95-
public ProfileAWSCredentialsProvider(URI uri, Configuration conf) {
96-
super(uri, conf);
97-
ProfileCredentialsProvider.Builder builder = ProfileCredentialsProvider.builder();
98-
builder.profileName(getCredentialsName(conf))
99-
.profileFile(ProfileFile.builder()
100-
.content(getCredentialsPath(conf))
101-
.type(ProfileFile.Type.CREDENTIALS)
102-
.build());
103-
pcp = builder.build();
68+
if (credentialsFile == null) {
69+
LOG.debug("Using default credentials file path");
70+
return FileSystems.getDefault().getPath(SystemUtils.getUserHome().getPath(),".aws", "credentials");
71+
}
72+
else {
73+
return FileSystems.getDefault().getPath(credentialsFile);
10474
}
75+
}
10576

106-
public AwsCredentials resolveCredentials() {
107-
return pcp.resolveCredentials();
77+
private static String getCredentialsName(Configuration conf) {
78+
String profileName = conf.get(PROFILE_NAME, null);
79+
if (profileName == null) {
80+
profileName = SystemUtils.getEnvironmentVariable(PROFILE_ENV, null);
81+
if (profileName == null) {
82+
profileName = "default";
83+
LOG.debug("Using default profile name");
84+
}
85+
else {
86+
LOG.debug("Fetched profile name from environment variable");
87+
}
10888
}
89+
else {
90+
LOG.debug("Fetched profile name from conf");
91+
}
92+
return profileName;
93+
}
94+
95+
public ProfileAWSCredentialsProvider(URI uri, Configuration conf) {
96+
super(uri, conf);
97+
ProfileCredentialsProvider.Builder builder = ProfileCredentialsProvider.builder();
98+
builder.profileName(getCredentialsName(conf))
99+
.profileFile(ProfileFile.builder()
100+
.content(getCredentialsPath(conf))
101+
.type(ProfileFile.Type.CREDENTIALS)
102+
.build());
103+
pcp = builder.build();
104+
}
105+
106+
public AwsCredentials resolveCredentials() {
107+
return pcp.resolveCredentials();
108+
}
109109
}

‎hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/authentication.md

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ For more information see [Upcoming upgrade to AWS Java SDK V2](./aws_sdk_upgrade
5757
<description>
5858
Comma-separated class names of credential provider classes which implement
5959
software.amazon.awssdk.auth.credentials.AwsCredentialsProvider.
60-
60+
6161
org.apache.hadoop.fs.s3a.auth.ProfileAWSCredentialsProvider is not included in
6262
the chain by default.
6363

@@ -229,18 +229,22 @@ Note:
229229
### <a name="auth_simple"></a> Credentials from profile with `ProfileAWSCredentialsProvider`*
230230
231231
This is a non-default provider that fetches credentials from a profile file,
232-
acting as a Hadoop wrapper around ProfileCredentialsProvider. The profile file and
232+
acting as a Hadoop wrapper around [ProfileCredentialsProvider](https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/auth/credentials/ProfileCredentialsProvider.html). The profile file and
233233
profile name are both resolved as follows.
234234
235-
1. If the configuration setting is specified, that takes priority (`fs.s3a.auth.profile.file`
235+
1. If the configuration setting is specified, that takes priority ( `fs.s3a.auth.profile.file`
236236
for profile file and `fs.s3a.auth.profile.name` for profile name).
237-
2. If a configuration setting is absent, but the environment variables for
238-
the setting(AWS_SHARED_CREDENTIALS_FILE for profile file and AWS_PROFILE for
237+
2. If a configuration setting is absent, but the environment variable for
238+
the setting( `AWS_SHARED_CREDENTIALS_FILE` for profile file and `AWS_PROFILE` for
239239
profile name) is defined, then the variable is used.
240240
3. If neither configuration setting nor environment variable is present, then
241241
the values default to `~/.aws/credentials` for the profile file, and `default`
242242
for the profile name.
243243
244+
245+
*Important*: This profile file must be on every node in the _cluster_.
246+
If this is not the case, delegation tokens can be used to collect the current credentials and propagate them.
247+
244248
### <a name="auth_session"></a> Using Session Credentials with `TemporaryAWSCredentialsProvider`
245249
246250
[Temporary Security Credentials](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html)

0 commit comments

Comments
 (0)