8000 YARN-11821. Support pattern matching for YARN proxy addresses. by kakao-sue-sylee · Pull Request #7697 · apache/hadoop · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

YARN-11821. Support pattern matching for YARN proxy addresses. #7697

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: trunk
Choose a base branch
from
+78 −1
Open

YARN-11821. Support pattern matching for YARN proxy addresses. #7697

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions ...oop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2977,6 +2977,12 @@ public static boolean isAclEnabled(Configuration conf) {
public static final String DEFAULT_PROXY_ADDRESS =
"0.0.0.0:" + DEFAULT_PROXY_PORT;

/** A regex pattern that matches the proxy's hostname.
* Used in AmIpFilter, if not set, reverse DNS hostname-based filtering will not be performed.
*/
public static final String PROXY_ADDRESS_PATTERN =
PROXY_ADDRESS + ".pattern";

/** Binding address for the web proxy. */
public static final String PROXY_BIND_HOST =
PROXY_PREFIX + "bind-host";
Expand Down
1 change: 1 addition & 0 deletions ...adoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,7 @@ public void initializeMemberVariables() {
YarnConfiguration.NM_HEALTH_CHECK_SCRIPT_INTERVAL_MS_TEMPLATE);
configurationPropsToSkipCompare.add(YarnConfiguration.NM_AUX_SERVICE_REMOTE_CLASSPATH);
configurationPropsToSkipCompare.add(YarnConfiguration.LINUX_CONTAINER_RUNTIME_CLASS_FMT);
configurationPropsToSkipCompare.add(YarnConfiguration.PROXY_ADDRESS_PATTERN);
}

/**
Expand Down
72 changes: 71 additions & 1 deletion ...r-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,11 @@

import org.apache.hadoop.classification.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.webproxy.ProxyUtils;
import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet;
import org.slf4j.Logger;
Expand All @@ -30,6 +33,7 @@
import javax.servlet.Filter;
import javax.servlet.FilterChain;
import javax.servlet.FilterConfig;
import javax.servlet.ServletContext;
import javax.servlet.ServletException;
import javax.servlet.ServletRequest;
import javax.servlet.ServletResponse;
Expand All @@ -47,6 +51,7 @@
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;

@Public
public class AmIpFilter implements Filter {
Expand All @@ -69,10 +74,15 @@ public class AmIpFilter implements Filter {
private long lastUpdate;
@VisibleForTesting
Map<String, String> proxyUriBases;
Pattern proxyHostPattern = null;
String rmUrls[] = null;

@Override
public void init(FilterConfig conf) throws ServletException {
Configuration config = loadConfiguration(conf);
YarnConfiguration yarnConf = (config instanceof YarnConfiguration) ?
(YarnConfiguration) config : new YarnConfiguration(config);

// Maintain for backwards compatibility
if (conf.getInitParameter(PROXY_HOST) != null
&& conf.getInitParameter(PROXY_URI_BASE) != null) {
Expand All @@ -94,6 +104,10 @@ public void init(FilterConfig conf) throws ServletException {
LOG.warn("{} does not appear to be a valid URL", proxyUriBase, e);
}
}
String proxyHostPatternStr = yarnConf.get(YarnConfiguration.PROXY_ADDRESS_PATTERN, "");
if (proxyHostPatternStr != null && !proxyHostPatternStr.isEmpty()) {
proxyHostPattern = Pattern.compile(proxyHostPatternStr);
}
}

if (conf.getInitParameter(AmFilterInitializer.RM_HA_URLS) != null) {
Expand Down Expand Up @@ -125,6 +139,62 @@ protected Set<String> getProxyAddresses() throws ServletException {
}
}

private Boolean filterByProxyAddress(String proxyAddress) throws ServletException {
if (proxyAddress == null) {
return false;
}

if (getProxyAddresses().contains(proxyAddress)) {
return true;
} else if (proxyHostPattern != null) {
try {
String hostName = InetAddress.getByName(proxyAddress).getCanonicalHostName();
return proxyHostPattern.matcher(hostName).matches();
} catch (UnknownHostException e) {
return false;
}
}
return false;
}

/**
* Helper method to load configuration using the hybrid approach.
* Tries ServletContext first, then falls back to loading default config.
*/
private Configuration loadConfiguration(FilterConfig filterConf) throws ServletException {
Configuration config = null;
ServletContext context = filterConf.getServletContext();

if (context != null) {
Object contextAttr = context.getAttribute(HttpServer2.CONF_CONTEXT_ATTRIBUTE); // "hadoop.conf"
if (contextAttr instanceof Configuration) {
config = (Configuration) contextAttr;
LOG.info("Retrieved Configuration from ServletContext attribute '{}'.", HttpServer2.CONF_CONTEXT_ATTRIBUTE);
} else if (contextAttr != null) {
LOG.warn("ServletContext attribute '{}' found, but it is not an instance of Configuration (Actual type: {}).",
HttpServer2.CONF_CONTEXT_ATTRIBUTE, contextAttr.getClass().getName());
}
} else {
LOG.warn("ServletContext is null during AmIpFilter init.");
}

// If config is still null (not found or wrong type in context), load default
if (config == null) {
LOG.warn("Could not retrieve valid Configuration from ServletContext. " +
"Attempting to load default configuration from classpath. " +
"Note: This might not reflect job-specific overrides.");
try {
config = new YarnConfiguration(); // Load defaults (yarn-default, yarn-site, core-site etc.)
LOG.info("Successfully loaded default YarnConfiguration from classpath as fallback.");
} catch (Exception e) {
LOG.error("Failed to load default YarnConfiguration from classpath.", e);
throw new ServletException("Failed to obtain Hadoop/YARN configuration for AmIpFilter.", e);
}
}
return config;
}


@Override
public void destroy() {
//Empty
Expand All @@ -140,7 +210,7 @@ public void doFilter(ServletRequest req, ServletResponse resp,

LOG.debug("Remote address for request is: {}", httpReq.getRemoteAddr());

if (!getProxyAddresses().contains(httpReq.getRemoteAddr())) {
if (!filterByProxyAddress(httpReq.getRemoteAddr())) {
StringBuilder redirect = new StringBuilder(findRedirectUrl());

redirect.append(httpReq.getRequestURI());
Expand Down
0