From 8ff472a0b33a3a9bab9e9fb3a509f5aced76d3f0 Mon Sep 17 00:00:00 2001 From: "sue.sylee" Date: Tue, 20 May 2025 15:18:37 +0900 Subject: [PATCH] YARN-11821. Support pattern matching for YARN proxy addresses. --- .../hadoop/yarn/conf/YarnConfiguration.java | 6 ++ .../conf/TestYarnConfigurationFields.java | 1 + .../server/webproxy/amfilter/AmIpFilter.java | 72 ++++++++++++++++++- 3 files changed, 78 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index f02ad15e3dbc3..972803958fc95 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2977,6 +2977,12 @@ public static boolean isAclEnabled(Configuration conf) { public static final String DEFAULT_PROXY_ADDRESS = "0.0.0.0:" + DEFAULT_PROXY_PORT; + /** A regex pattern that matches the proxy's hostname. + * Used in AmIpFilter, if not set, reverse DNS hostname-based filtering will not be performed. + */ + public static final String PROXY_ADDRESS_PATTERN = + PROXY_ADDRESS + ".pattern"; + /** Binding address for the web proxy. */ public static final String PROXY_BIND_HOST = PROXY_PREFIX + "bind-host"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java index b63ed40039102..9302a1a015700 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java @@ -236,6 +236,7 @@ public void initializeMemberVariables() { YarnConfiguration.NM_HEALTH_CHECK_SCRIPT_INTERVAL_MS_TEMPLATE); configurationPropsToSkipCompare.add(YarnConfiguration.NM_AUX_SERVICE_REMOTE_CLASSPATH); configurationPropsToSkipCompare.add(YarnConfiguration.LINUX_CONTAINER_RUNTIME_CLASS_FMT); + configurationPropsToSkipCompare.add(YarnConfiguration.PROXY_ADDRESS_PATTERN); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java index 1b10d225552b5..2650f154476a4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java @@ -20,8 +20,11 @@ import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.HttpServer2; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Time; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.webproxy.ProxyUtils; import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet; import org.slf4j.Logger; @@ -30,6 +33,7 @@ import javax.servlet.Filter; import javax.servlet.FilterChain; import javax.servlet.FilterConfig; +import javax.servlet.ServletContext; import javax.servlet.ServletException; import javax.servlet.ServletRequest; import javax.servlet.ServletResponse; @@ -47,6 +51,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; @Public public class AmIpFilter implements Filter { @@ -69,10 +74,15 @@ public class AmIpFilter implements Filter { private long lastUpdate; @VisibleForTesting Map proxyUriBases; + Pattern proxyHostPattern = null; String rmUrls[] = null; @Override public void init(FilterConfig conf) throws ServletException { + Configuration config = loadConfiguration(conf); + YarnConfiguration yarnConf = (config instanceof YarnConfiguration) ? + (YarnConfiguration) config : new YarnConfiguration(config); + // Maintain for backwards compatibility if (conf.getInitParameter(PROXY_HOST) != null && conf.getInitParameter(PROXY_URI_BASE) != null) { @@ -94,6 +104,10 @@ public void init(FilterConfig conf) throws ServletException { LOG.warn("{} does not appear to be a valid URL", proxyUriBase, e); } } + String proxyHostPatternStr = yarnConf.get(YarnConfiguration.PROXY_ADDRESS_PATTERN, ""); + if (proxyHostPatternStr != null && !proxyHostPatternStr.isEmpty()) { + proxyHostPattern = Pattern.compile(proxyHostPatternStr); + } } if (conf.getInitParameter(AmFilterInitializer.RM_HA_URLS) != null) { @@ -125,6 +139,62 @@ protected Set getProxyAddresses() throws ServletException { } } + private Boolean filterByProxyAddress(String proxyAddress) throws ServletException { + if (proxyAddress == null) { + return false; + } + + if (getProxyAddresses().contains(proxyAddress)) { + return true; + } else if (proxyHostPattern != null) { + try { + String hostName = InetAddress.getByName(proxyAddress).getCanonicalHostName(); + return proxyHostPattern.matcher(hostName).matches(); + } catch (UnknownHostException e) { + return false; + } + } + return false; + } + + /** + * Helper method to load configuration using the hybrid approach. + * Tries ServletContext first, then falls back to loading default config. + */ + private Configuration loadConfiguration(FilterConfig filterConf) throws ServletException { + Configuration config = null; + ServletContext context = filterConf.getServletContext(); + + if (context != null) { + Object contextAttr = context.getAttribute(HttpServer2.CONF_CONTEXT_ATTRIBUTE); // "hadoop.conf" + if (contextAttr instanceof Configuration) { + config = (Configuration) contextAttr; + LOG.info("Retrieved Configuration from ServletContext attribute '{}'.", HttpServer2.CONF_CONTEXT_ATTRIBUTE); + } else if (contextAttr != null) { + LOG.warn("ServletContext attribute '{}' found, but it is not an instance of Configuration (Actual type: {}).", + HttpServer2.CONF_CONTEXT_ATTRIBUTE, contextAttr.getClass().getName()); + } + } else { + LOG.warn("ServletContext is null during AmIpFilter init."); + } + + // If config is still null (not found or wrong type in context), load default + if (config == null) { + LOG.warn("Could not retrieve valid Configuration from ServletContext. " + + "Attempting to load default configuration from classpath. " + + "Note: This might not reflect job-specific overrides."); + try { + config = new YarnConfiguration(); // Load defaults (yarn-default, yarn-site, core-site etc.) + LOG.info("Successfully loaded default YarnConfiguration from classpath as fallback."); + } catch (Exception e) { + LOG.error("Failed to load default YarnConfiguration from classpath.", e); + throw new ServletException("Failed to obtain Hadoop/YARN configuration for AmIpFilter.", e); + } + } + return config; + } + + @Override public void destroy() { //Empty @@ -140,7 +210,7 @@ public void doFilter(ServletRequest req, ServletResponse resp, LOG.debug("Remote address for request is: {}", httpReq.getRemoteAddr()); - if (!getProxyAddresses().contains(httpReq.getRemoteAddr())) { + if (!filterByProxyAddress(httpReq.getRemoteAddr())) { StringBuilder redirect = new StringBuilder(findRedirectUrl()); redirect.append(httpReq.getRequestURI());