HADOOP-12847. hadoop daemonlog should support https and SPNEGO for Ke…

…rberized cluster. (Wei-Chiu Chuang via Yongjun Zhang)
apache · May 27, 2016 · 34cc21f · 34cc21f
1 parent 0a544f8
commit 34cc21f
Show file tree

Hide file tree

Showing 3 changed files with 674 additions and 122 deletions.
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java
@@ -17,64 +17,291 @@
  */
 package org.apache.hadoop.log;
 
-import java.io.*;
-import java.net.*;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.PrintWriter;
+import java.net.URL;
+import java.net.URLConnection;
 import java.util.regex.Pattern;
 
-import javax.servlet.*;
-import javax.servlet.http.*;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLSocketFactory;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Charsets;
-import org.apache.commons.logging.*;
-import org.apache.commons.logging.impl.*;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.impl.Jdk14Logger;
+import org.apache.commons.logging.impl.Log4JLogger;
+import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.http.HttpServer2;
+import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
+import org.apache.hadoop.security.authentication.client.KerberosAuthenticator;
+import org.apache.hadoop.security.ssl.SSLFactory;
 import org.apache.hadoop.util.ServletUtil;
+import org.apache.hadoop.util.Tool;
 
 /**
  * Change log level in runtime.
  */
 @InterfaceStability.Evolving
 public class LogLevel {
   public static final String USAGES = "\nUsage: General options are:\n"
-      + "\t[-getlevel <host:httpPort> <classname>]\n"
-      + "\t[-setlevel <host:httpPort> <classname> <level>]\n";
+      + "\t[-getlevel <host:port> <classname> [-protocol (http|https)]\n"
+      + "\t[-setlevel <host:port> <classname> <level> "
+      + "[-protocol (http|https)]\n";
 
+  public static final String PROTOCOL_HTTP = "http";
+  public static final String PROTOCOL_HTTPS = "https";
   /**
    * A command line implementation
    */
-  public static void main(String[] args) {
-    if (args.length == 3 && "-getlevel".equals(args[0])) {
-      process("http://" + args[1] + "/logLevel?log=" + args[2]);
-      return;
-    }
-    else if (args.length == 4 && "-setlevel".equals(args[0])) {
-      process("http://" + args[1] + "/logLevel?log=" + args[2]
-              + "&level=" + args[3]);
-      return;
-    }
+  public static void main(String[] args) throws Exception {
+    CLI cli = new CLI(new Configuration());
+    System.exit(cli.run(args));
+  }
+
+  /**
+   * Valid command line options.
+   */
+  private enum Operations {
+    GETLEVEL,
+    SETLEVEL,
+    UNKNOWN
+  }
 
+  private static void printUsage() {
     System.err.println(USAGES);
-    System.exit(-1);
   }
 
-  private static void process(String urlstring) {
-    try {
-      URL url = new URL(urlstring);
-      System.out.println("Connecting to " + url);
-      URLConnection connection = url.openConnection();
+  public static boolean isValidProtocol(String protocol) {
+    return ((protocol.equals(PROTOCOL_HTTP) ||
+      protocol.equals(PROTOCOL_HTTPS)));
+  }
+
+  @VisibleForTesting
+  static class CLI extends Configured implements Tool {
+    private Operations operation = Operations.UNKNOWN;
+    private String protocol;
+    private String hostName;
+    private String className;
+    private String level;
+
+    CLI(Configuration conf) {
+      setConf(conf);
+    }
+
+    @Override
+    public int run(String[] args) throws Exception {
+      try {
+        parseArguments(args);
+        sendLogLevelRequest();
+      } catch (HadoopIllegalArgumentException e) {
+        printUsage();
+        throw e;
+      }
+      return 0;
+    }
+
+    /**
+     * Send HTTP/HTTPS request to the daemon.
+     * @throws HadoopIllegalArgumentException if arguments are invalid.
+     * @throws Exception if unable to connect
+     */
+    private void sendLogLevelRequest()
+        throws HadoopIllegalArgumentException, Exception {
+      switch (operation) {
+      case GETLEVEL:
+        doGetLevel();
+        break;
+      case SETLEVEL:
+        doSetLevel();
+        break;
+      default:
+        throw new HadoopIllegalArgumentException(
+          "Expect either -getlevel or -setlevel");
+      }
+    }
+
+    public void parseArguments(String[] args) throws
+        HadoopIllegalArgumentException {
+      if (args.length == 0) {
+        throw new HadoopIllegalArgumentException("No arguments specified");
+      }
+      int nextArgIndex = 0;
+      while (nextArgIndex < args.length) {
+        if (args[nextArgIndex].equals("-getlevel")) {
+          nextArgIndex = parseGetLevelArgs(args, nextArgIndex);
+        } else if (args[nextArgIndex].equals("-setlevel")) {
+          nextArgIndex = parseSetLevelArgs(args, nextArgIndex);
+        } else if (args[nextArgIndex].equals("-protocol")) {
+          nextArgIndex = parseProtocolArgs(args, nextArgIndex);
+        } else {
+          throw new HadoopIllegalArgumentException(
+              "Unexpected argument " + args[nextArgIndex]);
+        }
+      }
+
+      // if operation is never specified in the arguments
+      if (operation == Operations.UNKNOWN) {
+        throw new HadoopIllegalArgumentException(
+            "Must specify either -getlevel or -setlevel");
+      }
+
+      // if protocol is unspecified, set it as http.
+      if (protocol == null) {
+        protocol = PROTOCOL_HTTP;
+      }
+    }
+
+    private int parseGetLevelArgs(String[] args, int index) throws
+        HadoopIllegalArgumentException {
+      // fail if multiple operations are specified in the arguments
+      if (operation != Operations.UNKNOWN) {
+        throw new HadoopIllegalArgumentException(
+            "Redundant -getlevel command");
+      }
+      // check number of arguments is sufficient
+      if (index+2 >= args.length) {
+        throw new HadoopIllegalArgumentException(
+            "-getlevel needs two parameters");
+      }
+      operation = Operations.GETLEVEL;
+      hostName = args[index+1];
+      className = args[index+2];
+      return index+3;
+    }
+
+    private int parseSetLevelArgs(String[] args, int index) throws
+        HadoopIllegalArgumentException {
+      // fail if multiple operations are specified in the arguments
+      if (operation != Operations.UNKNOWN) {
+        throw new HadoopIllegalArgumentException(
+            "Redundant -setlevel command");
+      }
+      // check number of arguments is sufficient
+      if (index+3 >= args.length) {
+        throw new HadoopIllegalArgumentException(
+            "-setlevel needs three parameters");
+      }
+      operation = Operations.SETLEVEL;
+      hostName = args[index+1];
+      className = args[index+2];
+      level = args[index+3];
+      return index+4;
+    }
+
+    private int parseProtocolArgs(String[] args, int index) throws
+        HadoopIllegalArgumentException {
+      // make sure only -protocol is specified
+      if (protocol != null) {
+        throw new HadoopIllegalArgumentException(
+            "Redundant -protocol command");
+      }
+      // check number of arguments is sufficient
+      if (index+1 >= args.length) {
+        throw new HadoopIllegalArgumentException(
+            "-protocol needs one parameter");
+      }
+      // check protocol is valid
+      protocol = args[index+1];
+      if (!isValidProtocol(protocol)) {
+        throw new HadoopIllegalArgumentException(
+            "Invalid protocol: " + protocol);
+      }
+      return index+2;
+    }
+
+    /**
+     * Send HTTP/HTTPS request to get log level.
+     *
+     * @throws HadoopIllegalArgumentException if arguments are invalid.
+     * @throws Exception if unable to connect
+     */
+    private void doGetLevel() throws Exception {
+      process(protocol + "://" + hostName + "/logLevel?log=" + className);
+    }
+
+    /**
+     * Send HTTP/HTTPS request to set log level.
+     *
+     * @throws HadoopIllegalArgumentException if arguments are invalid.
+     * @throws Exception if unable to connect
+     */
+    private void doSetLevel() throws Exception {
+      process(protocol + "://" + hostName + "/logLevel?log=" + className
+          + "&level=" + level);
+    }
+
+    /**
+     * Connect to the URL. Supports HTTP/HTTPS and supports SPNEGO
+     * authentication. It falls back to simple authentication if it fails to
+     * initiate SPNEGO.
+     *
+     * @param url the URL address of the daemon servlet
+     * @return a connected connection
+     * @throws Exception if it can not establish a connection.
+     */
+    private URLConnection connect(URL url) throws Exception {
+      AuthenticatedURL.Token token = new AuthenticatedURL.Token();
+      AuthenticatedURL aUrl;
+      SSLFactory clientSslFactory;
+      URLConnection connection;
+      // If https is chosen, configures SSL client.
+      if (PROTOCOL_HTTPS.equals(url.getProtocol())) {
+        clientSslFactory = new SSLFactory(
+            SSLFactory.Mode.CLIENT, this.getConf());
+        clientSslFactory.init();
+        SSLSocketFactory sslSocketF = clientSslFactory.createSSLSocketFactory();
+
+        aUrl = new AuthenticatedURL(
+            new KerberosAuthenticator(), clientSslFactory);
+        connection = aUrl.openConnection(url, token);
+        HttpsURLConnection httpsConn = (HttpsURLConnection) connection;
+        httpsConn.setSSLSocketFactory(sslSocketF);
+      } else {
+        aUrl = new AuthenticatedURL(new KerberosAuthenticator());
+        connection = aUrl.openConnection(url, token);
+      }
+
       connection.connect();
+      return connection;
+    }
+
+    /**
+     * Configures the client to send HTTP/HTTPS request to the URL.
+     * Supports SPENGO for authentication.
+     * @param urlString URL and query string to the daemon's web UI
+     * @throws Exception if unable to connect
+     */
+    private void process(String urlString) throws Exception {
+      URL url = new URL(urlString);
+      System.out.println("Connecting to " + url);
 
-      BufferedReader in = new BufferedReader(new InputStreamReader(
-          connection.getInputStream(), Charsets.UTF_8));
-      for(String line; (line = in.readLine()) != null; )
+      URLConnection connection = connect(url);
+
+      // read from the servlet
+      BufferedReader in = new BufferedReader(
+          new InputStreamReader(connection.getInputStream(), Charsets.UTF_8));
+      for (String line;;) {
+        line = in.readLine();
+        if (line == null) {
+          break;
+        }
         if (line.startsWith(MARKER)) {
           System.out.println(TAG.matcher(line).replaceAll(""));
         }
+      }
       in.close();
-    } catch (IOException ioe) {
-      System.err.println("" + ioe);
     }
   }
 

diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md b/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
@@ -229,17 +229,37 @@ Commands useful for administrators of a hadoop cluster.
 
 Usage:
 
-    hadoop daemonlog -getlevel <host:httpport> <classname>
-    hadoop daemonlog -setlevel <host:httpport> <classname> <level>
+    hadoop daemonlog -getlevel <host:port> <classname> [-protocol (http|https)]
+    hadoop daemonlog -setlevel <host:port> <classname> <level> [-protocol (http|https)]
 
 | COMMAND\_OPTION | Description |
 |:---- |:---- |
-| `-getlevel` *host:httpport* *classname* | Prints the log level of the log identified by a qualified *classname*, in the daemon running at *host:httpport*. This command internally connects to `http://<host:httpport>/logLevel?log=<classname>` |
-| `-setlevel` *host:httpport* *classname* *level* | Sets the log level of the log identified by a qualified *classname*, in the daemon running at *host:httpport*. This command internally connects to `http://<host:httpport>/logLevel?log=<classname>&level=<level>` |
+| `-getlevel` *host:port* *classname* [-protocol (http|https)] | Prints the log level of the log identified by a qualified *classname*, in the daemon running at *host:port*. The `-protocol` flag specifies the protocol for connection. |
+| `-setlevel` *host:port* *classname* *level* [-protocol (http|https)] | Sets the log level of the log identified by a qualified *classname*, in the daemon running at *host:port*.  The `-protocol` flag specifies the protocol for connection. |
 
-Get/Set the log level for a Log identified by a qualified class name in the daemon.
+Get/Set the log level for a Log identified by a qualified class name in the daemon dynamically.
+By default, the command sends a HTTP request, but this can be overridden by using argument `-protocol https` to send a HTTPS request.
+
+Example:
+
+    $ bin/hadoop daemonlog -setlevel 127.0.0.1:9870 org.apache.hadoop.hdfs.server.namenode.NameNode DEBUG
+    $ bin/hadoop daemonlog -getlevel 127.0.0.1:9871 org.apache.hadoop.hdfs.server.namenode.NameNode DEBUG -protocol https
+
+Note that the setting is not permanent and will be reset when the daemon is restarted.
+This command works by sending a HTTP/HTTPS request to the daemon's internal Jetty servlet, so it supports the following daemons:
+
+* HDFS
+    * name node
+    * secondary name node
+    * data node
+    * journal node
+* YARN
+    * resource manager
+    * node manager
+    * Timeline server
+
+However, the command does not support KMS server, because its web interface is based on Tomcat, which does not support the servlet.
 
-	Example: $ bin/hadoop daemonlog -setlevel 127.0.0.1:9870 org.apache.hadoop.hdfs.server.namenode.NameNode DEBUG
 
 Files
 -----