fix spark-shell NPE without explicit -usejavacp

Author: jiwen624

repl/src/main/scala/org/apache/spark/repl/Main.scala

@@ -65,13 +65,15 @@ object Main extends Logging {
   // Visible for testing
   private[repl] def doMain(args: Array[String], _interp: SparkILoop): Unit = {
     interp = _interp
-    val jars = Utils
+    val userJars = Utils
       .getLocalUserJarsForShell(conf)
       // Remove file:///, file:// or file:/ scheme if exists for each jar
       .map { x =>
         if (x.startsWith("file:")) new File(new URI(x)).getPath else x
       }
       .mkString(File.pathSeparator)
+    val jvmClasspath = sys.props.getOrElse("java.class.path", "")
+    val jars = if (userJars.nonEmpty) jvmClasspath + File.pathSeparator + userJars else jvmClasspath
     val interpArguments = List(
       "-Yrepl-class-based",
       "-Yrepl-outdir",

repl/src/test/scala/org/apache/spark/repl/ReplSuite.scala

@@ -72,6 +72,37 @@ class ReplSuite extends SparkFunSuite {
   def runInterpreterInPasteMode(master: String, input: String): String =
     runInterpreter(master, ":paste\n" + input + 4.toChar) // 4 is the ascii code of CTRL + D
 
+  test("SPARK-56447: spark-shell REPL initializes without explicit -classpath argument") {
+    // Regression test for SPARK-52587: doMain must include java.class.path in the REPL
+    // classpath even when the caller does not pass -classpath explicitly. Before the fix,
+    // the Scala compiler mirror failed to find `object scala` because the JVM classpath
+    // was not propagated to the REPL settings.
+    val CONF_EXECUTOR_CLASSPATH = "spark.executor.extraClassPath"
+    val oldExecutorClasspath = System.getProperty(CONF_EXECUTOR_CLASSPATH)
+    val classpath = System.getProperty("java.class.path")
+    System.setProperty(CONF_EXECUTOR_CLASSPATH, classpath)
+
+    Main.sparkContext = null
+    Main.sparkSession = null
+    Main.conf.set("spark.master", "local")
+
+    val in = new BufferedReader(new StringReader("spark.version\n"))
+    val out = new StringWriter()
+    // No -classpath arg: doMain must derive the classpath from java.class.path itself.
+    Main.doMain(Array.empty, new SparkILoop(in, new PrintWriter(out)))
+
+    if (oldExecutorClasspath != null) {
+      System.setProperty(CONF_EXECUTOR_CLASSPATH, oldExecutorClasspath)
+    } else {
+      System.clearProperty(CONF_EXECUTOR_CLASSPATH)
+    }
+
+    val output = out.toString
+    assertDoesNotContain("object scala in compiler mirror not found", output)
+    assertDoesNotContain("Failed to initialize compiler", output)
+    assertContains("res0: String =", output)
+  }
+
   def assertContains(message: String, output: String): Unit = {
     val isContain = output.contains(message)
     assert(isContain,