val sql = "SELECT id, name FROM users WHERE age > 18" // select // val sql = "CREATE TABLE users (id INT, name STRING)" // create // val sql = "INSERT INTO users VALUES (1, 'Alice')" // insert // val sql = "INSERT INTO xx.table2 SELECT * FROM xx.table1" // insert
val logicalPlan: LogicalPlan = CatalystSqlParser.parsePlan(sql) println(logicalPlan) logicalPlan match { // 解析建表语句 case createTable: CreateTable => println(s"SQL: [$sql] -> 这是一个 CREATE TABLE 语句") // 解析insert语句 case _: InsertIntoTable => println(s"SQL: [$sql] -> 这是一个 INSERT 语句") // 解析血缘 var inputTables = Set[String]() var outputTable: Option[String] = None // 遍历 LogicalPlan 解析血缘 logicalPlan.foreach { case UnresolvedRelation(tableIdentifier) => inputTables += tableIdentifier.quotedString // 解析输入表 case InsertIntoTable(table, _, _, _, _) => table match { case UnresolvedRelation(tableIdentifier) => outputTable = Some(tableIdentifier.quotedString) // 解析输出表 case _ => outputTable = Some(table.toString()) // 其他情况 } case _ => // 其他情况忽略 } println(s"输入表: ${inputTables.mkString(", ")}") println(s"输出表: ${outputTable.getOrElse("无")}") // 解析select语句 case _: Project | _: Filter | _: Aggregate | _: Join | _: LogicalPlan => println(s"SQL: [$sql] -> 这是一个 SELECT 查询语句") case _ => println(s"SQL: [$sql] -> 未知类型") } }
Exception in thread "main" java.lang.IllegalArgumentException: Unable to instantiate SparkSession with Hive support because Hive classes are not found. at org.apache.spark.sql.SparkSession$Builder.enableHiveSupport(SparkSession.scala:869) at com.bigdata.spark.SparkSQLParser$.main(SparkSQLParser.scala:17) at com.bigdata.spark.SparkSQLParser.main(SparkSQLParser.scala)
使用CatalystSqlParser解析会报
1 2 3 4 5 6 7 8 9 10 11
Exception in thread "main" org.apache.spark.sql.catalyst.parser.ParseException: Unsupported SQL statement == SQL == CREATE TABLE users (id INT, name STRING) at org.apache.spark.sql.catalyst.parser.AbstractSqlParser$$anonfun$parsePlan$1.apply(ParseDriver.scala:74) at org.apache.spark.sql.catalyst.parser.AbstractSqlParser$$anonfun$parsePlan$1.apply(ParseDriver.scala:69) at org.apache.spark.sql.catalyst.parser.AbstractSqlParser.parse(ParseDriver.scala:100) at org.apache.spark.sql.catalyst.parser.AbstractSqlParser.parsePlan(ParseDriver.scala:69) at com.bigdata.spark.SparkSQLParser$.main(SparkSQLParser.scala:26) at com.bigdata.spark.SparkSQLParser.main(SparkSQLParser.scala)