10000 feat: sql highlighting by Mensh1kov · Pull Request #7467 · scalameta/metals · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

feat: sql highlighting #7467

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jun 9, 2025
Merged

feat: sql highlighting #7467

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
215 changes: 215 additions & 0 deletions metals/src/main/scala/scala/meta/internal/metals/SQLTokenizer.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,215 @@
package scala.meta.internal.metals

import scala.collection.mutable.ListBuffer

sealed trait SQLToken
final case class Keyword(value: String) extends SQLToken
final case class Identifier(value: String) extends SQLToken
final case class Function(value: String) extends SQLToken
final case class Literal(value: String, isClosed: Boolean) extends SQLToken
final case class Number(value: String) extends SQLToken
final case class Operator(value: String) extends SQLToken
final case class Whitespace(value: String) extends SQLToken
final case class Other(value: String) extends SQLToken

object SQLTokenizer {
private val keywords = Set(
// Keywords
"select", "insert", "update", "delete", "from", "where", "group", "by",
"having", "order", "asc", "desc", "limit", "offset", "join", "inner",
"left", "right", "full", "outer", "on", "as", "distinct", "union", "all",
"except", "intersect", "exists", "not", "and", "or", "in", "between",
"like", "is", "null", "case", "when", "then", "if", "else", "end", "create",
"table", "primary", "key", "foreign", "references", "unique", "check",
"default", "drop", "alter", "add", "rename", "truncate", "set", "values",
"into", "index", "view", "with", "recursive", "materialized", "cascade",
"restrict", "window", "partition", "over",

// Types
"int", "integer", "smallint", "bigint", "float", "real", "double",
"boolean", "date", "time", "timestamp", "interval", "text", "json", "jsonb",
"uuid",

// Boolean values
"true", "false",
)

private val operators =
Set("=", "<", ">", "<=", ">=", "<>", "!=", "+", "-", "*", "/", "%")

def tokenize(
text: String,
lastToken: Option[SQLToken] = None,
): List[SQLToken] = {
@scala.annotation.tailrec
def loop(
tokenBufferAndChars: (ListBuffer[SQLToken], List[Char])
): List[SQLToken] = tokenBufferAndChars match {
case WhitespaceExtractor(updatedTokenBuffer, tail) =>
loop(updatedTokenBuffer, tail)
case IdentifierOrKeywordOrFunctionExtractor(updatedTokenBuffer, tail) =>
loop(updatedTokenBuffer, tail)
case NumberExtractor(updatedTokenBuffer, tail) =>
loop(updatedTokenBuffer, tail)
case LiteralExtractor(updatedTokenBuffer, tail) =>
loop(updatedTokenBuffer, tail)
case OperatorExtractor(updatedTokenBuffer, tail) =>
loop(updatedTokenBuffer, tail)
case (tokenBuffer, ch :: tail) =>
tokenBuffer += Other(ch.toString)
loop(tokenBuffer, tail)
case (tokenBuffer, Nil) => tokenBuffer.result()
}

val tokenBuffer = new ListBuffer[SQLToken]()

lastToken match {
case Some(Literal(_, false)) =>
LiteralExtractor
.extract(tokenBuffer, text.toList, isLiteralContinuation = true)
.map(loop)
.getOrElse(Nil)
case _ => loop((tokenBuffer, text.toList))
}
}

private object WhitespaceExtractor {
def unapply(
tokenBufferAndChars: (ListBuffer[SQLToken], List[Char])
): Option[(ListBuffer[SQLToken], List[Char])] = {
val (tokenBuffer, chars) = tokenBufferAndChars
val tokenBuilder = new StringBuilder()

@scala.annotation.tailrec
def loop(chars: List[Char]): Option[(ListBuffer[SQLToken], List[Char])] =
chars match {
case ch :: tail if ch.isWhitespace =>
tokenBuilder.addOne(ch)
loop(tail)
case _ if tokenBuilder.nonEmpty =>
tokenBuffer += Whitespace(tokenBuilder.result())
Some((tokenBuffer, chars))
case _ => None
}

loop(chars)
}
}

private object IdentifierOrKeywordOrFunctionExtractor {
def unapply(
tokenBufferAndChars: (ListBuffer[SQLToken], List[Char])
): Option[(ListBuffer[SQLToken], List[Char])] = {
val (tokenBuffer, chars) = tokenBufferAndChars
val tokenBuilder = new StringBuilder()

@scala.annotation.tailrec
def loop(chars: List[Char]): Option[(ListBuffer[SQLToken], List[Char])] =
chars match {
case ch :: tail if ch.isLetter || ch == '_' =>
tokenBuilder.addOne(ch)
loop(tail)
case '(' :: _ if tokenBuilder.nonEmpty =>
tokenBuffer += Function(tokenBuilder.result())
Some((tokenBuffer, chars))
case _ if tokenBuilder.nonEmpty =>
val token = tokenBuilder.result()
tokenBuffer += (if (keywords.contains(token.toLowerCase()))
Keyword(token)
else Identifier(token))
Some((tokenBuffer, chars))
case _ => None
}

loop(chars)
}
}

private object NumberExtractor {
def unapply(
tokenBufferAndChars: (ListBuffer[SQLToken], List[Char])
): Option[(ListBuffer[SQLToken], List[Char])] = {
val (tokenBuffer, chars) = tokenBufferAndChars
val tokenBuilder = new StringBuilder()

@scala.annotation.tailrec
def loop(
chars: List[Char],
hasDot: Boolean,
): Option[(ListBuffer[SQLToken], List[Char])] =
chars match {
case ch :: tail if ch.isDigit =>
tokenBuilder.addOne(ch)
loop(tail, false)
case '.' :: ch :: tail
if tokenBuilder.nonEmpty && ch.isDigit && !hasDot =>
tokenBuilder.addOne('.')
tokenBuilder.addOne(ch)
loop(tail, true)
case _ if tokenBuilder.nonEmpty =>
tokenBuffer += Number(tokenBuilder.result())
Some((tokenBuffer, chars))
case _ => None
}

loop(chars, false)
}
}

private object LiteralExtractor {
def unapply(
tokenBufferAndChars: (ListBuffer[SQLToken], List[Char])
): Option[(ListBuffer[SQLToken], List[Char])] =
extract(
tokenBuffer = tokenBufferAndChars._1,
chars = tokenBufferAndChars._2,
isLiteralContinuation = false,
)

def extract(
tokenBuffer: ListBuffer[SQLToken],
chars: List[Char],
isLiteralContinuation: Boolean,
): Option[(ListBuffer[SQLToken], List[Char])] = {
val tokenBuilder = new StringBuilder()

@scala.annotation.tailrec
def loop(chars: List[Char]): Option[(ListBuffer[SQLToken], List[Char])] =
chars match {
case '\'' :: tail if tokenBuilder.isEmpty && !isLiteralContinuation =>
tokenBuilder.addOne('\'')
loop(tail)
case '\'' :: tail if tokenBuilder.nonEmpty || isLiteralContinuation =>
tokenBuffer += Literal(tokenBuilder.result() + "'", isClosed = true)
Some((tokenBuffer, tail))
case ch :: tail if tokenBuilder.nonEmpty || isLiteralContinuation =>
tokenBuilder.addOne(ch)
loop(tail)
case _ if tokenBuilder.nonEmpty =>
tokenBuffer += Literal(tokenBuilder.result(), isClosed = false)
Some((tokenBuffer, chars))
case _ => None
}

loop(chars)
}
}

private object OperatorExtractor {
def unapply(
tokenBufferAndChars: (ListBuffer[SQLToken], List[Char])
): Option[(ListBuffer[SQLToken], List[Char])] = {
val (tokenBuffer, chars) = tokenBufferAndChars

chars match {
case ch1 :: ch2 :: tail if operators(ch1.toString + ch2.toString) =>
tokenBuffer += Operator(ch1.toString + ch2.toString)
Some((tokenBuffer, tail))
case ch :: tail if operators(ch.toString) =>
tokenBuffer += Operator(ch.toString)
Some((tokenBuffer, tail))
case _ => None
}
}
}
}
89 changes: 79 additions & 10 deletions metals/src/main/scala/scala/meta/internal/metals/SemanticTokensProvider.scala
View file
Open in desktop
1E0A
Original file line number Diff line number Diff line change
Expand Up @@ -118,21 +118,90 @@ object SemanticTokensProvider {
}
val buffer = ListBuffer.empty[Integer]

var delta = Line(0, 0)
var nodesIterator: List[Node] = nodes
for (tk <- tokens) {
val (toAdd, nodesIterator0, delta0) =
handleToken(tk, nodesIterator, isScala3, delta)
nodesIterator = nodesIterator0
buffer.addAll(
toAdd
)
delta = delta0
tokens.foldLeft((Line(0, 0), nodes, false, Option.empty[SQLToken])) {
case ((delta, nodesIterator, isSQLInterpolator, lastSQLToken), tk) =>
val (
(toAdd, nodesIterator0, delta0),
isSQLInterpolator0,
lastSQLToken0,
) =
handleTokenWithSQLSupport(
tk,
nodesIterator,
isScala3,
delta,
isSQLInterpolator,
lastSQLToken,
)
buffer.addAll(
toAdd
)
(delta0, nodesIterator0, isSQLInterpolator0, lastSQLToken0)
}
buffer.toList
}
}

private def handleTokenWithSQLSupport(
tk: scala.meta.tokens.Token,
nodesIterator: List[Node],
isScala3: Boolean,
delta: Line,
isSQLInterpolator: Boolean,
lastSQLToken: Option[SQLToken],
): ((List[Integer], List[Node], Line), Boolean, Option[SQLToken]) = tk match {
case Token.Interpolation.Id("sql") | Token.Interpolation.Id("fr") =>
(handleToken(tk, nodesIterator, isScala3, delta), true, None)
case Token.Interpolation.Part(value) if isSQLInterpolator =>
val buffer = ListBuffer.empty[Integer]
val sqlTokens = SQLTokenizer.tokenize(value, lastSQLToken)

val (delta0, lastToken0) =
sqlTokens.foldLeft((delta, Option.empty[SQLToken])) {
case ((delta, _), tk) =>
val (toAdd, delta0) = handleSQLToken(tk, delta)
buffer.addAll(toAdd)
(delta0, Some(tk))
}
((buffer.toList, nodesIterator, delta0), isSQLInterpolator, lastToken0)
case Token.Interpolation.End() if isSQLInterpolator =>
(handleToken(tk, nodesIterator, isScala3, delta), false, None)
case _ =>
(
handleToken(tk, nodesIterator, isScala3, delta),
isSQLInterpolator,
lastSQLToken,
)
}

private def handleSQLToken(
tk: SQLToken,
delta: Line,
): (List[Integer], Line) = {
def emitToken(token: String, tokenTypeId: Int) = convertTokensToIntList(
token,
delta,
tokenTypeId,
)

tk match {
case Keyword(value) =>
emitToken(value, getTypeId(SemanticTokenTypes.Keyword))
case Identifier(value) =>
emitToken(value, getTypeId(SemanticTokenTypes.Variable))
case Function(value) =>
emitToken(value, getTypeId(SemanticTokenTypes.Function))
case Number(value) =>
emitToken(value, getTypeId(SemanticTokenTypes.Number))
case Operator(value) =>
emitToken(value, getTypeId(SemanticTokenTypes.Operator))
case Literal(value, _) =>
emitToken(value, getTypeId(SemanticTokenTypes.String))
case Whitespace(value) => emitToken(value, -1)
case Other(value) => emitToken(value, -1)
}
}

case class Line(
val number: Int,
val offset: Int,
Expand Down
2 changes: 1 addition & 1 deletion metals/src/main/scala/scala/meta/internal/metals/StacktraceAnalyzer.scala
View file < 10000 div role="none" class="dropdown-divider">
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ class StacktraceAnalyzer(
for {
symbol <- symbolFromLine(line)
location <- toToplevelSymbol(symbol)
.collectFirst(Function.unlift(findLocationForSymbol))
.collectFirst(scala.Function.unlift(findLocationForSymbol))
} yield trySetLineFromStacktrace(location, line)

}
Expand Down
27 changes: 27 additions & 0 deletions tests/input/src/main/scala/example/SQLQueries.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
package example

class SQLQueries {
implicit class SQLStringContext(sc: StringContext) {
def sql(args: Any*): String = sc.s(args: _*)
}

val createTableQuery = sql"""
CREATE TABLE users (
id INT PRIMARY KEY,
name VARCHAR(100),
age DECIMAL(3, 1),
created_at TIMESTAMP
)
"""

val selectQuery = sql"""
SELECT name, age
FROM users
WHERE age > 30.5
"""

val insertQuery = sql"""
INSERT INTO users (id, name, age, created_at)
VALUES (1, 'John Doe', 25, CURRENT_TIMESTAMP)
"""
}
27 changes: 27 additions & 0 deletions tests/unit/src/test/resources/definition-scala3/example/SQLQueries.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
package example

class SQLQueries/*SQLQueries.scala*/ {
implicit class SQLStringContext/*SQLQueries.scala*/(sc/*SQLQueries.scala*/: StringContext/*StringContext.scala*/) {
def sql/*SQLQueries.scala*/(args/*SQLQueries.scala*/: Any/*Any.scala*/*/*<no symbol>*/): String/*Predef.scala*/ = sc/*SQLQueries.scala*/.s/*StringContext.scala fallback to scala.StringContext#*/(args/*SQLQueries.scala*/: _*/*<no symbol>*/)
}

val createTableQuery/*SQLQueries.scala*/ = sql/*SQLQueries.scala*/"""
CREATE TABLE users (
id INT PRIMARY KEY,
name VARCHAR(100),
age DECIMAL(3, 1),
created_at TIMESTAMP
)
"""

val selectQuery/*SQLQueries.scala*/ = sql/*SQLQueries.scala*/"""
SELECT name, age
FROM users
WHERE age > 30.5
"""

val insertQuery/*SQLQueries.scala*/ = sql/*SQLQueries.scala*/"""
INSERT INTO users (id, name, age, created_at)
VALUES (1, 'John Doe', 25, C 536A URRENT_TIMESTAMP)
"""
}
Loading
Loading
0