feat: use dialect when tokenizing (#30614)

superset/sql/parse.py

@@ -276,7 +276,7 @@ class SQLStatement(BaseSQLStatement[exp.Expression]):
         script: str,
         engine: str,
     ) -> list[SQLStatement]:
-        if engine in SQLGLOT_DIALECTS:
+        if dialect := SQLGLOT_DIALECTS.get(engine):
             try:
                 return [
                     cls(ast.sql(), engine, ast)
@@ -297,7 +297,7 @@ class SQLStatement(BaseSQLStatement[exp.Expression]):
         remainder = script
 
         try:
-            tokens = sqlglot.tokenize(script)
+            tokens = sqlglot.tokenize(script, dialect)
         except sqlglot.errors.TokenError as ex:
             raise SupersetParseError(
                 script,

tests/unit_tests/sql_parse_tests.py

@@ -281,7 +281,7 @@ def test_extract_tables_illdefined() -> None:
         extract_tables('SELECT * FROM "tbname')
     assert (
         str(excinfo.value)
-        == "You may have an error in your SQL statement. Unable to parse script"
+        == "You may have an error in your SQL statement. Unable to tokenize script"
     )
 
     # odd edge case that works