test for changing the regex

Author: mrmasterplan

sqlparse/lexer.py

@@ -99,14 +99,12 @@ def get_tokens(self, text, encoding=None):
                 text = text.decode(encoding)
             else:
                 try:
-                    text = text.decode("utf-8")
+                    text = text.decode('utf-8')
                 except UnicodeDecodeError:
-                    text = text.decode("unicode-escape")
+                    text = text.decode('unicode-escape')
         else:
-            raise TypeError(
-                "Expected text or file-like object, got {!r}"
-                .format(type(text))
-            )
+            raise TypeError("Expected text or file-like object, got {!r}".
+                            format(type(text)))
 
         iterable = enumerate(text)
         for pos, char in iterable:

tests/test_parse.py

@@ -1,10 +1,11 @@
 """Tests sqlparse.parse()."""
+import re
 from io import StringIO
 
 import pytest
 
 import sqlparse
-from sqlparse import sql, tokens as T
+from sqlparse import sql, tokens as T, keywords
 from sqlparse.lexer import Lexer
 
 
@@ -491,8 +492,7 @@ def test_parenthesis():
                                                     T.Newline,
                                                     T.Punctuation]
 
-
-def test_configurable_syntax():
+def test_configurable_keywords():
     sql = """select * from foo BACON SPAM EGGS;"""
     tokens = sqlparse.parse(sql)[0]
 
@@ -517,6 +517,9 @@ def test_configurable_syntax():
 
     tokens = sqlparse.parse(sql)[0]
 
+    # reset the syntax for later tests.
+    Lexer().default_initialization()
+
     assert list(
         (t.ttype, t.value) for t in tokens if t.ttype not in sqlparse.tokens.Whitespace
     ) == [
@@ -529,5 +532,30 @@ def test_configurable_syntax():
         (sqlparse.tokens.Keyword, "EGGS"),
         (sqlparse.tokens.Punctuation, ";"),
     ]
+
+
+def test_configurable_regex():
+    lex = Lexer()
+    lex.clear()
+
+    my_regex = (
+        re.compile(r"ZORDER\s+BY\b", keywords.FLAGS).match,
+        sqlparse.tokens.Keyword,
+    )
+
+    lex.set_SQL_REGEX(keywords.SQL_REGEX[:38] + [my_regex] + keywords.SQL_REGEX[38:])
+    lex.add_keywords(keywords.KEYWORDS_COMMON)
+    lex.add_keywords(keywords.KEYWORDS_ORACLE)
+    lex.add_keywords(keywords.KEYWORDS_PLPGSQL)
+    lex.add_keywords(keywords.KEYWORDS_HQL)
+    lex.add_keywords(keywords.KEYWORDS_MSACCESS)
+    lex.add_keywords(keywords.KEYWORDS)
+
+    tokens = sqlparse.parse("select * from foo zorder by bar;")[0]
+
     # reset the syntax for later tests.
     Lexer().default_initialization()
+
+    assert list(
+        (t.ttype, t.value) for t in tokens if t.ttype not in sqlparse.tokens.Whitespace
+    )[4] == (sqlparse.tokens.Keyword, "zorder by")