1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152
|
from .utils import FlexibleIterator, BaronError
class UnExpectedSpaceToken(BaronError):
pass
PRIORITY_ORDER = (
"IMPORT",
"ENDL",
)
BOTH = (
"SEMICOLON",
"AS",
"IMPORT",
"DOUBLE_STAR",
"DOT",
"LEFT_SQUARE_BRACKET",
"LEFT_PARENTHESIS",
"STAR",
"SLASH",
"PERCENT",
"DOUBLE_SLASH",
"PLUS",
"MINUS",
"AT",
"LEFT_SHIFT",
"RIGHT_SHIFT",
"AMPER",
"CIRCUMFLEX",
"VBAR",
"LESS",
"GREATER",
"EQUAL_EQUAL",
"LESS_EQUAL",
"GREATER_EQUAL",
"NOT_EQUAL",
"IN",
"IS",
"NOT",
"AND",
"OR",
"IF",
"ELSE",
"EQUAL",
"PLUS_EQUAL",
"MINUS_EQUAL",
"STAR_EQUAL",
"AT_EQUAL",
"SLASH_EQUAL",
"PERCENT_EQUAL",
"AMPER_EQUAL",
"VBAR_EQUAL",
"CIRCUMFLEX_EQUAL",
"LEFT_SHIFT_EQUAL",
"RIGHT_SHIFT_EQUAL",
"DOUBLE_STAR_EQUAL",
"DOUBLE_SLASH_EQUAL",
"ENDL",
"COMMA",
"FOR",
"COLON",
"BACKQUOTE",
"RIGHT_ARROW",
"FROM",
)
STRING = (
"STRING",
"RAW_STRING",
"INTERPOLATED_STRING",
"INTERPOLATED_RAW_STRING",
"UNICODE_STRING",
"UNICODE_RAW_STRING",
"BINARY_STRING",
"BINARY_RAW_STRING",
)
GROUP_SPACE_BEFORE = BOTH + (
"RIGHT_PARENTHESIS",
"COMMENT",
) + STRING
GROUP_SPACE_AFTER = BOTH + (
"TILDE",
"RETURN",
"YIELD",
"WITH",
"DEL",
"ASSERT",
"RAISE",
"EXEC",
"GLOBAL",
"NONLOCAL",
"PRINT",
"INDENT",
"WHILE",
"ELIF",
"EXCEPT",
"DEF",
"CLASS",
"LAMBDA",
)
def less_prioritary_than(a, b):
if b not in PRIORITY_ORDER:
return False
if a not in PRIORITY_ORDER:
return True
return PRIORITY_ORDER.index(a) < PRIORITY_ORDER.index(b)
def group(sequence):
return list(group_generator(sequence))
def group_generator(sequence):
iterator = FlexibleIterator(sequence)
while not iterator.end():
current = next(iterator)
if current is None:
return
if current[0] == "SPACE" and iterator.show_next() and iterator.show_next()[0] in GROUP_SPACE_BEFORE:
new_current = next(iterator)
current = (new_current[0], new_current[1], [current])
if current[0] in GROUP_SPACE_AFTER + STRING and\
(iterator.show_next() and iterator.show_next()[0] == "SPACE") and\
(not iterator.show_next(2) or (iterator.show_next(2) and not less_prioritary_than(current[0], iterator.show_next(2)[0]))):
# do not be greedy when you are grouping on strings
if current[0] in STRING and iterator.show_next(2) and iterator.show_next(2)[0] in GROUP_SPACE_BEFORE:
yield current
continue
after_space = next(iterator)
current = (current[0], current[1], current[2] if len(current) > 2 else [], [after_space])
# in case of "def a(): # comment\n pass"
# not really happy about this solution but that avoid a broken release
if current[0] == "COLON" and iterator.show_next() and iterator.show_next()[0] == "COMMENT":
comment = next(iterator)
current = (current[0], current[1], ((current[2]) if len(current) > 2 else []), ((current[3]) if len(current) > 3 else []) + [comment])
yield current
|