1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420
|
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
#
import argparse
import ast
from typing import Generator, List, Optional, Sequence, Set, Tuple
import libcst as cst
import libcst.matchers as m
from libcst import CSTLogicError
from libcst._exceptions import ParserSyntaxError
from libcst.codemod import (
CodemodContext,
ContextAwareTransformer,
ContextAwareVisitor,
VisitorBasedCodemodCommand,
)
def _get_lhs(field: cst.BaseExpression) -> cst.BaseExpression:
if isinstance(field, (cst.Name, cst.Integer)):
return field
elif isinstance(field, (cst.Attribute, cst.Subscript)):
return _get_lhs(field.value)
else:
raise TypeError("Unsupported node type!")
def _find_expr_from_field_name(
fieldname: str, args: Sequence[cst.Arg]
) -> Optional[cst.BaseExpression]:
# Things like "0.name" are invalid expressions in python since
# we can't tell if name is supposed to be the fraction or a name.
# So we do a trick to parse here where we wrap the LHS in parens
# and assume LibCST will handle it.
if "." in fieldname:
ind, exp = fieldname.split(".", 1)
fieldname = f"({ind}).{exp}"
field_expr = cst.parse_expression(fieldname)
lhs = _get_lhs(field_expr)
# Verify we don't have any *args or **kwargs attributes.
if any(arg.star != "" for arg in args):
return None
# Get the index into the arg
index: Optional[int] = None
if isinstance(lhs, cst.Integer):
index = int(lhs.value)
if index < 0 or index >= len(args):
raise CSTLogicError(f"Logic error, arg sequence {index} out of bounds!")
elif isinstance(lhs, cst.Name):
for i, arg in enumerate(args):
kw = arg.keyword
if kw is None:
continue
if kw.value == lhs.value:
index = i
break
if index is None:
raise CSTLogicError(f"Logic error, arg name {lhs.value} out of bounds!")
if index is None:
raise CSTLogicError(
f"Logic error, unsupported fieldname expression {fieldname}!"
)
# Format it!
return field_expr.deep_replace(lhs, args[index].value)
def _get_field(formatstr: str) -> Tuple[str, Optional[str], Optional[str]]:
in_index: int = 0
format_spec: Optional[str] = None
conversion: Optional[str] = None
# Grab any format spec as long as its not an array slice
for pos, char in enumerate(formatstr):
if char == "[":
in_index += 1
elif char == "]":
in_index -= 1
elif char == ":":
if in_index == 0:
formatstr, format_spec = (formatstr[:pos], formatstr[pos + 1 :])
break
# Grab any conversion
if "!" in formatstr:
formatstr, conversion = formatstr.split("!", 1)
# Return it
return formatstr, format_spec, conversion
def _get_tokens( # noqa: C901
string: str,
) -> Generator[Tuple[str, Optional[str], Optional[str], Optional[str]], None, None]:
length = len(string)
prefix: str = ""
format_accum: str = ""
in_brackets: int = 0
seen_escape: bool = False
for pos, char in enumerate(string):
if seen_escape:
# The last character was an escape character, so consume
# this one as well, and then pop out of the escape.
if in_brackets == 0:
prefix += char
else:
format_accum += char
seen_escape = False
continue
# We can't escape inside a f-string/format specifier.
if in_brackets == 0:
# Grab the next character to see if we are an escape sequence.
next_char: Optional[str] = None
if pos < length - 1:
next_char = string[pos + 1]
# If this current character is an escape, we want to
# not react to it, append it to the current accumulator and
# then do the same for the next character.
if char == "{" and next_char == "{":
seen_escape = True
if char == "}" and next_char == "}":
seen_escape = True
# Only if we are not an escape sequence do we consider these
# brackets.
if not seen_escape:
if char == "{":
in_brackets += 1
# We want to add brackets to the format accumulator as
# long as they aren't the outermost, because format
# specs allow {} expansion.
if in_brackets == 1:
continue
if char == "}":
in_brackets -= 1
if in_brackets < 0:
raise ValueError("Stray } in format string!")
if in_brackets == 0:
field_name, format_spec, conversion = _get_field(format_accum)
yield (prefix, field_name, format_spec, conversion)
prefix = ""
format_accum = ""
continue
# Place in the correct accumulator
if in_brackets == 0:
prefix += char
else:
format_accum += char
if in_brackets > 0:
raise ParserSyntaxError(
"Stray { in format string!", lines=[string], raw_line=0, raw_column=0
)
if format_accum:
raise CSTLogicError("Logic error!")
# Yield the last bit of information
yield (prefix, None, None, None)
class StringQuoteGatherer(ContextAwareVisitor):
def __init__(self, context: CodemodContext) -> None:
super().__init__(context)
self.stringends: Set[str] = set()
def visit_SimpleString(self, node: cst.SimpleString) -> None:
self.stringends.add(node.value[-1])
class StripNewlinesTransformer(ContextAwareTransformer):
def leave_ParenthesizedWhitespace(
self,
original_node: cst.ParenthesizedWhitespace,
updated_node: cst.ParenthesizedWhitespace,
) -> cst.SimpleWhitespace:
return cst.SimpleWhitespace(" ")
class SwitchStringQuotesTransformer(ContextAwareTransformer):
def __init__(self, context: CodemodContext, avoid_quote: str) -> None:
super().__init__(context)
if avoid_quote not in {'"', "'"}:
raise ValueError("Must specify either ' or \" single quote to avoid.")
self.avoid_quote: str = avoid_quote
self.replace_quote: str = '"' if avoid_quote == "'" else "'"
def leave_SimpleString(
self, original_node: cst.SimpleString, updated_node: cst.SimpleString
) -> cst.SimpleString:
if self.avoid_quote in updated_node.quote:
# Attempt to swap the value out, verify that the string is still identical
# before and after transformation.
new_quote = updated_node.quote.replace(self.avoid_quote, self.replace_quote)
new_value = (
f"{updated_node.prefix}{new_quote}{updated_node.raw_value}{new_quote}"
)
try:
new_str = ast.literal_eval(new_value)
if updated_node.evaluated_value != new_str:
# This isn't the same!
return updated_node
return updated_node.with_changes(value=new_value)
except Exception:
# Failed to parse string, changing the quoting screwed us up.
pass
# Either failed to parse the new string, or don't need to make changes.
return updated_node
class ConvertFormatStringCommand(VisitorBasedCodemodCommand):
DESCRIPTION: str = "Converts instances of str.format() to f-string."
@staticmethod
def add_args(arg_parser: argparse.ArgumentParser) -> None:
arg_parser.add_argument(
"--allow-strip-comments",
dest="allow_strip_comments",
help=(
"Allow stripping comments inside .format() calls when converting "
+ "to f-strings."
),
action="store_true",
)
arg_parser.add_argument(
"--allow-await",
dest="allow_await",
help=(
"Allow converting expressions inside .format() calls that contain "
+ "an await expression (only compatible with Python 3.7+)."
),
action="store_true",
)
def __init__(
self,
context: CodemodContext,
allow_strip_comments: bool = False,
allow_await: bool = False,
) -> None:
super().__init__(context)
self.allow_strip_comments = allow_strip_comments
self.allow_await = allow_await
def leave_Call( # noqa: C901
self, original_node: cst.Call, updated_node: cst.Call
) -> cst.BaseExpression:
# Lets figure out if this is a "".format() call
extraction = self.extract(
updated_node,
m.Call(
func=m.Attribute(
value=m.SaveMatchedNode(m.SimpleString(), "string"),
attr=m.Name("format"),
)
),
)
if extraction is not None:
fstring: List[cst.BaseFormattedStringContent] = []
inserted_sequence: int = 0
stringnode = cst.ensure_type(extraction["string"], cst.SimpleString)
tokens = _get_tokens(stringnode.raw_value)
for literal_text, field_name, format_spec, conversion in tokens:
if literal_text:
fstring.append(cst.FormattedStringText(literal_text))
if field_name is None:
# This is not a format-specification
continue
# Auto-insert field sequence if it is empty
if field_name == "":
field_name = str(inserted_sequence)
inserted_sequence += 1
# Now, if there is a valid format spec, parse it as a f-string
# as well, since it allows for insertion of parameters just
# like regular f-strings.
format_spec_parts: List[cst.BaseFormattedStringContent] = []
if format_spec is not None and len(format_spec) > 0:
# Parse the format spec out as a series of tokens as well.
format_spec_tokens = _get_tokens(format_spec)
for (
spec_literal_text,
spec_field_name,
spec_format_spec,
spec_conversion,
) in format_spec_tokens:
if spec_format_spec is not None:
# This shouldn't be possible, we don't allow it in the spec!
raise CSTLogicError("Logic error!")
if spec_literal_text:
format_spec_parts.append(
cst.FormattedStringText(spec_literal_text)
)
if spec_field_name is None:
# This is not a format-specification
continue
# Auto-insert field sequence if it is empty
if spec_field_name == "":
spec_field_name = str(inserted_sequence)
inserted_sequence += 1
# Now, convert the spec expression itself.
fstring_expression = self._convert_token_to_fstring_expression(
spec_field_name,
spec_conversion,
updated_node.args,
stringnode,
)
if fstring_expression is None:
return updated_node
format_spec_parts.append(fstring_expression)
# Finally, output the converted value.
fstring_expression = self._convert_token_to_fstring_expression(
field_name, conversion, updated_node.args, stringnode
)
if fstring_expression is None:
return updated_node
# Technically its valid to add the parts even if it is empty, but
# it results in an empty format spec being added which is ugly.
if format_spec_parts:
fstring_expression = fstring_expression.with_changes(
format_spec=format_spec_parts
)
fstring.append(fstring_expression)
# We converted each part, so lets bang together the f-string itself.
return cst.FormattedString(
parts=fstring,
start=f"f{stringnode.prefix}{stringnode.quote}",
end=stringnode.quote,
)
return updated_node
def _convert_token_to_fstring_expression(
self,
field_name: str,
conversion: Optional[str],
arguments: Sequence[cst.Arg],
containing_string: cst.SimpleString,
) -> Optional[cst.FormattedStringExpression]:
expr = _find_expr_from_field_name(field_name, arguments)
if expr is None:
# Most likely they used * expansion in a format.
self.warn(f"Unsupported field_name {field_name} in format() call")
return None
# Verify that we don't have any comments or newlines. Comments aren't
# allowed in f-strings, and newlines need parenthesization. We can
# have formattedstrings inside other formattedstrings, but I chose not
# to doeal with that for now.
if self.findall(expr, m.Comment()) and not self.allow_strip_comments:
# We could strip comments, but this is a formatting change so
# we choose not to for now.
self.warn("Unsupported comment in format() call")
return None
if self.findall(expr, m.FormattedString()):
self.warn("Unsupported f-string in format() call")
return None
if self.findall(expr, m.Await()) and not self.allow_await:
# This is fixed in 3.7 but we don't currently have a flag
# to enable/disable it.
self.warn("Unsupported await in format() call")
return None
# Stripping newlines is effectively a format-only change.
expr = cst.ensure_type(
expr.visit(StripNewlinesTransformer(self.context)),
cst.BaseExpression,
)
# Try our best to swap quotes on any strings that won't fit
expr = cst.ensure_type(
expr.visit(
SwitchStringQuotesTransformer(self.context, containing_string.quote[0])
),
cst.BaseExpression,
)
# Verify that the resulting expression doesn't have a backslash
# in it.
raw_expr_string = self.module.code_for_node(expr)
if "\\" in raw_expr_string:
self.warn("Unsupported backslash in format expression")
return None
# For safety sake, if this is a dict/set or dict/set comprehension,
# wrap it in parens so that it doesn't accidentally create an
# escape.
if (raw_expr_string.startswith("{") or raw_expr_string.endswith("}")) and (
not expr.lpar or not expr.rpar
):
expr = expr.with_changes(lpar=[cst.LeftParen()], rpar=[cst.RightParen()])
# Verify that any strings we insert don't have the same quote
quote_gatherer = StringQuoteGatherer(self.context)
expr.visit(quote_gatherer)
for stringend in quote_gatherer.stringends:
if stringend in containing_string.quote:
self.warn("Cannot embed string with same quote from format() call")
return None
return cst.FormattedStringExpression(expression=expr, conversion=conversion)
|