Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.10/site-packages/autopep8.py: 43%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#!/usr/bin/env python
3# Copyright (C) 2010-2011 Hideo Hattori
4# Copyright (C) 2011-2013 Hideo Hattori, Steven Myint
5# Copyright (C) 2013-2016 Hideo Hattori, Steven Myint, Bill Wendling
6#
7# Permission is hereby granted, free of charge, to any person obtaining
8# a copy of this software and associated documentation files (the
9# "Software"), to deal in the Software without restriction, including
10# without limitation the rights to use, copy, modify, merge, publish,
11# distribute, sublicense, and/or sell copies of the Software, and to
12# permit persons to whom the Software is furnished to do so, subject to
13# the following conditions:
14#
15# The above copyright notice and this permission notice shall be
16# included in all copies or substantial portions of the Software.
17#
18# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
21# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
22# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
23# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
24# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
25# SOFTWARE.
27# Copyright (C) 2006-2009 Johann C. Rocholl <johann@rocholl.net>
28# Copyright (C) 2009-2013 Florent Xicluna <florent.xicluna@gmail.com>
29#
30# Permission is hereby granted, free of charge, to any person
31# obtaining a copy of this software and associated documentation files
32# (the "Software"), to deal in the Software without restriction,
33# including without limitation the rights to use, copy, modify, merge,
34# publish, distribute, sublicense, and/or sell copies of the Software,
35# and to permit persons to whom the Software is furnished to do so,
36# subject to the following conditions:
37#
38# The above copyright notice and this permission notice shall be
39# included in all copies or substantial portions of the Software.
40#
41# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
42# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
43# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
44# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
45# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
46# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
47# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
48# SOFTWARE.
50"""Automatically formats Python code to conform to the PEP 8 style guide.
52Fixes that only need be done once can be added by adding a function of the form
53"fix_<code>(source)" to this module. They should return the fixed source code.
54These fixes are picked up by apply_global_fixes().
56Fixes that depend on pycodestyle should be added as methods to FixPEP8. See the
57class documentation for more information.
59"""
61from __future__ import absolute_import
62from __future__ import division
63from __future__ import print_function
64from __future__ import unicode_literals
66import argparse
67import codecs
68import collections
69import copy
70import difflib
71import fnmatch
72import importlib
73import inspect
74import io
75import itertools
76import keyword
77import locale
78import os
79import re
80import signal
81import sys
82import textwrap
83import token
84import tokenize
85import warnings
86import ast
87from configparser import ConfigParser as SafeConfigParser, Error
89import pycodestyle
92__version__ = '2.3.2'
95CR = '\r'
96LF = '\n'
97CRLF = '\r\n'
100PYTHON_SHEBANG_REGEX = re.compile(r'^#!.*\bpython[23]?\b\s*$')
101LAMBDA_REGEX = re.compile(r'([\w.]+)\s=\slambda\s*([)(=\w,\s.]*):')
102COMPARE_NEGATIVE_REGEX = re.compile(r'\b(not)\s+([^][)(}{]+?)\s+(in|is)\s')
103COMPARE_NEGATIVE_REGEX_THROUGH = re.compile(r'\b(not\s+in|is\s+not)\s')
104BARE_EXCEPT_REGEX = re.compile(r'except\s*:')
105STARTSWITH_DEF_REGEX = re.compile(r'^(async\s+def|def)\s.*\):')
106DOCSTRING_START_REGEX = re.compile(r'^u?r?(?P<kind>["\']{3})')
107ENABLE_REGEX = re.compile(r'# *(fmt|autopep8): *on')
108DISABLE_REGEX = re.compile(r'# *(fmt|autopep8): *off')
109ENCODING_MAGIC_COMMENT = re.compile(
110 r'^[ \t\f]*#.*?coding[:=][ \t]*([-_.a-zA-Z0-9]+)'
111)
112COMPARE_TYPE_REGEX = re.compile(
113 r'([=!]=)\s+type(?:\s*\(\s*([^)]*[^ )])\s*\))'
114 r'|\btype(?:\s*\(\s*([^)]*[^ )])\s*\))\s+([=!]=)'
115)
116TYPE_REGEX = re.compile(r'(type\s*\(\s*[^)]*?[^\s)]\s*\))')
118EXIT_CODE_OK = 0
119EXIT_CODE_ERROR = 1
120EXIT_CODE_EXISTS_DIFF = 2
121EXIT_CODE_ARGPARSE_ERROR = 99
123# For generating line shortening candidates.
124SHORTEN_OPERATOR_GROUPS = frozenset([
125 frozenset([',']),
126 frozenset(['%']),
127 frozenset([',', '(', '[', '{']),
128 frozenset(['%', '(', '[', '{']),
129 frozenset([',', '(', '[', '{', '%', '+', '-', '*', '/', '//']),
130 frozenset(['%', '+', '-', '*', '/', '//']),
131])
134DEFAULT_IGNORE = 'E226,E24,W50,W690' # TODO: use pycodestyle.DEFAULT_IGNORE
135DEFAULT_INDENT_SIZE = 4
136# these fixes conflict with each other, if the `--ignore` setting causes both
137# to be enabled, disable both of them
138CONFLICTING_CODES = ('W503', 'W504')
140if sys.platform == 'win32': # pragma: no cover
141 DEFAULT_CONFIG = os.path.expanduser(r'~\.pycodestyle')
142else:
143 DEFAULT_CONFIG = os.path.join(os.getenv('XDG_CONFIG_HOME') or
144 os.path.expanduser('~/.config'),
145 'pycodestyle')
146# fallback, use .pep8
147if not os.path.exists(DEFAULT_CONFIG): # pragma: no cover
148 if sys.platform == 'win32':
149 DEFAULT_CONFIG = os.path.expanduser(r'~\.pep8')
150 else:
151 DEFAULT_CONFIG = os.path.join(os.path.expanduser('~/.config'), 'pep8')
152PROJECT_CONFIG = ('setup.cfg', 'tox.ini', '.pep8', '.flake8')
155MAX_PYTHON_FILE_DETECTION_BYTES = 1024
157IS_SUPPORT_TOKEN_FSTRING = False
158if sys.version_info >= (3, 12): # pgrama: no cover
159 IS_SUPPORT_TOKEN_FSTRING = True
162def _custom_formatwarning(message, category, _, __, line=None):
163 return f"{category.__name__}: {message}\n"
166def open_with_encoding(filename, mode='r', encoding=None, limit_byte_check=-1):
167 """Return opened file with a specific encoding."""
168 if not encoding:
169 encoding = detect_encoding(filename, limit_byte_check=limit_byte_check)
171 return io.open(filename, mode=mode, encoding=encoding,
172 newline='') # Preserve line endings
175def _detect_encoding_from_file(filename: str):
176 try:
177 with open(filename) as input_file:
178 for idx, line in enumerate(input_file):
179 if idx == 0 and line[0] == '\ufeff':
180 return "utf-8-sig"
181 if idx >= 2:
182 break
183 match = ENCODING_MAGIC_COMMENT.search(line)
184 if match:
185 return match.groups()[0]
186 except Exception:
187 pass
188 # Python3's default encoding
189 return 'utf-8'
192def detect_encoding(filename, limit_byte_check=-1):
193 """Return file encoding."""
194 encoding = _detect_encoding_from_file(filename)
195 if encoding == "utf-8-sig":
196 return encoding
197 try:
198 with open_with_encoding(filename, encoding=encoding) as test_file:
199 test_file.read(limit_byte_check)
200 return encoding
201 except (LookupError, SyntaxError, UnicodeDecodeError):
202 return 'latin-1'
205def readlines_from_file(filename):
206 """Return contents of file."""
207 with open_with_encoding(filename) as input_file:
208 return input_file.readlines()
211def extended_blank_lines(logical_line,
212 blank_lines,
213 blank_before,
214 indent_level,
215 previous_logical):
216 """Check for missing blank lines after class declaration."""
217 if previous_logical.startswith(('def ', 'async def ')):
218 if blank_lines and pycodestyle.DOCSTRING_REGEX.match(logical_line):
219 yield (0, 'E303 too many blank lines ({})'.format(blank_lines))
220 elif pycodestyle.DOCSTRING_REGEX.match(previous_logical):
221 # Missing blank line between class docstring and method declaration.
222 if (
223 indent_level and
224 not blank_lines and
225 not blank_before and
226 logical_line.startswith(('def ', 'async def ')) and
227 '(self' in logical_line
228 ):
229 yield (0, 'E301 expected 1 blank line, found 0')
232def continued_indentation(logical_line, tokens, indent_level, hang_closing,
233 indent_char, noqa):
234 """Override pycodestyle's function to provide indentation information."""
235 first_row = tokens[0][2][0]
236 nrows = 1 + tokens[-1][2][0] - first_row
237 if noqa or nrows == 1:
238 return
240 # indent_next tells us whether the next block is indented. Assuming
241 # that it is indented by 4 spaces, then we should not allow 4-space
242 # indents on the final continuation line. In turn, some other
243 # indents are allowed to have an extra 4 spaces.
244 indent_next = logical_line.endswith(':')
246 row = depth = 0
247 valid_hangs = (
248 (DEFAULT_INDENT_SIZE,)
249 if indent_char != '\t' else (DEFAULT_INDENT_SIZE,
250 2 * DEFAULT_INDENT_SIZE)
251 )
253 # Remember how many brackets were opened on each line.
254 parens = [0] * nrows
256 # Relative indents of physical lines.
257 rel_indent = [0] * nrows
259 # For each depth, collect a list of opening rows.
260 open_rows = [[0]]
261 # For each depth, memorize the hanging indentation.
262 hangs = [None]
264 # Visual indents.
265 indent_chances = {}
266 last_indent = tokens[0][2]
267 indent = [last_indent[1]]
269 last_token_multiline = None
270 line = None
271 last_line = ''
272 last_line_begins_with_multiline = False
273 for token_type, text, start, end, line in tokens:
275 newline = row < start[0] - first_row
276 if newline:
277 row = start[0] - first_row
278 newline = (not last_token_multiline and
279 token_type not in (tokenize.NL, tokenize.NEWLINE))
280 last_line_begins_with_multiline = last_token_multiline
282 if newline:
283 # This is the beginning of a continuation line.
284 last_indent = start
286 # Record the initial indent.
287 rel_indent[row] = pycodestyle.expand_indent(line) - indent_level
289 # Identify closing bracket.
290 close_bracket = (token_type == tokenize.OP and text in ']})')
292 # Is the indent relative to an opening bracket line?
293 for open_row in reversed(open_rows[depth]):
294 hang = rel_indent[row] - rel_indent[open_row]
295 hanging_indent = hang in valid_hangs
296 if hanging_indent:
297 break
298 if hangs[depth]:
299 hanging_indent = (hang == hangs[depth])
301 visual_indent = (not close_bracket and hang > 0 and
302 indent_chances.get(start[1]))
304 if close_bracket and indent[depth]:
305 # Closing bracket for visual indent.
306 if start[1] != indent[depth]:
307 yield (start, 'E124 {}'.format(indent[depth]))
308 elif close_bracket and not hang:
309 # closing bracket matches indentation of opening bracket's line
310 if hang_closing:
311 yield (start, 'E133 {}'.format(indent[depth]))
312 elif indent[depth] and start[1] < indent[depth]:
313 if visual_indent is not True:
314 # Visual indent is broken.
315 yield (start, 'E128 {}'.format(indent[depth]))
316 elif (hanging_indent or
317 (indent_next and
318 rel_indent[row] == 2 * DEFAULT_INDENT_SIZE)):
319 # Hanging indent is verified.
320 if close_bracket and not hang_closing:
321 yield (start, 'E123 {}'.format(indent_level +
322 rel_indent[open_row]))
323 hangs[depth] = hang
324 elif visual_indent is True:
325 # Visual indent is verified.
326 indent[depth] = start[1]
327 elif visual_indent in (text, str):
328 # Ignore token lined up with matching one from a previous line.
329 pass
330 else:
331 one_indented = (indent_level + rel_indent[open_row] +
332 DEFAULT_INDENT_SIZE)
333 # Indent is broken.
334 if hang <= 0:
335 error = ('E122', one_indented)
336 elif indent[depth]:
337 error = ('E127', indent[depth])
338 elif not close_bracket and hangs[depth]:
339 error = ('E131', one_indented)
340 elif hang > DEFAULT_INDENT_SIZE:
341 error = ('E126', one_indented)
342 else:
343 hangs[depth] = hang
344 error = ('E121', one_indented)
346 yield (start, '{} {}'.format(*error))
348 # Look for visual indenting.
349 if (
350 parens[row] and
351 token_type not in (tokenize.NL, tokenize.COMMENT) and
352 not indent[depth]
353 ):
354 indent[depth] = start[1]
355 indent_chances[start[1]] = True
356 # Deal with implicit string concatenation.
357 elif (token_type in (tokenize.STRING, tokenize.COMMENT) or
358 text in ('u', 'ur', 'b', 'br')):
359 indent_chances[start[1]] = str
360 # Special case for the "if" statement because len("if (") is equal to
361 # 4.
362 elif not indent_chances and not row and not depth and text == 'if':
363 indent_chances[end[1] + 1] = True
364 elif text == ':' and line[end[1]:].isspace():
365 open_rows[depth].append(row)
367 # Keep track of bracket depth.
368 if token_type == tokenize.OP:
369 if text in '([{':
370 depth += 1
371 indent.append(0)
372 hangs.append(None)
373 if len(open_rows) == depth:
374 open_rows.append([])
375 open_rows[depth].append(row)
376 parens[row] += 1
377 elif text in ')]}' and depth > 0:
378 # Parent indents should not be more than this one.
379 prev_indent = indent.pop() or last_indent[1]
380 hangs.pop()
381 for d in range(depth):
382 if indent[d] > prev_indent:
383 indent[d] = 0
384 for ind in list(indent_chances):
385 if ind >= prev_indent:
386 del indent_chances[ind]
387 del open_rows[depth + 1:]
388 depth -= 1
389 if depth:
390 indent_chances[indent[depth]] = True
391 for idx in range(row, -1, -1):
392 if parens[idx]:
393 parens[idx] -= 1
394 break
395 assert len(indent) == depth + 1
396 if (
397 start[1] not in indent_chances and
398 # This is for purposes of speeding up E121 (GitHub #90).
399 not last_line.rstrip().endswith(',')
400 ):
401 # Allow to line up tokens.
402 indent_chances[start[1]] = text
404 last_token_multiline = (start[0] != end[0])
405 if last_token_multiline:
406 rel_indent[end[0] - first_row] = rel_indent[row]
408 last_line = line
410 if (
411 indent_next and
412 not last_line_begins_with_multiline and
413 pycodestyle.expand_indent(line) == indent_level + DEFAULT_INDENT_SIZE
414 ):
415 pos = (start[0], indent[0] + 4)
416 desired_indent = indent_level + 2 * DEFAULT_INDENT_SIZE
417 if visual_indent:
418 yield (pos, 'E129 {}'.format(desired_indent))
419 else:
420 yield (pos, 'E125 {}'.format(desired_indent))
423# NOTE: need reload with runpy and call twice
424# see: https://github.com/hhatto/autopep8/issues/625
425importlib.reload(pycodestyle)
426del pycodestyle._checks['logical_line'][pycodestyle.continued_indentation]
427pycodestyle.register_check(extended_blank_lines)
428pycodestyle.register_check(continued_indentation)
431class FixPEP8(object):
433 """Fix invalid code.
435 Fixer methods are prefixed "fix_". The _fix_source() method looks for these
436 automatically.
438 The fixer method can take either one or two arguments (in addition to
439 self). The first argument is "result", which is the error information from
440 pycodestyle. The second argument, "logical", is required only for
441 logical-line fixes.
443 The fixer method can return the list of modified lines or None. An empty
444 list would mean that no changes were made. None would mean that only the
445 line reported in the pycodestyle error was modified. Note that the modified
446 line numbers that are returned are indexed at 1. This typically would
447 correspond with the line number reported in the pycodestyle error
448 information.
450 [fixed method list]
451 - e111,e114,e115,e116
452 - e121,e122,e123,e124,e125,e126,e127,e128,e129
453 - e201,e202,e203
454 - e211
455 - e221,e222,e223,e224,e225
456 - e231
457 - e251,e252
458 - e261,e262
459 - e271,e272,e273,e274,e275
460 - e301,e302,e303,e304,e305,e306
461 - e401,e402
462 - e502
463 - e701,e702,e703,e704
464 - e711,e712,e713,e714
465 - e721,e722
466 - e731
467 - w291
468 - w503,504
470 """
472 def __init__(self, filename,
473 options,
474 contents=None,
475 long_line_ignore_cache=None):
476 self.filename = filename
477 if contents is None:
478 self.source = readlines_from_file(filename)
479 else:
480 sio = io.StringIO(contents)
481 self.source = sio.readlines()
482 self.options = options
483 self.indent_word = _get_indentword(''.join(self.source))
484 self.original_source = copy.copy(self.source)
486 # collect imports line
487 self.imports = {}
488 for i, line in enumerate(self.source):
489 if (line.find("import ") == 0 or line.find("from ") == 0) and \
490 line not in self.imports:
491 # collect only import statements that first appeared
492 self.imports[line] = i
494 self.long_line_ignore_cache = (
495 set() if long_line_ignore_cache is None
496 else long_line_ignore_cache)
498 # Many fixers are the same even though pycodestyle categorizes them
499 # differently.
500 self.fix_e115 = self.fix_e112
501 self.fix_e121 = self._fix_reindent
502 self.fix_e122 = self._fix_reindent
503 self.fix_e123 = self._fix_reindent
504 self.fix_e124 = self._fix_reindent
505 self.fix_e126 = self._fix_reindent
506 self.fix_e127 = self._fix_reindent
507 self.fix_e128 = self._fix_reindent
508 self.fix_e129 = self._fix_reindent
509 self.fix_e133 = self.fix_e131
510 self.fix_e202 = self.fix_e201
511 self.fix_e203 = self.fix_e201
512 self.fix_e204 = self.fix_e201
513 self.fix_e211 = self.fix_e201
514 self.fix_e221 = self.fix_e271
515 self.fix_e222 = self.fix_e271
516 self.fix_e223 = self.fix_e271
517 self.fix_e226 = self.fix_e225
518 self.fix_e227 = self.fix_e225
519 self.fix_e228 = self.fix_e225
520 self.fix_e241 = self.fix_e271
521 self.fix_e242 = self.fix_e224
522 self.fix_e252 = self.fix_e225
523 self.fix_e261 = self.fix_e262
524 self.fix_e272 = self.fix_e271
525 self.fix_e273 = self.fix_e271
526 self.fix_e274 = self.fix_e271
527 self.fix_e275 = self.fix_e271
528 self.fix_e306 = self.fix_e301
529 self.fix_e501 = (
530 self.fix_long_line_logically if
531 options and (options.aggressive >= 2 or options.experimental) else
532 self.fix_long_line_physically)
533 self.fix_e703 = self.fix_e702
534 self.fix_w292 = self.fix_w291
535 self.fix_w293 = self.fix_w291
537 def _check_affected_anothers(self, result) -> bool:
538 """Check if the fix affects the number of lines of another remark."""
539 line_index = result['line'] - 1
540 target = self.source[line_index]
541 original_target = self.original_source[line_index]
542 return target != original_target
544 def _fix_source(self, results):
545 try:
546 (logical_start, logical_end) = _find_logical(self.source)
547 logical_support = True
548 except (SyntaxError, tokenize.TokenError): # pragma: no cover
549 logical_support = False
551 completed_lines = set()
552 for result in sorted(results, key=_priority_key):
553 if result['line'] in completed_lines:
554 continue
556 fixed_methodname = 'fix_' + result['id'].lower()
557 if hasattr(self, fixed_methodname):
558 fix = getattr(self, fixed_methodname)
560 line_index = result['line'] - 1
561 original_line = self.source[line_index]
563 is_logical_fix = len(_get_parameters(fix)) > 2
564 if is_logical_fix:
565 logical = None
566 if logical_support:
567 logical = _get_logical(self.source,
568 result,
569 logical_start,
570 logical_end)
571 if logical and set(range(
572 logical[0][0] + 1,
573 logical[1][0] + 1)).intersection(
574 completed_lines):
575 continue
577 if self._check_affected_anothers(result):
578 continue
579 modified_lines = fix(result, logical)
580 else:
581 if self._check_affected_anothers(result):
582 continue
583 modified_lines = fix(result)
585 if modified_lines is None:
586 # Force logical fixes to report what they modified.
587 assert not is_logical_fix
589 if self.source[line_index] == original_line:
590 modified_lines = []
592 if modified_lines:
593 completed_lines.update(modified_lines)
594 elif modified_lines == []: # Empty list means no fix
595 if self.options.verbose >= 2:
596 print(
597 '---> Not fixing {error} on line {line}'.format(
598 error=result['id'], line=result['line']),
599 file=sys.stderr)
600 else: # We assume one-line fix when None.
601 completed_lines.add(result['line'])
602 else:
603 if self.options.verbose >= 3:
604 print(
605 "---> '{}' is not defined.".format(fixed_methodname),
606 file=sys.stderr)
608 info = result['info'].strip()
609 print('---> {}:{}:{}:{}'.format(self.filename,
610 result['line'],
611 result['column'],
612 info),
613 file=sys.stderr)
615 def fix(self):
616 """Return a version of the source code with PEP 8 violations fixed."""
617 pep8_options = {
618 'ignore': self.options.ignore,
619 'select': self.options.select,
620 'max_line_length': self.options.max_line_length,
621 'hang_closing': self.options.hang_closing,
622 }
623 results = _execute_pep8(pep8_options, self.source)
625 if self.options.verbose:
626 progress = {}
627 for r in results:
628 if r['id'] not in progress:
629 progress[r['id']] = set()
630 progress[r['id']].add(r['line'])
631 print('---> {n} issue(s) to fix {progress}'.format(
632 n=len(results), progress=progress), file=sys.stderr)
634 if self.options.line_range:
635 start, end = self.options.line_range
636 results = [r for r in results
637 if start <= r['line'] <= end]
639 self._fix_source(filter_results(source=''.join(self.source),
640 results=results,
641 aggressive=self.options.aggressive))
643 if self.options.line_range:
644 # If number of lines has changed then change line_range.
645 count = sum(sline.count('\n')
646 for sline in self.source[start - 1:end])
647 self.options.line_range[1] = start + count - 1
649 return ''.join(self.source)
651 def _fix_reindent(self, result):
652 """Fix a badly indented line.
654 This is done by adding or removing from its initial indent only.
656 """
657 num_indent_spaces = int(result['info'].split()[1])
658 line_index = result['line'] - 1
659 target = self.source[line_index]
661 self.source[line_index] = ' ' * num_indent_spaces + target.lstrip()
663 def fix_e112(self, result):
664 """Fix under-indented comments."""
665 line_index = result['line'] - 1
666 target = self.source[line_index]
668 if not target.lstrip().startswith('#'):
669 # Don't screw with invalid syntax.
670 return []
672 self.source[line_index] = self.indent_word + target
674 def fix_e113(self, result):
675 """Fix unexpected indentation."""
676 line_index = result['line'] - 1
677 target = self.source[line_index]
678 indent = _get_indentation(target)
679 stripped = target.lstrip()
680 self.source[line_index] = indent[1:] + stripped
682 def fix_e116(self, result):
683 """Fix over-indented comments."""
684 line_index = result['line'] - 1
685 target = self.source[line_index]
687 indent = _get_indentation(target)
688 stripped = target.lstrip()
690 if not stripped.startswith('#'):
691 # Don't screw with invalid syntax.
692 return []
694 self.source[line_index] = indent[1:] + stripped
696 def fix_e117(self, result):
697 """Fix over-indented."""
698 line_index = result['line'] - 1
699 target = self.source[line_index]
701 indent = _get_indentation(target)
702 if indent == '\t':
703 return []
705 stripped = target.lstrip()
707 self.source[line_index] = indent[1:] + stripped
709 def fix_e125(self, result):
710 """Fix indentation undistinguish from the next logical line."""
711 num_indent_spaces = int(result['info'].split()[1])
712 line_index = result['line'] - 1
713 target = self.source[line_index]
715 spaces_to_add = num_indent_spaces - len(_get_indentation(target))
716 indent = len(_get_indentation(target))
717 modified_lines = []
719 while len(_get_indentation(self.source[line_index])) >= indent:
720 self.source[line_index] = (' ' * spaces_to_add +
721 self.source[line_index])
722 modified_lines.append(1 + line_index) # Line indexed at 1.
723 line_index -= 1
725 return modified_lines
727 def fix_e131(self, result):
728 """Fix indentation undistinguish from the next logical line."""
729 num_indent_spaces = int(result['info'].split()[1])
730 line_index = result['line'] - 1
731 target = self.source[line_index]
733 spaces_to_add = num_indent_spaces - len(_get_indentation(target))
735 indent_length = len(_get_indentation(target))
736 spaces_to_add = num_indent_spaces - indent_length
737 if num_indent_spaces == 0 and indent_length == 0:
738 spaces_to_add = 4
740 if spaces_to_add >= 0:
741 self.source[line_index] = (' ' * spaces_to_add +
742 self.source[line_index])
743 else:
744 offset = abs(spaces_to_add)
745 self.source[line_index] = self.source[line_index][offset:]
747 def fix_e201(self, result):
748 """Remove extraneous whitespace."""
749 line_index = result['line'] - 1
750 target = self.source[line_index]
751 offset = result['column'] - 1
753 fixed = fix_whitespace(target,
754 offset=offset,
755 replacement='')
757 self.source[line_index] = fixed
759 def fix_e224(self, result):
760 """Remove extraneous whitespace around operator."""
761 target = self.source[result['line'] - 1]
762 offset = result['column'] - 1
763 fixed = target[:offset] + target[offset:].replace('\t', ' ')
764 self.source[result['line'] - 1] = fixed
766 def fix_e225(self, result):
767 """Fix missing whitespace around operator."""
768 target = self.source[result['line'] - 1]
769 offset = result['column'] - 1
770 fixed = target[:offset] + ' ' + target[offset:]
772 # Only proceed if non-whitespace characters match.
773 # And make sure we don't break the indentation.
774 if (
775 fixed.replace(' ', '') == target.replace(' ', '') and
776 _get_indentation(fixed) == _get_indentation(target)
777 ):
778 self.source[result['line'] - 1] = fixed
779 error_code = result.get('id', 0)
780 try:
781 ts = generate_tokens(fixed)
782 except (SyntaxError, tokenize.TokenError):
783 return
784 if not check_syntax(fixed.lstrip()):
785 return
786 try:
787 _missing_whitespace = (
788 pycodestyle.missing_whitespace_around_operator
789 )
790 except AttributeError:
791 # pycodestyle >= 2.11.0
792 _missing_whitespace = pycodestyle.missing_whitespace
793 errors = list(_missing_whitespace(fixed, ts))
794 for e in reversed(errors):
795 if error_code != e[1].split()[0]:
796 continue
797 offset = e[0][1]
798 fixed = fixed[:offset] + ' ' + fixed[offset:]
799 self.source[result['line'] - 1] = fixed
800 else:
801 return []
803 def fix_e231(self, result):
804 """Add missing whitespace."""
805 line_index = result['line'] - 1
806 target = self.source[line_index]
807 offset = result['column']
808 fixed = target[:offset].rstrip() + ' ' + target[offset:].lstrip()
809 self.source[line_index] = fixed
811 def fix_e251(self, result):
812 """Remove whitespace around parameter '=' sign."""
813 line_index = result['line'] - 1
814 target = self.source[line_index]
816 # This is necessary since pycodestyle sometimes reports columns that
817 # goes past the end of the physical line. This happens in cases like,
818 # foo(bar\n=None)
819 c = min(result['column'] - 1,
820 len(target) - 1)
822 if target[c].strip():
823 fixed = target
824 else:
825 fixed = target[:c].rstrip() + target[c:].lstrip()
827 # There could be an escaped newline
828 #
829 # def foo(a=\
830 # 1)
831 if fixed.endswith(('=\\\n', '=\\\r\n', '=\\\r')):
832 self.source[line_index] = fixed.rstrip('\n\r \t\\')
833 self.source[line_index + 1] = self.source[line_index + 1].lstrip()
834 return [line_index + 1, line_index + 2] # Line indexed at 1
836 self.source[result['line'] - 1] = fixed
838 def fix_e262(self, result):
839 """Fix spacing after inline comment hash."""
840 target = self.source[result['line'] - 1]
841 offset = result['column']
843 code = target[:offset].rstrip(' \t#')
844 comment = target[offset:].lstrip(' \t#')
846 fixed = code + (' # ' + comment if comment.strip() else '\n')
848 self.source[result['line'] - 1] = fixed
850 def fix_e265(self, result):
851 """Fix spacing after block comment hash."""
852 target = self.source[result['line'] - 1]
854 indent = _get_indentation(target)
855 line = target.lstrip(' \t')
856 pos = next((index for index, c in enumerate(line) if c != '#'))
857 hashes = line[:pos]
858 comment = line[pos:].lstrip(' \t')
860 # Ignore special comments, even in the middle of the file.
861 if comment.startswith('!'):
862 return
864 fixed = indent + hashes + (' ' + comment if comment.strip() else '\n')
866 self.source[result['line'] - 1] = fixed
868 def fix_e266(self, result):
869 """Fix too many block comment hashes."""
870 target = self.source[result['line'] - 1]
872 # Leave stylistic outlined blocks alone.
873 if target.strip().endswith('#'):
874 return
876 indentation = _get_indentation(target)
877 fixed = indentation + '# ' + target.lstrip('# \t')
879 self.source[result['line'] - 1] = fixed
881 def fix_e271(self, result):
882 """Fix extraneous whitespace around keywords."""
883 line_index = result['line'] - 1
884 target = self.source[line_index]
885 offset = result['column'] - 1
887 fixed = fix_whitespace(target,
888 offset=offset,
889 replacement=' ')
891 if fixed == target:
892 return []
893 else:
894 self.source[line_index] = fixed
896 def fix_e301(self, result):
897 """Add missing blank line."""
898 cr = '\n'
899 self.source[result['line'] - 1] = cr + self.source[result['line'] - 1]
901 def fix_e302(self, result):
902 """Add missing 2 blank lines."""
903 add_linenum = 2 - int(result['info'].split()[-1])
904 offset = 1
905 if self.source[result['line'] - 2].strip() == "\\":
906 offset = 2
907 cr = '\n' * add_linenum
908 self.source[result['line'] - offset] = (
909 cr + self.source[result['line'] - offset]
910 )
912 def fix_e303(self, result):
913 """Remove extra blank lines."""
914 delete_linenum = int(result['info'].split('(')[1].split(')')[0]) - 2
915 delete_linenum = max(1, delete_linenum)
917 # We need to count because pycodestyle reports an offset line number if
918 # there are comments.
919 cnt = 0
920 line = result['line'] - 2
921 modified_lines = []
922 while cnt < delete_linenum and line >= 0:
923 if not self.source[line].strip():
924 self.source[line] = ''
925 modified_lines.append(1 + line) # Line indexed at 1
926 cnt += 1
927 line -= 1
929 return modified_lines
931 def fix_e304(self, result):
932 """Remove blank line following function decorator."""
933 line = result['line'] - 2
934 if not self.source[line].strip():
935 self.source[line] = ''
937 def fix_e305(self, result):
938 """Add missing 2 blank lines after end of function or class."""
939 add_delete_linenum = 2 - int(result['info'].split()[-1])
940 cnt = 0
941 offset = result['line'] - 2
942 modified_lines = []
943 if add_delete_linenum < 0:
944 # delete cr
945 add_delete_linenum = abs(add_delete_linenum)
946 while cnt < add_delete_linenum and offset >= 0:
947 if not self.source[offset].strip():
948 self.source[offset] = ''
949 modified_lines.append(1 + offset) # Line indexed at 1
950 cnt += 1
951 offset -= 1
952 else:
953 # add cr
954 cr = '\n'
955 # check comment line
956 while True:
957 if offset < 0:
958 break
959 line = self.source[offset].lstrip()
960 if not line:
961 break
962 if line[0] != '#':
963 break
964 offset -= 1
965 offset += 1
966 self.source[offset] = cr + self.source[offset]
967 modified_lines.append(1 + offset) # Line indexed at 1.
968 return modified_lines
970 def fix_e401(self, result):
971 """Put imports on separate lines."""
972 line_index = result['line'] - 1
973 target = self.source[line_index]
974 offset = result['column'] - 1
976 if not target.lstrip().startswith('import'):
977 return []
979 indentation = re.split(pattern=r'\bimport\b',
980 string=target, maxsplit=1)[0]
981 fixed = (target[:offset].rstrip('\t ,') + '\n' +
982 indentation + 'import ' + target[offset:].lstrip('\t ,'))
983 self.source[line_index] = fixed
985 def fix_e402(self, result):
986 (line_index, offset, target) = get_index_offset_contents(result,
987 self.source)
988 for i in range(1, 100):
989 line = "".join(self.source[line_index:line_index+i])
990 try:
991 generate_tokens("".join(line))
992 except (SyntaxError, tokenize.TokenError):
993 continue
994 break
995 if not (target in self.imports and self.imports[target] != line_index):
996 mod_offset = get_module_imports_on_top_of_file(self.source,
997 line_index)
998 self.source[mod_offset] = line + self.source[mod_offset]
999 for offset in range(i):
1000 self.source[line_index+offset] = ''
1002 def fix_long_line_logically(self, result, logical):
1003 """Try to make lines fit within --max-line-length characters."""
1004 if (
1005 not logical or
1006 len(logical[2]) == 1 or
1007 self.source[result['line'] - 1].lstrip().startswith('#')
1008 ):
1009 return self.fix_long_line_physically(result)
1011 start_line_index = logical[0][0]
1012 end_line_index = logical[1][0]
1013 logical_lines = logical[2]
1015 previous_line = get_item(self.source, start_line_index - 1, default='')
1016 next_line = get_item(self.source, end_line_index + 1, default='')
1018 single_line = join_logical_line(''.join(logical_lines))
1020 try:
1021 fixed = self.fix_long_line(
1022 target=single_line,
1023 previous_line=previous_line,
1024 next_line=next_line,
1025 original=''.join(logical_lines))
1026 except (SyntaxError, tokenize.TokenError):
1027 return self.fix_long_line_physically(result)
1029 if fixed:
1030 for line_index in range(start_line_index, end_line_index + 1):
1031 self.source[line_index] = ''
1032 self.source[start_line_index] = fixed
1033 return range(start_line_index + 1, end_line_index + 1)
1035 return []
1037 def fix_long_line_physically(self, result):
1038 """Try to make lines fit within --max-line-length characters."""
1039 line_index = result['line'] - 1
1040 target = self.source[line_index]
1042 previous_line = get_item(self.source, line_index - 1, default='')
1043 next_line = get_item(self.source, line_index + 1, default='')
1045 try:
1046 fixed = self.fix_long_line(
1047 target=target,
1048 previous_line=previous_line,
1049 next_line=next_line,
1050 original=target)
1051 except (SyntaxError, tokenize.TokenError):
1052 return []
1054 if fixed:
1055 self.source[line_index] = fixed
1056 return [line_index + 1]
1058 return []
1060 def fix_long_line(self, target, previous_line,
1061 next_line, original):
1062 cache_entry = (target, previous_line, next_line)
1063 if cache_entry in self.long_line_ignore_cache:
1064 return []
1066 if target.lstrip().startswith('#'):
1067 if self.options.aggressive:
1068 # Wrap commented lines.
1069 return shorten_comment(
1070 line=target,
1071 max_line_length=self.options.max_line_length,
1072 last_comment=not next_line.lstrip().startswith('#'))
1073 return []
1075 fixed = get_fixed_long_line(
1076 target=target,
1077 previous_line=previous_line,
1078 original=original,
1079 indent_word=self.indent_word,
1080 max_line_length=self.options.max_line_length,
1081 aggressive=self.options.aggressive,
1082 experimental=self.options.experimental,
1083 verbose=self.options.verbose)
1085 if fixed and not code_almost_equal(original, fixed):
1086 return fixed
1088 self.long_line_ignore_cache.add(cache_entry)
1089 return None
1091 def fix_e502(self, result):
1092 """Remove extraneous escape of newline."""
1093 (line_index, _, target) = get_index_offset_contents(result,
1094 self.source)
1095 self.source[line_index] = target.rstrip('\n\r \t\\') + '\n'
1097 def fix_e701(self, result):
1098 """Put colon-separated compound statement on separate lines."""
1099 line_index = result['line'] - 1
1100 target = self.source[line_index]
1101 c = result['column']
1103 fixed_source = (target[:c] + '\n' +
1104 _get_indentation(target) + self.indent_word +
1105 target[c:].lstrip('\n\r \t\\'))
1106 self.source[result['line'] - 1] = fixed_source
1107 return [result['line'], result['line'] + 1]
1109 def fix_e702(self, result, logical):
1110 """Put semicolon-separated compound statement on separate lines."""
1111 if not logical:
1112 return [] # pragma: no cover
1113 logical_lines = logical[2]
1115 # Avoid applying this when indented.
1116 # https://docs.python.org/reference/compound_stmts.html
1117 for line in logical_lines:
1118 if (
1119 result['id'] == 'E702'
1120 and ':' in line
1121 and pycodestyle.STARTSWITH_INDENT_STATEMENT_REGEX.match(line)
1122 ):
1123 if self.options.verbose:
1124 print(
1125 '---> avoid fixing {error} with '
1126 'other compound statements'.format(error=result['id']),
1127 file=sys.stderr
1128 )
1129 return []
1131 line_index = result['line'] - 1
1132 target = self.source[line_index]
1134 if target.rstrip().endswith('\\'):
1135 # Normalize '1; \\\n2' into '1; 2'.
1136 self.source[line_index] = target.rstrip('\n \r\t\\')
1137 self.source[line_index + 1] = self.source[line_index + 1].lstrip()
1138 return [line_index + 1, line_index + 2]
1140 if target.rstrip().endswith(';'):
1141 self.source[line_index] = target.rstrip('\n \r\t;') + '\n'
1142 return [line_index + 1]
1144 offset = result['column'] - 1
1145 first = target[:offset].rstrip(';').rstrip()
1146 second = (_get_indentation(logical_lines[0]) +
1147 target[offset:].lstrip(';').lstrip())
1149 # Find inline comment.
1150 inline_comment = None
1151 if target[offset:].lstrip(';').lstrip()[:2] == '# ':
1152 inline_comment = target[offset:].lstrip(';')
1154 if inline_comment:
1155 self.source[line_index] = first + inline_comment
1156 else:
1157 self.source[line_index] = first + '\n' + second
1158 return [line_index + 1]
1160 def fix_e704(self, result):
1161 """Fix multiple statements on one line def"""
1162 (line_index, _, target) = get_index_offset_contents(result,
1163 self.source)
1164 match = STARTSWITH_DEF_REGEX.match(target)
1165 if match:
1166 self.source[line_index] = '{}\n{}{}'.format(
1167 match.group(0),
1168 _get_indentation(target) + self.indent_word,
1169 target[match.end(0):].lstrip())
1171 def fix_e711(self, result):
1172 """Fix comparison with None."""
1173 (line_index, offset, target) = get_index_offset_contents(result,
1174 self.source)
1176 right_offset = offset + 2
1177 if right_offset >= len(target):
1178 return []
1180 left = target[:offset].rstrip()
1181 center = target[offset:right_offset]
1182 right = target[right_offset:].lstrip()
1184 if center.strip() == '==':
1185 new_center = 'is'
1186 elif center.strip() == '!=':
1187 new_center = 'is not'
1188 else:
1189 return []
1191 self.source[line_index] = ' '.join([left, new_center, right])
1193 def fix_e712(self, result):
1194 """Fix (trivial case of) comparison with boolean."""
1195 (line_index, offset, target) = get_index_offset_contents(result,
1196 self.source)
1198 # Handle very easy "not" special cases.
1199 if re.match(r'^\s*if [\w."\'\[\]]+ == False:$', target):
1200 self.source[line_index] = re.sub(r'if ([\w."\'\[\]]+) == False:',
1201 r'if not \1:', target, count=1)
1202 elif re.match(r'^\s*if [\w."\'\[\]]+ != True:$', target):
1203 self.source[line_index] = re.sub(r'if ([\w."\'\[\]]+) != True:',
1204 r'if not \1:', target, count=1)
1205 else:
1206 right_offset = offset + 2
1207 if right_offset >= len(target):
1208 return []
1210 left = target[:offset].rstrip()
1211 center = target[offset:right_offset]
1212 right = target[right_offset:].lstrip()
1214 # Handle simple cases only.
1215 new_right = None
1216 if center.strip() == '==':
1217 if re.match(r'\bTrue\b', right):
1218 new_right = re.sub(r'\bTrue\b *', '', right, count=1)
1219 elif center.strip() == '!=':
1220 if re.match(r'\bFalse\b', right):
1221 new_right = re.sub(r'\bFalse\b *', '', right, count=1)
1223 if new_right is None:
1224 return []
1226 if new_right[0].isalnum():
1227 new_right = ' ' + new_right
1229 self.source[line_index] = left + new_right
1231 def fix_e713(self, result):
1232 """Fix (trivial case of) non-membership check."""
1233 (line_index, offset, target) = get_index_offset_contents(result,
1234 self.source)
1236 # to convert once 'not in' -> 'in'
1237 before_target = target[:offset]
1238 target = target[offset:]
1239 match_notin = COMPARE_NEGATIVE_REGEX_THROUGH.search(target)
1240 notin_pos_start, notin_pos_end = 0, 0
1241 if match_notin:
1242 notin_pos_start = match_notin.start(1)
1243 notin_pos_end = match_notin.end()
1244 target = '{}{} {}'.format(
1245 target[:notin_pos_start], 'in', target[notin_pos_end:])
1247 # fix 'not in'
1248 match = COMPARE_NEGATIVE_REGEX.search(target)
1249 if match:
1250 if match.group(3) == 'in':
1251 pos_start = match.start(1)
1252 new_target = '{5}{0}{1} {2} {3} {4}'.format(
1253 target[:pos_start], match.group(2), match.group(1),
1254 match.group(3), target[match.end():], before_target)
1255 if match_notin:
1256 # revert 'in' -> 'not in'
1257 pos_start = notin_pos_start + offset
1258 pos_end = notin_pos_end + offset - 4 # len('not ')
1259 new_target = '{}{} {}'.format(
1260 new_target[:pos_start], 'not in', new_target[pos_end:])
1261 self.source[line_index] = new_target
1263 def fix_e714(self, result):
1264 """Fix object identity should be 'is not' case."""
1265 (line_index, offset, target) = get_index_offset_contents(result,
1266 self.source)
1268 # to convert once 'is not' -> 'is'
1269 before_target = target[:offset]
1270 target = target[offset:]
1271 match_isnot = COMPARE_NEGATIVE_REGEX_THROUGH.search(target)
1272 isnot_pos_start, isnot_pos_end = 0, 0
1273 if match_isnot:
1274 isnot_pos_start = match_isnot.start(1)
1275 isnot_pos_end = match_isnot.end()
1276 target = '{}{} {}'.format(
1277 target[:isnot_pos_start], 'in', target[isnot_pos_end:])
1279 match = COMPARE_NEGATIVE_REGEX.search(target)
1280 if match:
1281 if match.group(3).startswith('is'):
1282 pos_start = match.start(1)
1283 new_target = '{5}{0}{1} {2} {3} {4}'.format(
1284 target[:pos_start], match.group(2), match.group(3),
1285 match.group(1), target[match.end():], before_target)
1286 if match_isnot:
1287 # revert 'is' -> 'is not'
1288 pos_start = isnot_pos_start + offset
1289 pos_end = isnot_pos_end + offset - 4 # len('not ')
1290 new_target = '{}{} {}'.format(
1291 new_target[:pos_start], 'is not', new_target[pos_end:])
1292 self.source[line_index] = new_target
1294 def fix_e721(self, result):
1295 """fix comparison type"""
1296 (line_index, _, target) = get_index_offset_contents(result,
1297 self.source)
1298 match = COMPARE_TYPE_REGEX.search(target)
1299 if match:
1300 # NOTE: match objects
1301 # * type(a) == type(b) -> (None, None, 'a', '==')
1302 # * str == type(b) -> ('==', 'b', None, None)
1303 # * type(b) == str -> (None, None, 'b', '==')
1304 # * type("") != type(b) -> (None, None, '""', '!=')
1305 start = match.start()
1306 end = match.end()
1307 _prefix = ""
1308 _suffix = ""
1309 first_match_type_obj = match.groups()[1]
1310 if first_match_type_obj is None:
1311 _target_obj = match.groups()[2]
1312 else:
1313 _target_obj = match.groups()[1]
1314 _suffix = target[end:]
1316 isinstance_stmt = " isinstance"
1317 is_not_condition = (
1318 match.groups()[0] == "!=" or match.groups()[3] == "!="
1319 )
1320 if is_not_condition:
1321 isinstance_stmt = " not isinstance"
1323 _type_comp = f"{_target_obj}, {target[:start]}"
1324 indent_match = re.match(r'^\s+', target)
1325 indent = ""
1326 if indent_match:
1327 indent = indent_match.group()
1329 _prefix_tmp = target[:start].split()
1330 if len(_prefix_tmp) >= 1:
1331 _type_comp = f"{_target_obj}, {target[:start]}"
1332 if first_match_type_obj is not None:
1333 _prefix = " ".join(_prefix_tmp[:-1])
1334 _type_comp = f"{_target_obj}, {_prefix_tmp[-1]}"
1335 else:
1336 _prefix = " ".join(_prefix_tmp)
1338 _suffix_tmp = target[end:]
1339 _suffix_type_match = TYPE_REGEX.search(_suffix_tmp)
1340 if _suffix_type_match:
1341 if len(_suffix_tmp.split()) >= 1:
1342 type_match_end = _suffix_type_match.end()
1343 _suffix = _suffix_tmp[type_match_end:]
1344 cmp_b = _suffix_type_match.groups()[0]
1345 _type_comp = f"{_target_obj}, {cmp_b}"
1346 else:
1347 _else_suffix_match = re.match(
1348 r"^\s*([^\s:]+)(.*)$",
1349 _suffix_tmp,
1350 )
1351 if _else_suffix_match:
1352 _else_suffix = _else_suffix_match.group(1)
1353 _else_suffix_other = _else_suffix_match.group(2)
1354 _type_comp = f"{_target_obj}, {_else_suffix}"
1355 _else_suffix_end = _suffix_tmp[_else_suffix_match.end():]
1356 _suffix = f"{_else_suffix_other}{_else_suffix_end}"
1357 # `else` route is not care
1359 fix_line = (
1360 f"{indent}{_prefix}{isinstance_stmt}({_type_comp}){_suffix}"
1361 )
1362 self.source[line_index] = fix_line
1364 def fix_e722(self, result):
1365 """fix bare except"""
1366 (line_index, _, target) = get_index_offset_contents(result,
1367 self.source)
1368 match = BARE_EXCEPT_REGEX.search(target)
1369 if match:
1370 self.source[line_index] = '{}{}{}'.format(
1371 target[:result['column'] - 1], "except BaseException:",
1372 target[match.end():])
1374 def fix_e731(self, result):
1375 """Fix do not assign a lambda expression check."""
1376 (line_index, _, target) = get_index_offset_contents(result,
1377 self.source)
1378 match = LAMBDA_REGEX.search(target)
1379 if match:
1380 end = match.end()
1381 self.source[line_index] = '{}def {}({}): return {}'.format(
1382 target[:match.start(0)], match.group(1), match.group(2),
1383 target[end:].lstrip())
1385 def fix_w291(self, result):
1386 """Remove trailing whitespace."""
1387 fixed_line = self.source[result['line'] - 1].rstrip()
1388 self.source[result['line'] - 1] = fixed_line + '\n'
1390 def fix_w391(self, _):
1391 """Remove trailing blank lines."""
1392 blank_count = 0
1393 for line in reversed(self.source):
1394 line = line.rstrip()
1395 if line:
1396 break
1397 else:
1398 blank_count += 1
1400 original_length = len(self.source)
1401 self.source = self.source[:original_length - blank_count]
1402 return range(1, 1 + original_length)
1404 def fix_w503(self, result):
1405 (line_index, _, target) = get_index_offset_contents(result,
1406 self.source)
1407 one_string_token = target.split()[0]
1408 try:
1409 ts = generate_tokens(one_string_token)
1410 except (SyntaxError, tokenize.TokenError):
1411 return
1412 if not _is_binary_operator(ts[0][0], one_string_token):
1413 return
1414 # find comment
1415 comment_index = 0
1416 found_not_comment_only_line = False
1417 comment_only_linenum = 0
1418 for i in range(5):
1419 # NOTE: try to parse code in 5 times
1420 if (line_index - i) < 0:
1421 break
1422 from_index = line_index - i - 1
1423 if from_index < 0 or len(self.source) <= from_index:
1424 break
1425 to_index = line_index + 1
1426 strip_line = self.source[from_index].lstrip()
1427 if (
1428 not found_not_comment_only_line and
1429 strip_line and strip_line[0] == '#'
1430 ):
1431 comment_only_linenum += 1
1432 continue
1433 found_not_comment_only_line = True
1434 try:
1435 ts = generate_tokens("".join(self.source[from_index:to_index]))
1436 except (SyntaxError, tokenize.TokenError):
1437 continue
1438 newline_count = 0
1439 newline_index = []
1440 for index, t in enumerate(ts):
1441 if t[0] in (tokenize.NEWLINE, tokenize.NL):
1442 newline_index.append(index)
1443 newline_count += 1
1444 if newline_count > 2:
1445 tts = ts[newline_index[-3]:]
1446 else:
1447 tts = ts
1448 old = []
1449 for t in tts:
1450 if t[0] in (tokenize.NEWLINE, tokenize.NL):
1451 newline_count -= 1
1452 if newline_count <= 1:
1453 break
1454 if tokenize.COMMENT == t[0] and old and old[0] != tokenize.NL:
1455 comment_index = old[3][1]
1456 break
1457 old = t
1458 break
1459 i = target.index(one_string_token)
1460 fix_target_line = line_index - 1 - comment_only_linenum
1461 self.source[line_index] = '{}{}'.format(
1462 target[:i], target[i + len(one_string_token):].lstrip())
1463 nl = find_newline(self.source[fix_target_line:line_index])
1464 before_line = self.source[fix_target_line]
1465 bl = before_line.index(nl)
1466 if comment_index:
1467 self.source[fix_target_line] = '{} {} {}'.format(
1468 before_line[:comment_index], one_string_token,
1469 before_line[comment_index + 1:])
1470 else:
1471 if before_line[:bl].endswith("#"):
1472 # special case
1473 # see: https://github.com/hhatto/autopep8/issues/503
1474 self.source[fix_target_line] = '{}{} {}'.format(
1475 before_line[:bl-2], one_string_token, before_line[bl-2:])
1476 else:
1477 self.source[fix_target_line] = '{} {}{}'.format(
1478 before_line[:bl], one_string_token, before_line[bl:])
1480 def fix_w504(self, result):
1481 (line_index, _, target) = get_index_offset_contents(result,
1482 self.source)
1483 # NOTE: is not collect pointed out in pycodestyle==2.4.0
1484 comment_index = 0
1485 operator_position = None # (start_position, end_position)
1486 for i in range(1, 6):
1487 to_index = line_index + i
1488 try:
1489 ts = generate_tokens("".join(self.source[line_index:to_index]))
1490 except (SyntaxError, tokenize.TokenError):
1491 continue
1492 newline_count = 0
1493 newline_index = []
1494 for index, t in enumerate(ts):
1495 if _is_binary_operator(t[0], t[1]):
1496 if t[2][0] == 1 and t[3][0] == 1:
1497 operator_position = (t[2][1], t[3][1])
1498 elif t[0] == tokenize.NAME and t[1] in ("and", "or"):
1499 if t[2][0] == 1 and t[3][0] == 1:
1500 operator_position = (t[2][1], t[3][1])
1501 elif t[0] in (tokenize.NEWLINE, tokenize.NL):
1502 newline_index.append(index)
1503 newline_count += 1
1504 if newline_count > 2:
1505 tts = ts[:newline_index[-3]]
1506 else:
1507 tts = ts
1508 old = []
1509 for t in tts:
1510 if tokenize.COMMENT == t[0] and old:
1511 comment_row, comment_index = old[3]
1512 break
1513 old = t
1514 break
1515 if not operator_position:
1516 return
1517 target_operator = target[operator_position[0]:operator_position[1]]
1519 if comment_index and comment_row == 1:
1520 self.source[line_index] = '{}{}'.format(
1521 target[:operator_position[0]].rstrip(),
1522 target[comment_index:])
1523 else:
1524 self.source[line_index] = '{}{}{}'.format(
1525 target[:operator_position[0]].rstrip(),
1526 target[operator_position[1]:].lstrip(),
1527 target[operator_position[1]:])
1529 next_line = self.source[line_index + 1]
1530 next_line_indent = 0
1531 m = re.match(r'\s*', next_line)
1532 if m:
1533 next_line_indent = m.span()[1]
1534 self.source[line_index + 1] = '{}{} {}'.format(
1535 next_line[:next_line_indent], target_operator,
1536 next_line[next_line_indent:])
1538 def fix_w605(self, result):
1539 (line_index, offset, target) = get_index_offset_contents(result,
1540 self.source)
1541 self.source[line_index] = '{}\\{}'.format(
1542 target[:offset + 1], target[offset + 1:])
1545def get_module_imports_on_top_of_file(source, import_line_index):
1546 """return import or from keyword position
1548 example:
1549 > 0: import sys
1550 1: import os
1551 2:
1552 3: def function():
1553 """
1554 def is_string_literal(line):
1555 if line[0] in 'uUbB':
1556 line = line[1:]
1557 if line and line[0] in 'rR':
1558 line = line[1:]
1559 return line and (line[0] == '"' or line[0] == "'")
1561 def is_future_import(line):
1562 nodes = ast.parse(line)
1563 for n in nodes.body:
1564 if isinstance(n, ast.ImportFrom) and n.module == '__future__':
1565 return True
1566 return False
1568 def has_future_import(source):
1569 offset = 0
1570 line = ''
1571 for _, next_line in source:
1572 for line_part in next_line.strip().splitlines(True):
1573 line = line + line_part
1574 try:
1575 return is_future_import(line), offset
1576 except SyntaxError:
1577 continue
1578 offset += 1
1579 return False, offset
1581 allowed_try_keywords = ('try', 'except', 'else', 'finally')
1582 in_docstring = False
1583 docstring_kind = '"""'
1584 source_stream = iter(enumerate(source))
1585 for cnt, line in source_stream:
1586 if not in_docstring:
1587 m = DOCSTRING_START_REGEX.match(line.lstrip())
1588 if m is not None:
1589 in_docstring = True
1590 docstring_kind = m.group('kind')
1591 remain = line[m.end(): m.endpos].rstrip()
1592 if remain[-3:] == docstring_kind: # one line doc
1593 in_docstring = False
1594 continue
1595 if in_docstring:
1596 if line.rstrip()[-3:] == docstring_kind:
1597 in_docstring = False
1598 continue
1600 if not line.rstrip():
1601 continue
1602 elif line.startswith('#'):
1603 continue
1605 if line.startswith('import '):
1606 if cnt == import_line_index:
1607 continue
1608 return cnt
1609 elif line.startswith('from '):
1610 if cnt == import_line_index:
1611 continue
1612 hit, offset = has_future_import(
1613 itertools.chain([(cnt, line)], source_stream)
1614 )
1615 if hit:
1616 # move to the back
1617 return cnt + offset + 1
1618 return cnt
1619 elif pycodestyle.DUNDER_REGEX.match(line):
1620 return cnt
1621 elif any(line.startswith(kw) for kw in allowed_try_keywords):
1622 continue
1623 elif is_string_literal(line):
1624 return cnt
1625 else:
1626 return cnt
1627 return 0
1630def get_index_offset_contents(result, source):
1631 """Return (line_index, column_offset, line_contents)."""
1632 line_index = result['line'] - 1
1633 return (line_index,
1634 result['column'] - 1,
1635 source[line_index])
1638def get_fixed_long_line(target, previous_line, original,
1639 indent_word=' ', max_line_length=79,
1640 aggressive=0, experimental=False, verbose=False):
1641 """Break up long line and return result.
1643 Do this by generating multiple reformatted candidates and then
1644 ranking the candidates to heuristically select the best option.
1646 """
1647 indent = _get_indentation(target)
1648 source = target[len(indent):]
1649 assert source.lstrip() == source
1650 assert not target.lstrip().startswith('#')
1652 # Check for partial multiline.
1653 tokens = list(generate_tokens(source))
1655 candidates = shorten_line(
1656 tokens, source, indent,
1657 indent_word,
1658 max_line_length,
1659 aggressive=aggressive,
1660 experimental=experimental,
1661 previous_line=previous_line)
1663 # Also sort alphabetically as a tie breaker (for determinism).
1664 candidates = sorted(
1665 sorted(set(candidates).union([target, original])),
1666 key=lambda x: line_shortening_rank(
1667 x,
1668 indent_word,
1669 max_line_length,
1670 experimental=experimental))
1672 if verbose >= 4:
1673 print(('-' * 79 + '\n').join([''] + candidates + ['']),
1674 file=wrap_output(sys.stderr, 'utf-8'))
1676 if candidates:
1677 best_candidate = candidates[0]
1679 # Don't allow things to get longer.
1680 if longest_line_length(best_candidate) > longest_line_length(original):
1681 return None
1683 return best_candidate
1686def longest_line_length(code):
1687 """Return length of longest line."""
1688 if len(code) == 0:
1689 return 0
1690 return max(len(line) for line in code.splitlines())
1693def join_logical_line(logical_line):
1694 """Return single line based on logical line input."""
1695 indentation = _get_indentation(logical_line)
1697 return indentation + untokenize_without_newlines(
1698 generate_tokens(logical_line.lstrip())) + '\n'
1701def untokenize_without_newlines(tokens):
1702 """Return source code based on tokens."""
1703 text = ''
1704 last_row = 0
1705 last_column = -1
1707 for t in tokens:
1708 token_string = t[1]
1709 (start_row, start_column) = t[2]
1710 (end_row, end_column) = t[3]
1712 if start_row > last_row:
1713 last_column = 0
1714 if (
1715 (start_column > last_column or token_string == '\n') and
1716 not text.endswith(' ')
1717 ):
1718 text += ' '
1720 if token_string != '\n':
1721 text += token_string
1723 last_row = end_row
1724 last_column = end_column
1726 return text.rstrip()
1729def _find_logical(source_lines):
1730 # Make a variable which is the index of all the starts of lines.
1731 logical_start = []
1732 logical_end = []
1733 last_newline = True
1734 parens = 0
1735 for t in generate_tokens(''.join(source_lines)):
1736 if t[0] in [tokenize.COMMENT, tokenize.DEDENT,
1737 tokenize.INDENT, tokenize.NL,
1738 tokenize.ENDMARKER]:
1739 continue
1740 if not parens and t[0] in [tokenize.NEWLINE, tokenize.SEMI]:
1741 last_newline = True
1742 logical_end.append((t[3][0] - 1, t[2][1]))
1743 continue
1744 if last_newline and not parens:
1745 logical_start.append((t[2][0] - 1, t[2][1]))
1746 last_newline = False
1747 if t[0] == tokenize.OP:
1748 if t[1] in '([{':
1749 parens += 1
1750 elif t[1] in '}])':
1751 parens -= 1
1752 return (logical_start, logical_end)
1755def _get_logical(source_lines, result, logical_start, logical_end):
1756 """Return the logical line corresponding to the result.
1758 Assumes input is already E702-clean.
1760 """
1761 row = result['line'] - 1
1762 col = result['column'] - 1
1763 ls = None
1764 le = None
1765 for i in range(0, len(logical_start), 1):
1766 assert logical_end
1767 x = logical_end[i]
1768 if x[0] > row or (x[0] == row and x[1] > col):
1769 le = x
1770 ls = logical_start[i]
1771 break
1772 if ls is None:
1773 return None
1774 original = source_lines[ls[0]:le[0] + 1]
1775 return ls, le, original
1778def get_item(items, index, default=None):
1779 if 0 <= index < len(items):
1780 return items[index]
1782 return default
1785def reindent(source, indent_size, leave_tabs=False):
1786 """Reindent all lines."""
1787 reindenter = Reindenter(source, leave_tabs)
1788 return reindenter.run(indent_size)
1791def code_almost_equal(a, b):
1792 """Return True if code is similar.
1794 Ignore whitespace when comparing specific line.
1796 """
1797 split_a = split_and_strip_non_empty_lines(a)
1798 split_b = split_and_strip_non_empty_lines(b)
1800 if len(split_a) != len(split_b):
1801 return False
1803 for (index, _) in enumerate(split_a):
1804 if ''.join(split_a[index].split()) != ''.join(split_b[index].split()):
1805 return False
1807 return True
1810def split_and_strip_non_empty_lines(text):
1811 """Return lines split by newline.
1813 Ignore empty lines.
1815 """
1816 return [line.strip() for line in text.splitlines() if line.strip()]
1819def find_newline(source):
1820 """Return type of newline used in source.
1822 Input is a list of lines.
1824 """
1825 assert not isinstance(source, str)
1827 counter = collections.defaultdict(int)
1828 for line in source:
1829 if line.endswith(CRLF):
1830 counter[CRLF] += 1
1831 elif line.endswith(CR):
1832 counter[CR] += 1
1833 elif line.endswith(LF):
1834 counter[LF] += 1
1836 return (sorted(counter, key=counter.get, reverse=True) or [LF])[0]
1839def _get_indentword(source):
1840 """Return indentation type."""
1841 indent_word = ' ' # Default in case source has no indentation
1842 try:
1843 for t in generate_tokens(source):
1844 if t[0] == token.INDENT:
1845 indent_word = t[1]
1846 break
1847 except (SyntaxError, tokenize.TokenError):
1848 pass
1849 return indent_word
1852def _get_indentation(line):
1853 """Return leading whitespace."""
1854 if line.strip():
1855 non_whitespace_index = len(line) - len(line.lstrip())
1856 return line[:non_whitespace_index]
1858 return ''
1861def get_diff_text(old, new, filename):
1862 """Return text of unified diff between old and new."""
1863 newline = '\n'
1864 diff = difflib.unified_diff(
1865 old, new,
1866 'original/' + filename,
1867 'fixed/' + filename,
1868 lineterm=newline)
1870 text = ''
1871 for line in diff:
1872 text += line
1874 # Work around missing newline (http://bugs.python.org/issue2142).
1875 if text and not line.endswith(newline):
1876 text += newline + r'\ No newline at end of file' + newline
1878 return text
1881def _priority_key(pep8_result):
1882 """Key for sorting PEP8 results.
1884 Global fixes should be done first. This is important for things like
1885 indentation.
1887 """
1888 priority = [
1889 # Fix multiline colon-based before semicolon based.
1890 'e701',
1891 # Break multiline statements early.
1892 'e702',
1893 # Things that make lines longer.
1894 'e225', 'e231',
1895 # Remove extraneous whitespace before breaking lines.
1896 'e201',
1897 # Shorten whitespace in comment before resorting to wrapping.
1898 'e262'
1899 ]
1900 middle_index = 10000
1901 lowest_priority = [
1902 # We need to shorten lines last since the logical fixer can get in a
1903 # loop, which causes us to exit early.
1904 'e501',
1905 ]
1906 key = pep8_result['id'].lower()
1907 try:
1908 return priority.index(key)
1909 except ValueError:
1910 try:
1911 return middle_index + lowest_priority.index(key) + 1
1912 except ValueError:
1913 return middle_index
1916def shorten_line(tokens, source, indentation, indent_word, max_line_length,
1917 aggressive=0, experimental=False, previous_line=''):
1918 """Separate line at OPERATOR.
1920 Multiple candidates will be yielded.
1922 """
1923 for candidate in _shorten_line(tokens=tokens,
1924 source=source,
1925 indentation=indentation,
1926 indent_word=indent_word,
1927 aggressive=aggressive,
1928 previous_line=previous_line):
1929 yield candidate
1931 if aggressive:
1932 for key_token_strings in SHORTEN_OPERATOR_GROUPS:
1933 shortened = _shorten_line_at_tokens(
1934 tokens=tokens,
1935 source=source,
1936 indentation=indentation,
1937 indent_word=indent_word,
1938 key_token_strings=key_token_strings,
1939 aggressive=aggressive)
1941 if shortened is not None and shortened != source:
1942 yield shortened
1944 if experimental:
1945 for shortened in _shorten_line_at_tokens_new(
1946 tokens=tokens,
1947 source=source,
1948 indentation=indentation,
1949 max_line_length=max_line_length):
1951 yield shortened
1954def _shorten_line(tokens, source, indentation, indent_word,
1955 aggressive=0, previous_line=''):
1956 """Separate line at OPERATOR.
1958 The input is expected to be free of newlines except for inside multiline
1959 strings and at the end.
1961 Multiple candidates will be yielded.
1963 """
1964 in_string = False
1965 for (token_type,
1966 token_string,
1967 start_offset,
1968 end_offset) in token_offsets(tokens):
1970 if IS_SUPPORT_TOKEN_FSTRING:
1971 if token_type == tokenize.FSTRING_START:
1972 in_string = True
1973 elif token_type == tokenize.FSTRING_END:
1974 in_string = False
1975 if in_string:
1976 continue
1978 if (
1979 token_type == tokenize.COMMENT and
1980 not is_probably_part_of_multiline(previous_line) and
1981 not is_probably_part_of_multiline(source) and
1982 not source[start_offset + 1:].strip().lower().startswith(
1983 ('noqa', 'pragma:', 'pylint:'))
1984 ):
1985 # Move inline comments to previous line.
1986 first = source[:start_offset]
1987 second = source[start_offset:]
1988 yield (indentation + second.strip() + '\n' +
1989 indentation + first.strip() + '\n')
1990 elif token_type == token.OP and token_string != '=':
1991 # Don't break on '=' after keyword as this violates PEP 8.
1993 assert token_type != token.INDENT
1995 first = source[:end_offset]
1997 second_indent = indentation
1998 if (first.rstrip().endswith('(') and
1999 source[end_offset:].lstrip().startswith(')')):
2000 pass
2001 elif first.rstrip().endswith('('):
2002 second_indent += indent_word
2003 elif '(' in first:
2004 second_indent += ' ' * (1 + first.find('('))
2005 else:
2006 second_indent += indent_word
2008 second = (second_indent + source[end_offset:].lstrip())
2009 if (
2010 not second.strip() or
2011 second.lstrip().startswith('#')
2012 ):
2013 continue
2015 # Do not begin a line with a comma
2016 if second.lstrip().startswith(','):
2017 continue
2018 # Do end a line with a dot
2019 if first.rstrip().endswith('.'):
2020 continue
2021 if token_string in '+-*/':
2022 fixed = first + ' \\' + '\n' + second
2023 else:
2024 fixed = first + '\n' + second
2026 # Only fix if syntax is okay.
2027 if check_syntax(normalize_multiline(fixed)
2028 if aggressive else fixed):
2029 yield indentation + fixed
2032def _is_binary_operator(token_type, text):
2033 return ((token_type == tokenize.OP or text in ['and', 'or']) and
2034 text not in '()[]{},:.;@=%~')
2037# A convenient way to handle tokens.
2038Token = collections.namedtuple('Token', ['token_type', 'token_string',
2039 'spos', 'epos', 'line'])
2042class ReformattedLines(object):
2044 """The reflowed lines of atoms.
2046 Each part of the line is represented as an "atom." They can be moved
2047 around when need be to get the optimal formatting.
2049 """
2051 ###########################################################################
2052 # Private Classes
2054 class _Indent(object):
2056 """Represent an indentation in the atom stream."""
2058 def __init__(self, indent_amt):
2059 self._indent_amt = indent_amt
2061 def emit(self):
2062 return ' ' * self._indent_amt
2064 @property
2065 def size(self):
2066 return self._indent_amt
2068 class _Space(object):
2070 """Represent a space in the atom stream."""
2072 def emit(self):
2073 return ' '
2075 @property
2076 def size(self):
2077 return 1
2079 class _LineBreak(object):
2081 """Represent a line break in the atom stream."""
2083 def emit(self):
2084 return '\n'
2086 @property
2087 def size(self):
2088 return 0
2090 def __init__(self, max_line_length):
2091 self._max_line_length = max_line_length
2092 self._lines = []
2093 self._bracket_depth = 0
2094 self._prev_item = None
2095 self._prev_prev_item = None
2096 self._in_fstring = False
2098 def __repr__(self):
2099 return self.emit()
2101 ###########################################################################
2102 # Public Methods
2104 def add(self, obj, indent_amt, break_after_open_bracket):
2105 if isinstance(obj, Atom):
2106 self._add_item(obj, indent_amt)
2107 return
2109 self._add_container(obj, indent_amt, break_after_open_bracket)
2111 def add_comment(self, item):
2112 num_spaces = 2
2113 if len(self._lines) > 1:
2114 if isinstance(self._lines[-1], self._Space):
2115 num_spaces -= 1
2116 if len(self._lines) > 2:
2117 if isinstance(self._lines[-2], self._Space):
2118 num_spaces -= 1
2120 while num_spaces > 0:
2121 self._lines.append(self._Space())
2122 num_spaces -= 1
2123 self._lines.append(item)
2125 def add_indent(self, indent_amt):
2126 self._lines.append(self._Indent(indent_amt))
2128 def add_line_break(self, indent):
2129 self._lines.append(self._LineBreak())
2130 self.add_indent(len(indent))
2132 def add_line_break_at(self, index, indent_amt):
2133 self._lines.insert(index, self._LineBreak())
2134 self._lines.insert(index + 1, self._Indent(indent_amt))
2136 def add_space_if_needed(self, curr_text, equal=False):
2137 if (
2138 not self._lines or isinstance(
2139 self._lines[-1], (self._LineBreak, self._Indent, self._Space))
2140 ):
2141 return
2143 prev_text = str(self._prev_item)
2144 prev_prev_text = (
2145 str(self._prev_prev_item) if self._prev_prev_item else '')
2147 if (
2148 # The previous item was a keyword or identifier and the current
2149 # item isn't an operator that doesn't require a space.
2150 ((self._prev_item.is_keyword or self._prev_item.is_string or
2151 self._prev_item.is_name or self._prev_item.is_number) and
2152 (curr_text[0] not in '([{.,:}])' or
2153 (curr_text[0] == '=' and equal))) or
2155 # Don't place spaces around a '.', unless it's in an 'import'
2156 # statement.
2157 ((prev_prev_text != 'from' and prev_text[-1] != '.' and
2158 curr_text != 'import') and
2160 # Don't place a space before a colon.
2161 curr_text[0] != ':' and
2163 # Don't split up ending brackets by spaces.
2164 ((prev_text[-1] in '}])' and curr_text[0] not in '.,}])') or
2166 # Put a space after a colon or comma.
2167 prev_text[-1] in ':,' or
2169 # Put space around '=' if asked to.
2170 (equal and prev_text == '=') or
2172 # Put spaces around non-unary arithmetic operators.
2173 ((self._prev_prev_item and
2174 (prev_text not in '+-' and
2175 (self._prev_prev_item.is_name or
2176 self._prev_prev_item.is_number or
2177 self._prev_prev_item.is_string)) and
2178 prev_text in ('+', '-', '%', '*', '/', '//', '**', 'in')))))
2179 ):
2180 self._lines.append(self._Space())
2182 def previous_item(self):
2183 """Return the previous non-whitespace item."""
2184 return self._prev_item
2186 def fits_on_current_line(self, item_extent):
2187 return self.current_size() + item_extent <= self._max_line_length
2189 def current_size(self):
2190 """The size of the current line minus the indentation."""
2191 size = 0
2192 for item in reversed(self._lines):
2193 size += item.size
2194 if isinstance(item, self._LineBreak):
2195 break
2197 return size
2199 def line_empty(self):
2200 return (self._lines and
2201 isinstance(self._lines[-1],
2202 (self._LineBreak, self._Indent)))
2204 def emit(self):
2205 string = ''
2206 for item in self._lines:
2207 if isinstance(item, self._LineBreak):
2208 string = string.rstrip()
2209 string += item.emit()
2211 return string.rstrip() + '\n'
2213 ###########################################################################
2214 # Private Methods
2216 def _add_item(self, item, indent_amt):
2217 """Add an item to the line.
2219 Reflow the line to get the best formatting after the item is
2220 inserted. The bracket depth indicates if the item is being
2221 inserted inside of a container or not.
2223 """
2224 if item.is_fstring_start:
2225 self._in_fstring = True
2226 elif self._prev_item and self._prev_item.is_fstring_end:
2227 self._in_fstring = False
2229 if self._prev_item and self._prev_item.is_string and item.is_string:
2230 # Place consecutive string literals on separate lines.
2231 self._lines.append(self._LineBreak())
2232 self._lines.append(self._Indent(indent_amt))
2234 item_text = str(item)
2235 if self._lines and self._bracket_depth:
2236 # Adding the item into a container.
2237 self._prevent_default_initializer_splitting(item, indent_amt)
2239 if item_text in '.,)]}':
2240 self._split_after_delimiter(item, indent_amt)
2242 elif self._lines and not self.line_empty():
2243 # Adding the item outside of a container.
2244 if self.fits_on_current_line(len(item_text)):
2245 self._enforce_space(item)
2247 else:
2248 # Line break for the new item.
2249 self._lines.append(self._LineBreak())
2250 self._lines.append(self._Indent(indent_amt))
2252 self._lines.append(item)
2253 self._prev_item, self._prev_prev_item = item, self._prev_item
2255 if item_text in '([{' and not self._in_fstring:
2256 self._bracket_depth += 1
2258 elif item_text in '}])' and not self._in_fstring:
2259 self._bracket_depth -= 1
2260 assert self._bracket_depth >= 0
2262 def _add_container(self, container, indent_amt, break_after_open_bracket):
2263 actual_indent = indent_amt + 1
2265 if (
2266 str(self._prev_item) != '=' and
2267 not self.line_empty() and
2268 not self.fits_on_current_line(
2269 container.size + self._bracket_depth + 2)
2270 ):
2272 if str(container)[0] == '(' and self._prev_item.is_name:
2273 # Don't split before the opening bracket of a call.
2274 break_after_open_bracket = True
2275 actual_indent = indent_amt + 4
2276 elif (
2277 break_after_open_bracket or
2278 str(self._prev_item) not in '([{'
2279 ):
2280 # If the container doesn't fit on the current line and the
2281 # current line isn't empty, place the container on the next
2282 # line.
2283 self._lines.append(self._LineBreak())
2284 self._lines.append(self._Indent(indent_amt))
2285 break_after_open_bracket = False
2286 else:
2287 actual_indent = self.current_size() + 1
2288 break_after_open_bracket = False
2290 if isinstance(container, (ListComprehension, IfExpression)):
2291 actual_indent = indent_amt
2293 # Increase the continued indentation only if recursing on a
2294 # container.
2295 container.reflow(self, ' ' * actual_indent,
2296 break_after_open_bracket=break_after_open_bracket)
2298 def _prevent_default_initializer_splitting(self, item, indent_amt):
2299 """Prevent splitting between a default initializer.
2301 When there is a default initializer, it's best to keep it all on
2302 the same line. It's nicer and more readable, even if it goes
2303 over the maximum allowable line length. This goes back along the
2304 current line to determine if we have a default initializer, and,
2305 if so, to remove extraneous whitespaces and add a line
2306 break/indent before it if needed.
2308 """
2309 if str(item) == '=':
2310 # This is the assignment in the initializer. Just remove spaces for
2311 # now.
2312 self._delete_whitespace()
2313 return
2315 if (not self._prev_item or not self._prev_prev_item or
2316 str(self._prev_item) != '='):
2317 return
2319 self._delete_whitespace()
2320 prev_prev_index = self._lines.index(self._prev_prev_item)
2322 if (
2323 isinstance(self._lines[prev_prev_index - 1], self._Indent) or
2324 self.fits_on_current_line(item.size + 1)
2325 ):
2326 # The default initializer is already the only item on this line.
2327 # Don't insert a newline here.
2328 return
2330 # Replace the space with a newline/indent combo.
2331 if isinstance(self._lines[prev_prev_index - 1], self._Space):
2332 del self._lines[prev_prev_index - 1]
2334 self.add_line_break_at(self._lines.index(self._prev_prev_item),
2335 indent_amt)
2337 def _split_after_delimiter(self, item, indent_amt):
2338 """Split the line only after a delimiter."""
2339 self._delete_whitespace()
2341 if self.fits_on_current_line(item.size):
2342 return
2344 last_space = None
2345 for current_item in reversed(self._lines):
2346 if (
2347 last_space and
2348 (not isinstance(current_item, Atom) or
2349 not current_item.is_colon)
2350 ):
2351 break
2352 else:
2353 last_space = None
2354 if isinstance(current_item, self._Space):
2355 last_space = current_item
2356 if isinstance(current_item, (self._LineBreak, self._Indent)):
2357 return
2359 if not last_space:
2360 return
2362 self.add_line_break_at(self._lines.index(last_space), indent_amt)
2364 def _enforce_space(self, item):
2365 """Enforce a space in certain situations.
2367 There are cases where we will want a space where normally we
2368 wouldn't put one. This just enforces the addition of a space.
2370 """
2371 if isinstance(self._lines[-1],
2372 (self._Space, self._LineBreak, self._Indent)):
2373 return
2375 if not self._prev_item:
2376 return
2378 item_text = str(item)
2379 prev_text = str(self._prev_item)
2381 # Prefer a space around a '.' in an import statement, and between the
2382 # 'import' and '('.
2383 if (
2384 (item_text == '.' and prev_text == 'from') or
2385 (item_text == 'import' and prev_text == '.') or
2386 (item_text == '(' and prev_text == 'import')
2387 ):
2388 self._lines.append(self._Space())
2390 def _delete_whitespace(self):
2391 """Delete all whitespace from the end of the line."""
2392 while isinstance(self._lines[-1], (self._Space, self._LineBreak,
2393 self._Indent)):
2394 del self._lines[-1]
2397class Atom(object):
2399 """The smallest unbreakable unit that can be reflowed."""
2401 def __init__(self, atom):
2402 self._atom = atom
2404 def __repr__(self):
2405 return self._atom.token_string
2407 def __len__(self):
2408 return self.size
2410 def reflow(
2411 self, reflowed_lines, continued_indent, extent,
2412 break_after_open_bracket=False,
2413 is_list_comp_or_if_expr=False,
2414 next_is_dot=False
2415 ):
2416 if self._atom.token_type == tokenize.COMMENT:
2417 reflowed_lines.add_comment(self)
2418 return
2420 total_size = extent if extent else self.size
2422 if self._atom.token_string not in ',:([{}])':
2423 # Some atoms will need an extra 1-sized space token after them.
2424 total_size += 1
2426 prev_item = reflowed_lines.previous_item()
2427 if (
2428 not is_list_comp_or_if_expr and
2429 not reflowed_lines.fits_on_current_line(total_size) and
2430 not (next_is_dot and
2431 reflowed_lines.fits_on_current_line(self.size + 1)) and
2432 not reflowed_lines.line_empty() and
2433 not self.is_colon and
2434 not (prev_item and prev_item.is_name and
2435 str(self) == '(')
2436 ):
2437 # Start a new line if there is already something on the line and
2438 # adding this atom would make it go over the max line length.
2439 reflowed_lines.add_line_break(continued_indent)
2440 else:
2441 reflowed_lines.add_space_if_needed(str(self))
2443 reflowed_lines.add(self, len(continued_indent),
2444 break_after_open_bracket)
2446 def emit(self):
2447 return self.__repr__()
2449 @property
2450 def is_keyword(self):
2451 return keyword.iskeyword(self._atom.token_string)
2453 @property
2454 def is_string(self):
2455 return self._atom.token_type == tokenize.STRING
2457 @property
2458 def is_fstring_start(self):
2459 if not IS_SUPPORT_TOKEN_FSTRING:
2460 return False
2461 return self._atom.token_type == tokenize.FSTRING_START
2463 @property
2464 def is_fstring_end(self):
2465 if not IS_SUPPORT_TOKEN_FSTRING:
2466 return False
2467 return self._atom.token_type == tokenize.FSTRING_END
2469 @property
2470 def is_name(self):
2471 return self._atom.token_type == tokenize.NAME
2473 @property
2474 def is_number(self):
2475 return self._atom.token_type == tokenize.NUMBER
2477 @property
2478 def is_comma(self):
2479 return self._atom.token_string == ','
2481 @property
2482 def is_colon(self):
2483 return self._atom.token_string == ':'
2485 @property
2486 def size(self):
2487 return len(self._atom.token_string)
2490class Container(object):
2492 """Base class for all container types."""
2494 def __init__(self, items):
2495 self._items = items
2497 def __repr__(self):
2498 string = ''
2499 last_was_keyword = False
2501 for item in self._items:
2502 if item.is_comma:
2503 string += ', '
2504 elif item.is_colon:
2505 string += ': '
2506 else:
2507 item_string = str(item)
2508 if (
2509 string and
2510 (last_was_keyword or
2511 (not string.endswith(tuple('([{,.:}]) ')) and
2512 not item_string.startswith(tuple('([{,.:}])'))))
2513 ):
2514 string += ' '
2515 string += item_string
2517 last_was_keyword = item.is_keyword
2518 return string
2520 def __iter__(self):
2521 for element in self._items:
2522 yield element
2524 def __getitem__(self, idx):
2525 return self._items[idx]
2527 def reflow(self, reflowed_lines, continued_indent,
2528 break_after_open_bracket=False):
2529 last_was_container = False
2530 for (index, item) in enumerate(self._items):
2531 next_item = get_item(self._items, index + 1)
2533 if isinstance(item, Atom):
2534 is_list_comp_or_if_expr = (
2535 isinstance(self, (ListComprehension, IfExpression)))
2536 item.reflow(reflowed_lines, continued_indent,
2537 self._get_extent(index),
2538 is_list_comp_or_if_expr=is_list_comp_or_if_expr,
2539 next_is_dot=(next_item and
2540 str(next_item) == '.'))
2541 if last_was_container and item.is_comma:
2542 reflowed_lines.add_line_break(continued_indent)
2543 last_was_container = False
2544 else: # isinstance(item, Container)
2545 reflowed_lines.add(item, len(continued_indent),
2546 break_after_open_bracket)
2547 last_was_container = not isinstance(item, (ListComprehension,
2548 IfExpression))
2550 if (
2551 break_after_open_bracket and index == 0 and
2552 # Prefer to keep empty containers together instead of
2553 # separating them.
2554 str(item) == self.open_bracket and
2555 (not next_item or str(next_item) != self.close_bracket) and
2556 (len(self._items) != 3 or not isinstance(next_item, Atom))
2557 ):
2558 reflowed_lines.add_line_break(continued_indent)
2559 break_after_open_bracket = False
2560 else:
2561 next_next_item = get_item(self._items, index + 2)
2562 if (
2563 str(item) not in ['.', '%', 'in'] and
2564 next_item and not isinstance(next_item, Container) and
2565 str(next_item) != ':' and
2566 next_next_item and (not isinstance(next_next_item, Atom) or
2567 str(next_item) == 'not') and
2568 not reflowed_lines.line_empty() and
2569 not reflowed_lines.fits_on_current_line(
2570 self._get_extent(index + 1) + 2)
2571 ):
2572 reflowed_lines.add_line_break(continued_indent)
2574 def _get_extent(self, index):
2575 """The extent of the full element.
2577 E.g., the length of a function call or keyword.
2579 """
2580 extent = 0
2581 prev_item = get_item(self._items, index - 1)
2582 seen_dot = prev_item and str(prev_item) == '.'
2583 while index < len(self._items):
2584 item = get_item(self._items, index)
2585 index += 1
2587 if isinstance(item, (ListComprehension, IfExpression)):
2588 break
2590 if isinstance(item, Container):
2591 if prev_item and prev_item.is_name:
2592 if seen_dot:
2593 extent += 1
2594 else:
2595 extent += item.size
2597 prev_item = item
2598 continue
2599 elif (str(item) not in ['.', '=', ':', 'not'] and
2600 not item.is_name and not item.is_string):
2601 break
2603 if str(item) == '.':
2604 seen_dot = True
2606 extent += item.size
2607 prev_item = item
2609 return extent
2611 @property
2612 def is_string(self):
2613 return False
2615 @property
2616 def size(self):
2617 return len(self.__repr__())
2619 @property
2620 def is_keyword(self):
2621 return False
2623 @property
2624 def is_name(self):
2625 return False
2627 @property
2628 def is_comma(self):
2629 return False
2631 @property
2632 def is_colon(self):
2633 return False
2635 @property
2636 def open_bracket(self):
2637 return None
2639 @property
2640 def close_bracket(self):
2641 return None
2644class Tuple(Container):
2646 """A high-level representation of a tuple."""
2648 @property
2649 def open_bracket(self):
2650 return '('
2652 @property
2653 def close_bracket(self):
2654 return ')'
2657class List(Container):
2659 """A high-level representation of a list."""
2661 @property
2662 def open_bracket(self):
2663 return '['
2665 @property
2666 def close_bracket(self):
2667 return ']'
2670class DictOrSet(Container):
2672 """A high-level representation of a dictionary or set."""
2674 @property
2675 def open_bracket(self):
2676 return '{'
2678 @property
2679 def close_bracket(self):
2680 return '}'
2683class ListComprehension(Container):
2685 """A high-level representation of a list comprehension."""
2687 @property
2688 def size(self):
2689 length = 0
2690 for item in self._items:
2691 if isinstance(item, IfExpression):
2692 break
2693 length += item.size
2694 return length
2697class IfExpression(Container):
2699 """A high-level representation of an if-expression."""
2702def _parse_container(tokens, index, for_or_if=None):
2703 """Parse a high-level container, such as a list, tuple, etc."""
2705 # Store the opening bracket.
2706 items = [Atom(Token(*tokens[index]))]
2707 index += 1
2709 num_tokens = len(tokens)
2710 while index < num_tokens:
2711 tok = Token(*tokens[index])
2713 if tok.token_string in ',)]}':
2714 # First check if we're at the end of a list comprehension or
2715 # if-expression. Don't add the ending token as part of the list
2716 # comprehension or if-expression, because they aren't part of those
2717 # constructs.
2718 if for_or_if == 'for':
2719 return (ListComprehension(items), index - 1)
2721 elif for_or_if == 'if':
2722 return (IfExpression(items), index - 1)
2724 # We've reached the end of a container.
2725 items.append(Atom(tok))
2727 # If not, then we are at the end of a container.
2728 if tok.token_string == ')':
2729 # The end of a tuple.
2730 return (Tuple(items), index)
2732 elif tok.token_string == ']':
2733 # The end of a list.
2734 return (List(items), index)
2736 elif tok.token_string == '}':
2737 # The end of a dictionary or set.
2738 return (DictOrSet(items), index)
2740 elif tok.token_string in '([{':
2741 # A sub-container is being defined.
2742 (container, index) = _parse_container(tokens, index)
2743 items.append(container)
2745 elif tok.token_string == 'for':
2746 (container, index) = _parse_container(tokens, index, 'for')
2747 items.append(container)
2749 elif tok.token_string == 'if':
2750 (container, index) = _parse_container(tokens, index, 'if')
2751 items.append(container)
2753 else:
2754 items.append(Atom(tok))
2756 index += 1
2758 return (None, None)
2761def _parse_tokens(tokens):
2762 """Parse the tokens.
2764 This converts the tokens into a form where we can manipulate them
2765 more easily.
2767 """
2769 index = 0
2770 parsed_tokens = []
2772 num_tokens = len(tokens)
2773 while index < num_tokens:
2774 tok = Token(*tokens[index])
2776 assert tok.token_type != token.INDENT
2777 if tok.token_type == tokenize.NEWLINE:
2778 # There's only one newline and it's at the end.
2779 break
2781 if tok.token_string in '([{':
2782 (container, index) = _parse_container(tokens, index)
2783 if not container:
2784 return None
2785 parsed_tokens.append(container)
2786 else:
2787 parsed_tokens.append(Atom(tok))
2789 index += 1
2791 return parsed_tokens
2794def _reflow_lines(parsed_tokens, indentation, max_line_length,
2795 start_on_prefix_line):
2796 """Reflow the lines so that it looks nice."""
2798 if str(parsed_tokens[0]) == 'def':
2799 # A function definition gets indented a bit more.
2800 continued_indent = indentation + ' ' * 2 * DEFAULT_INDENT_SIZE
2801 else:
2802 continued_indent = indentation + ' ' * DEFAULT_INDENT_SIZE
2804 break_after_open_bracket = not start_on_prefix_line
2806 lines = ReformattedLines(max_line_length)
2807 lines.add_indent(len(indentation.lstrip('\r\n')))
2809 if not start_on_prefix_line:
2810 # If splitting after the opening bracket will cause the first element
2811 # to be aligned weirdly, don't try it.
2812 first_token = get_item(parsed_tokens, 0)
2813 second_token = get_item(parsed_tokens, 1)
2815 if (
2816 first_token and second_token and
2817 str(second_token)[0] == '(' and
2818 len(indentation) + len(first_token) + 1 == len(continued_indent)
2819 ):
2820 return None
2822 for item in parsed_tokens:
2823 lines.add_space_if_needed(str(item), equal=True)
2825 save_continued_indent = continued_indent
2826 if start_on_prefix_line and isinstance(item, Container):
2827 start_on_prefix_line = False
2828 continued_indent = ' ' * (lines.current_size() + 1)
2830 item.reflow(lines, continued_indent, break_after_open_bracket)
2831 continued_indent = save_continued_indent
2833 return lines.emit()
2836def _shorten_line_at_tokens_new(tokens, source, indentation,
2837 max_line_length):
2838 """Shorten the line taking its length into account.
2840 The input is expected to be free of newlines except for inside
2841 multiline strings and at the end.
2843 """
2844 # Yield the original source so to see if it's a better choice than the
2845 # shortened candidate lines we generate here.
2846 yield indentation + source
2848 parsed_tokens = _parse_tokens(tokens)
2850 if parsed_tokens:
2851 # Perform two reflows. The first one starts on the same line as the
2852 # prefix. The second starts on the line after the prefix.
2853 fixed = _reflow_lines(parsed_tokens, indentation, max_line_length,
2854 start_on_prefix_line=True)
2855 if fixed and check_syntax(normalize_multiline(fixed.lstrip())):
2856 yield fixed
2858 fixed = _reflow_lines(parsed_tokens, indentation, max_line_length,
2859 start_on_prefix_line=False)
2860 if fixed and check_syntax(normalize_multiline(fixed.lstrip())):
2861 yield fixed
2864def _shorten_line_at_tokens(tokens, source, indentation, indent_word,
2865 key_token_strings, aggressive):
2866 """Separate line by breaking at tokens in key_token_strings.
2868 The input is expected to be free of newlines except for inside
2869 multiline strings and at the end.
2871 """
2872 offsets = []
2873 for (index, _t) in enumerate(token_offsets(tokens)):
2874 (token_type,
2875 token_string,
2876 start_offset,
2877 end_offset) = _t
2879 assert token_type != token.INDENT
2881 if token_string in key_token_strings:
2882 # Do not break in containers with zero or one items.
2883 unwanted_next_token = {
2884 '(': ')',
2885 '[': ']',
2886 '{': '}'}.get(token_string)
2887 if unwanted_next_token:
2888 if (
2889 get_item(tokens,
2890 index + 1,
2891 default=[None, None])[1] == unwanted_next_token or
2892 get_item(tokens,
2893 index + 2,
2894 default=[None, None])[1] == unwanted_next_token
2895 ):
2896 continue
2898 if (
2899 index > 2 and token_string == '(' and
2900 tokens[index - 1][1] in ',(%['
2901 ):
2902 # Don't split after a tuple start, or before a tuple start if
2903 # the tuple is in a list.
2904 continue
2906 if end_offset < len(source) - 1:
2907 # Don't split right before newline.
2908 offsets.append(end_offset)
2909 else:
2910 # Break at adjacent strings. These were probably meant to be on
2911 # separate lines in the first place.
2912 previous_token = get_item(tokens, index - 1)
2913 if (
2914 token_type == tokenize.STRING and
2915 previous_token and previous_token[0] == tokenize.STRING
2916 ):
2917 offsets.append(start_offset)
2919 current_indent = None
2920 fixed = None
2921 for line in split_at_offsets(source, offsets):
2922 if fixed:
2923 fixed += '\n' + current_indent + line
2925 for symbol in '([{':
2926 if line.endswith(symbol):
2927 current_indent += indent_word
2928 else:
2929 # First line.
2930 fixed = line
2931 assert not current_indent
2932 current_indent = indent_word
2934 assert fixed is not None
2936 if check_syntax(normalize_multiline(fixed)
2937 if aggressive > 1 else fixed):
2938 return indentation + fixed
2940 return None
2943def token_offsets(tokens):
2944 """Yield tokens and offsets."""
2945 end_offset = 0
2946 previous_end_row = 0
2947 previous_end_column = 0
2948 for t in tokens:
2949 token_type = t[0]
2950 token_string = t[1]
2951 (start_row, start_column) = t[2]
2952 (end_row, end_column) = t[3]
2954 # Account for the whitespace between tokens.
2955 end_offset += start_column
2956 if previous_end_row == start_row:
2957 end_offset -= previous_end_column
2959 # Record the start offset of the token.
2960 start_offset = end_offset
2962 # Account for the length of the token itself.
2963 end_offset += len(token_string)
2965 yield (token_type,
2966 token_string,
2967 start_offset,
2968 end_offset)
2970 previous_end_row = end_row
2971 previous_end_column = end_column
2974def normalize_multiline(line):
2975 """Normalize multiline-related code that will cause syntax error.
2977 This is for purposes of checking syntax.
2979 """
2980 if line.startswith(('def ', 'async def ')) and line.rstrip().endswith(':'):
2981 return line + ' pass'
2982 elif line.startswith('return '):
2983 return 'def _(): ' + line
2984 elif line.startswith('@'):
2985 return line + 'def _(): pass'
2986 elif line.startswith('class '):
2987 return line + ' pass'
2988 elif line.startswith(('if ', 'elif ', 'for ', 'while ')):
2989 return line + ' pass'
2991 return line
2994def fix_whitespace(line, offset, replacement):
2995 """Replace whitespace at offset and return fixed line."""
2996 # Replace escaped newlines too
2997 left = line[:offset].rstrip('\n\r \t\\')
2998 right = line[offset:].lstrip('\n\r \t\\')
2999 if right.startswith('#'):
3000 return line
3002 return left + replacement + right
3005def _execute_pep8(pep8_options, source):
3006 """Execute pycodestyle via python method calls."""
3007 class QuietReport(pycodestyle.BaseReport):
3009 """Version of checker that does not print."""
3011 def __init__(self, options):
3012 super(QuietReport, self).__init__(options)
3013 self.__full_error_results = []
3015 def error(self, line_number, offset, text, check):
3016 """Collect errors."""
3017 code = super(QuietReport, self).error(line_number,
3018 offset,
3019 text,
3020 check)
3021 if code:
3022 self.__full_error_results.append(
3023 {'id': code,
3024 'line': line_number,
3025 'column': offset + 1,
3026 'info': text})
3028 def full_error_results(self):
3029 """Return error results in detail.
3031 Results are in the form of a list of dictionaries. Each
3032 dictionary contains 'id', 'line', 'column', and 'info'.
3034 """
3035 return self.__full_error_results
3037 checker = pycodestyle.Checker('', lines=source, reporter=QuietReport,
3038 **pep8_options)
3039 checker.check_all()
3040 return checker.report.full_error_results()
3043def _remove_leading_and_normalize(line, with_rstrip=True):
3044 # ignore FF in first lstrip()
3045 if with_rstrip:
3046 return line.lstrip(' \t\v').rstrip(CR + LF) + '\n'
3047 return line.lstrip(' \t\v')
3050class Reindenter(object):
3052 """Reindents badly-indented code to uniformly use four-space indentation.
3054 Released to the public domain, by Tim Peters, 03 October 2000.
3056 """
3058 def __init__(self, input_text, leave_tabs=False):
3059 sio = io.StringIO(input_text)
3060 source_lines = sio.readlines()
3062 self.string_content_line_numbers = multiline_string_lines(input_text)
3064 # File lines, rstripped & tab-expanded. Dummy at start is so
3065 # that we can use tokenize's 1-based line numbering easily.
3066 # Note that a line is all-blank iff it is a newline.
3067 self.lines = []
3068 for line_number, line in enumerate(source_lines, start=1):
3069 # Do not modify if inside a multiline string.
3070 if line_number in self.string_content_line_numbers:
3071 self.lines.append(line)
3072 else:
3073 # Only expand leading tabs.
3074 with_rstrip = line_number != len(source_lines)
3075 if leave_tabs:
3076 self.lines.append(
3077 _get_indentation(line) +
3078 _remove_leading_and_normalize(line, with_rstrip)
3079 )
3080 else:
3081 self.lines.append(
3082 _get_indentation(line).expandtabs() +
3083 _remove_leading_and_normalize(line, with_rstrip)
3084 )
3086 self.lines.insert(0, None)
3087 self.index = 1 # index into self.lines of next line
3088 self.input_text = input_text
3090 def run(self, indent_size=DEFAULT_INDENT_SIZE):
3091 """Fix indentation and return modified line numbers.
3093 Line numbers are indexed at 1.
3095 """
3096 if indent_size < 1:
3097 return self.input_text
3099 try:
3100 stats = _reindent_stats(tokenize.generate_tokens(self.getline))
3101 except (SyntaxError, tokenize.TokenError):
3102 return self.input_text
3103 # Remove trailing empty lines.
3104 lines = self.lines
3105 # Sentinel.
3106 stats.append((len(lines), 0))
3107 # Map count of leading spaces to # we want.
3108 have2want = {}
3109 # Program after transformation.
3110 after = []
3111 # Copy over initial empty lines -- there's nothing to do until
3112 # we see a line with *something* on it.
3113 i = stats[0][0]
3114 after.extend(lines[1:i])
3115 for i in range(len(stats) - 1):
3116 thisstmt, thislevel = stats[i]
3117 nextstmt = stats[i + 1][0]
3118 have = _leading_space_count(lines[thisstmt])
3119 want = thislevel * indent_size
3120 if want < 0:
3121 # A comment line.
3122 if have:
3123 # An indented comment line. If we saw the same
3124 # indentation before, reuse what it most recently
3125 # mapped to.
3126 want = have2want.get(have, -1)
3127 if want < 0:
3128 # Then it probably belongs to the next real stmt.
3129 for j in range(i + 1, len(stats) - 1):
3130 jline, jlevel = stats[j]
3131 if jlevel >= 0:
3132 if have == _leading_space_count(lines[jline]):
3133 want = jlevel * indent_size
3134 break
3135 # Maybe it's a hanging comment like this one,
3136 if want < 0:
3137 # in which case we should shift it like its base
3138 # line got shifted.
3139 for j in range(i - 1, -1, -1):
3140 jline, jlevel = stats[j]
3141 if jlevel >= 0:
3142 want = (have + _leading_space_count(
3143 after[jline - 1]) -
3144 _leading_space_count(lines[jline]))
3145 break
3146 if want < 0:
3147 # Still no luck -- leave it alone.
3148 want = have
3149 else:
3150 want = 0
3151 assert want >= 0
3152 have2want[have] = want
3153 diff = want - have
3154 if diff == 0 or have == 0:
3155 after.extend(lines[thisstmt:nextstmt])
3156 else:
3157 for line_number, line in enumerate(lines[thisstmt:nextstmt],
3158 start=thisstmt):
3159 if line_number in self.string_content_line_numbers:
3160 after.append(line)
3161 elif diff > 0:
3162 if line == '\n':
3163 after.append(line)
3164 else:
3165 after.append(' ' * diff + line)
3166 else:
3167 remove = min(_leading_space_count(line), -diff)
3168 after.append(line[remove:])
3170 return ''.join(after)
3172 def getline(self):
3173 """Line-getter for tokenize."""
3174 if self.index >= len(self.lines):
3175 line = ''
3176 else:
3177 line = self.lines[self.index]
3178 self.index += 1
3179 return line
3182def _reindent_stats(tokens):
3183 """Return list of (lineno, indentlevel) pairs.
3185 One for each stmt and comment line. indentlevel is -1 for comment
3186 lines, as a signal that tokenize doesn't know what to do about them;
3187 indeed, they're our headache!
3189 """
3190 find_stmt = 1 # Next token begins a fresh stmt?
3191 level = 0 # Current indent level.
3192 stats = []
3194 for t in tokens:
3195 token_type = t[0]
3196 sline = t[2][0]
3197 line = t[4]
3199 if token_type == tokenize.NEWLINE:
3200 # A program statement, or ENDMARKER, will eventually follow,
3201 # after some (possibly empty) run of tokens of the form
3202 # (NL | COMMENT)* (INDENT | DEDENT+)?
3203 find_stmt = 1
3205 elif token_type == tokenize.INDENT:
3206 find_stmt = 1
3207 level += 1
3209 elif token_type == tokenize.DEDENT:
3210 find_stmt = 1
3211 level -= 1
3213 elif token_type == tokenize.COMMENT:
3214 if find_stmt:
3215 stats.append((sline, -1))
3216 # But we're still looking for a new stmt, so leave
3217 # find_stmt alone.
3219 elif token_type == tokenize.NL:
3220 pass
3222 elif find_stmt:
3223 # This is the first "real token" following a NEWLINE, so it
3224 # must be the first token of the next program statement, or an
3225 # ENDMARKER.
3226 find_stmt = 0
3227 if line: # Not endmarker.
3228 stats.append((sline, level))
3230 return stats
3233def _leading_space_count(line):
3234 """Return number of leading spaces in line."""
3235 i = 0
3236 while i < len(line) and line[i] == ' ':
3237 i += 1
3238 return i
3241def check_syntax(code):
3242 """Return True if syntax is okay."""
3243 try:
3244 return compile(code, '<string>', 'exec', dont_inherit=True)
3245 except (SyntaxError, TypeError, ValueError):
3246 return False
3249def find_with_line_numbers(pattern, contents):
3250 """A wrapper around 're.finditer' to find line numbers.
3252 Returns a list of line numbers where pattern was found in contents.
3253 """
3254 matches = list(re.finditer(pattern, contents))
3255 if not matches:
3256 return []
3258 end = matches[-1].start()
3260 # -1 so a failed `rfind` maps to the first line.
3261 newline_offsets = {
3262 -1: 0
3263 }
3264 for line_num, m in enumerate(re.finditer(r'\n', contents), 1):
3265 offset = m.start()
3266 if offset > end:
3267 break
3268 newline_offsets[offset] = line_num
3270 def get_line_num(match, contents):
3271 """Get the line number of string in a files contents.
3273 Failing to find the newline is OK, -1 maps to 0
3275 """
3276 newline_offset = contents.rfind('\n', 0, match.start())
3277 return newline_offsets[newline_offset]
3279 return [get_line_num(match, contents) + 1 for match in matches]
3282def get_disabled_ranges(source):
3283 """Returns a list of tuples representing the disabled ranges.
3285 If disabled and no re-enable will disable for rest of file.
3287 """
3288 enable_line_nums = find_with_line_numbers(ENABLE_REGEX, source)
3289 disable_line_nums = find_with_line_numbers(DISABLE_REGEX, source)
3290 total_lines = len(re.findall("\n", source)) + 1
3292 enable_commands = {}
3293 for num in enable_line_nums:
3294 enable_commands[num] = True
3295 for num in disable_line_nums:
3296 enable_commands[num] = False
3298 disabled_ranges = []
3299 currently_enabled = True
3300 disabled_start = None
3302 for line, commanded_enabled in sorted(enable_commands.items()):
3303 if commanded_enabled is False and currently_enabled is True:
3304 disabled_start = line
3305 currently_enabled = False
3306 elif commanded_enabled is True and currently_enabled is False:
3307 disabled_ranges.append((disabled_start, line))
3308 currently_enabled = True
3310 if currently_enabled is False:
3311 disabled_ranges.append((disabled_start, total_lines))
3313 return disabled_ranges
3316def filter_disabled_results(result, disabled_ranges):
3317 """Filter out reports based on tuple of disabled ranges.
3319 """
3320 line = result['line']
3321 for disabled_range in disabled_ranges:
3322 if disabled_range[0] <= line <= disabled_range[1]:
3323 return False
3324 return True
3327def filter_results(source, results, aggressive):
3328 """Filter out spurious reports from pycodestyle.
3330 If aggressive is True, we allow possibly unsafe fixes (E711, E712).
3332 """
3333 non_docstring_string_line_numbers = multiline_string_lines(
3334 source, include_docstrings=False)
3335 all_string_line_numbers = multiline_string_lines(
3336 source, include_docstrings=True)
3338 commented_out_code_line_numbers = commented_out_code_lines(source)
3340 # Filter out the disabled ranges
3341 disabled_ranges = get_disabled_ranges(source)
3342 if disabled_ranges:
3343 results = [
3344 result for result in results if filter_disabled_results(
3345 result,
3346 disabled_ranges,
3347 )
3348 ]
3350 has_e901 = any(result['id'].lower() == 'e901' for result in results)
3352 for r in results:
3353 issue_id = r['id'].lower()
3355 if r['line'] in non_docstring_string_line_numbers:
3356 if issue_id.startswith(('e1', 'e501', 'w191')):
3357 continue
3359 if r['line'] in all_string_line_numbers:
3360 if issue_id in ['e501']:
3361 continue
3363 # We must offset by 1 for lines that contain the trailing contents of
3364 # multiline strings.
3365 if not aggressive and (r['line'] + 1) in all_string_line_numbers:
3366 # Do not modify multiline strings in non-aggressive mode. Remove
3367 # trailing whitespace could break doctests.
3368 if issue_id.startswith(('w29', 'w39')):
3369 continue
3371 if aggressive <= 0:
3372 if issue_id.startswith(('e711', 'e72', 'w6')):
3373 continue
3375 if aggressive <= 1:
3376 if issue_id.startswith(('e712', 'e713', 'e714')):
3377 continue
3379 if aggressive <= 2:
3380 if issue_id.startswith(('e704')):
3381 continue
3383 if r['line'] in commented_out_code_line_numbers:
3384 if issue_id.startswith(('e261', 'e262', 'e501')):
3385 continue
3387 # Do not touch indentation if there is a token error caused by
3388 # incomplete multi-line statement. Otherwise, we risk screwing up the
3389 # indentation.
3390 if has_e901:
3391 if issue_id.startswith(('e1', 'e7')):
3392 continue
3394 yield r
3397def multiline_string_lines(source, include_docstrings=False):
3398 """Return line numbers that are within multiline strings.
3400 The line numbers are indexed at 1.
3402 Docstrings are ignored.
3404 """
3405 line_numbers = set()
3406 previous_token_type = ''
3407 _check_target_tokens = [tokenize.STRING]
3408 if IS_SUPPORT_TOKEN_FSTRING:
3409 _check_target_tokens.extend([
3410 tokenize.FSTRING_START,
3411 tokenize.FSTRING_MIDDLE,
3412 tokenize.FSTRING_END,
3413 ])
3414 try:
3415 for t in generate_tokens(source):
3416 token_type = t[0]
3417 start_row = t[2][0]
3418 end_row = t[3][0]
3420 if token_type in _check_target_tokens and start_row != end_row:
3421 if (
3422 include_docstrings or
3423 previous_token_type != tokenize.INDENT
3424 ):
3425 # We increment by one since we want the contents of the
3426 # string.
3427 line_numbers |= set(range(1 + start_row, 1 + end_row))
3429 previous_token_type = token_type
3430 except (SyntaxError, tokenize.TokenError):
3431 pass
3433 return line_numbers
3436def commented_out_code_lines(source):
3437 """Return line numbers of comments that are likely code.
3439 Commented-out code is bad practice, but modifying it just adds even
3440 more clutter.
3442 """
3443 line_numbers = []
3444 try:
3445 for t in generate_tokens(source):
3446 token_type = t[0]
3447 token_string = t[1]
3448 start_row = t[2][0]
3449 line = t[4]
3451 # Ignore inline comments.
3452 if not line.lstrip().startswith('#'):
3453 continue
3455 if token_type == tokenize.COMMENT:
3456 stripped_line = token_string.lstrip('#').strip()
3457 with warnings.catch_warnings():
3458 # ignore SyntaxWarning in Python3.8+
3459 # refs:
3460 # https://bugs.python.org/issue15248
3461 # https://docs.python.org/3.8/whatsnew/3.8.html#other-language-changes
3462 warnings.filterwarnings("ignore", category=SyntaxWarning)
3463 if (
3464 ' ' in stripped_line and
3465 '#' not in stripped_line and
3466 check_syntax(stripped_line)
3467 ):
3468 line_numbers.append(start_row)
3469 except (SyntaxError, tokenize.TokenError):
3470 pass
3472 return line_numbers
3475def shorten_comment(line, max_line_length, last_comment=False):
3476 """Return trimmed or split long comment line.
3478 If there are no comments immediately following it, do a text wrap.
3479 Doing this wrapping on all comments in general would lead to jagged
3480 comment text.
3482 """
3483 assert len(line) > max_line_length
3484 line = line.rstrip()
3486 # PEP 8 recommends 72 characters for comment text.
3487 indentation = _get_indentation(line) + '# '
3488 max_line_length = min(max_line_length,
3489 len(indentation) + 72)
3491 MIN_CHARACTER_REPEAT = 5
3492 if (
3493 len(line) - len(line.rstrip(line[-1])) >= MIN_CHARACTER_REPEAT and
3494 not line[-1].isalnum()
3495 ):
3496 # Trim comments that end with things like ---------
3497 return line[:max_line_length] + '\n'
3498 elif last_comment and re.match(r'\s*#+\s*\w+', line):
3499 split_lines = textwrap.wrap(line.lstrip(' \t#'),
3500 initial_indent=indentation,
3501 subsequent_indent=indentation,
3502 width=max_line_length,
3503 break_long_words=False,
3504 break_on_hyphens=False)
3505 return '\n'.join(split_lines) + '\n'
3507 return line + '\n'
3510def normalize_line_endings(lines, newline):
3511 """Return fixed line endings.
3513 All lines will be modified to use the most common line ending.
3514 """
3515 line = [line.rstrip('\n\r') + newline for line in lines]
3516 if line and lines[-1] == lines[-1].rstrip('\n\r'):
3517 line[-1] = line[-1].rstrip('\n\r')
3518 return line
3521def mutual_startswith(a, b):
3522 return b.startswith(a) or a.startswith(b)
3525def code_match(code, select, ignore):
3526 if ignore:
3527 assert not isinstance(ignore, str)
3528 for ignored_code in [c.strip() for c in ignore]:
3529 if mutual_startswith(code.lower(), ignored_code.lower()):
3530 return False
3532 if select:
3533 assert not isinstance(select, str)
3534 for selected_code in [c.strip() for c in select]:
3535 if mutual_startswith(code.lower(), selected_code.lower()):
3536 return True
3537 return False
3539 return True
3542def fix_code(source, options=None, encoding=None, apply_config=False):
3543 """Return fixed source code.
3545 "encoding" will be used to decode "source" if it is a byte string.
3547 """
3548 options = _get_options(options, apply_config)
3549 # normalize
3550 options.ignore = [opt.upper() for opt in options.ignore]
3551 options.select = [opt.upper() for opt in options.select]
3553 # check ignore args
3554 # NOTE: If W50x is not included, add W50x because the code
3555 # correction result is indefinite.
3556 ignore_opt = options.ignore
3557 if not {"W50", "W503", "W504"} & set(ignore_opt):
3558 options.ignore.append("W50")
3560 if not isinstance(source, str):
3561 source = source.decode(encoding or get_encoding())
3563 sio = io.StringIO(source)
3564 return fix_lines(sio.readlines(), options=options)
3567def _get_options(raw_options, apply_config):
3568 """Return parsed options."""
3569 if not raw_options:
3570 return parse_args([''], apply_config=apply_config)
3572 if isinstance(raw_options, dict):
3573 options = parse_args([''], apply_config=apply_config)
3574 for name, value in raw_options.items():
3575 if not hasattr(options, name):
3576 raise ValueError("No such option '{}'".format(name))
3578 # Check for very basic type errors.
3579 expected_type = type(getattr(options, name))
3580 if not isinstance(expected_type, (str, )):
3581 if isinstance(value, (str, )):
3582 raise ValueError(
3583 "Option '{}' should not be a string".format(name))
3584 setattr(options, name, value)
3585 else:
3586 options = raw_options
3588 return options
3591def fix_lines(source_lines, options, filename=''):
3592 """Return fixed source code."""
3593 # Transform everything to line feed. Then change them back to original
3594 # before returning fixed source code.
3595 original_newline = find_newline(source_lines)
3596 tmp_source = ''.join(normalize_line_endings(source_lines, '\n'))
3598 # Keep a history to break out of cycles.
3599 previous_hashes = set()
3601 if options.line_range:
3602 # Disable "apply_local_fixes()" for now due to issue #175.
3603 fixed_source = tmp_source
3604 else:
3605 # Apply global fixes only once (for efficiency).
3606 fixed_source = apply_global_fixes(tmp_source,
3607 options,
3608 filename=filename)
3610 passes = 0
3611 long_line_ignore_cache = set()
3612 while hash(fixed_source) not in previous_hashes:
3613 if options.pep8_passes >= 0 and passes > options.pep8_passes:
3614 break
3615 passes += 1
3617 previous_hashes.add(hash(fixed_source))
3619 tmp_source = copy.copy(fixed_source)
3621 fix = FixPEP8(
3622 filename,
3623 options,
3624 contents=tmp_source,
3625 long_line_ignore_cache=long_line_ignore_cache)
3627 fixed_source = fix.fix()
3629 sio = io.StringIO(fixed_source)
3630 return ''.join(normalize_line_endings(sio.readlines(), original_newline))
3633def fix_file(filename, options=None, output=None, apply_config=False):
3634 if not options:
3635 options = parse_args([filename], apply_config=apply_config)
3637 original_source = readlines_from_file(filename)
3639 fixed_source = original_source
3641 if options.in_place or options.diff or output:
3642 encoding = detect_encoding(filename)
3644 if output:
3645 output = LineEndingWrapper(wrap_output(output, encoding=encoding))
3647 fixed_source = fix_lines(fixed_source, options, filename=filename)
3649 if options.diff:
3650 new = io.StringIO(fixed_source)
3651 new = new.readlines()
3652 diff = get_diff_text(original_source, new, filename)
3653 if output:
3654 output.write(diff)
3655 output.flush()
3656 elif options.jobs > 1:
3657 diff = diff.encode(encoding)
3658 return diff
3659 elif options.in_place:
3660 original = "".join(original_source).splitlines()
3661 fixed = fixed_source.splitlines()
3662 original_source_last_line = (
3663 original_source[-1].split("\n")[-1] if original_source else ""
3664 )
3665 fixed_source_last_line = fixed_source.split("\n")[-1]
3666 if original != fixed or (
3667 original_source_last_line != fixed_source_last_line
3668 ):
3669 with open_with_encoding(filename, 'w', encoding=encoding) as fp:
3670 fp.write(fixed_source)
3671 return fixed_source
3672 return None
3673 else:
3674 if output:
3675 output.write(fixed_source)
3676 output.flush()
3677 return fixed_source
3680def global_fixes():
3681 """Yield multiple (code, function) tuples."""
3682 for function in list(globals().values()):
3683 if inspect.isfunction(function):
3684 arguments = _get_parameters(function)
3685 if arguments[:1] != ['source']:
3686 continue
3688 code = extract_code_from_function(function)
3689 if code:
3690 yield (code, function)
3693def _get_parameters(function):
3694 # pylint: disable=deprecated-method
3695 if sys.version_info.major >= 3:
3696 # We need to match "getargspec()", which includes "self" as the first
3697 # value for methods.
3698 # https://bugs.python.org/issue17481#msg209469
3699 if inspect.ismethod(function):
3700 function = function.__func__
3702 return list(inspect.signature(function).parameters)
3703 else:
3704 return inspect.getargspec(function)[0]
3707def apply_global_fixes(source, options, where='global', filename='',
3708 codes=None):
3709 """Run global fixes on source code.
3711 These are fixes that only need be done once (unlike those in
3712 FixPEP8, which are dependent on pycodestyle).
3714 """
3715 if codes is None:
3716 codes = []
3717 if any(code_match(code, select=options.select, ignore=options.ignore)
3718 for code in ['E101', 'E111']):
3719 source = reindent(
3720 source,
3721 indent_size=options.indent_size,
3722 leave_tabs=not (
3723 code_match(
3724 'W191',
3725 select=options.select,
3726 ignore=options.ignore
3727 )
3728 )
3729 )
3731 for (code, function) in global_fixes():
3732 if code_match(code, select=options.select, ignore=options.ignore):
3733 if options.verbose:
3734 print('---> Applying {} fix for {}'.format(where,
3735 code.upper()),
3736 file=sys.stderr)
3737 source = function(source,
3738 aggressive=options.aggressive)
3740 return source
3743def extract_code_from_function(function):
3744 """Return code handled by function."""
3745 if not function.__name__.startswith('fix_'):
3746 return None
3748 code = re.sub('^fix_', '', function.__name__)
3749 if not code:
3750 return None
3752 try:
3753 int(code[1:])
3754 except ValueError:
3755 return None
3757 return code
3760def _get_package_version():
3761 packages = ["pycodestyle: {}".format(pycodestyle.__version__)]
3762 return ", ".join(packages)
3765def create_parser():
3766 """Return command-line parser."""
3767 parser = argparse.ArgumentParser(description=docstring_summary(__doc__),
3768 prog='autopep8')
3769 parser.add_argument('--version', action='version',
3770 version='%(prog)s {} ({})'.format(
3771 __version__, _get_package_version()))
3772 parser.add_argument('-v', '--verbose', action='count',
3773 default=0,
3774 help='print verbose messages; '
3775 'multiple -v result in more verbose messages')
3776 parser.add_argument('-d', '--diff', action='store_true',
3777 help='print the diff for the fixed source')
3778 parser.add_argument('-i', '--in-place', action='store_true',
3779 help='make changes to files in place')
3780 parser.add_argument('--global-config', metavar='filename',
3781 default=DEFAULT_CONFIG,
3782 help='path to a global pep8 config file; if this file '
3783 'does not exist then this is ignored '
3784 '(default: {})'.format(DEFAULT_CONFIG))
3785 parser.add_argument('--ignore-local-config', action='store_true',
3786 help="don't look for and apply local config files; "
3787 'if not passed, defaults are updated with any '
3788 "config files in the project's root directory")
3789 parser.add_argument('-r', '--recursive', action='store_true',
3790 help='run recursively over directories; '
3791 'must be used with --in-place or --diff')
3792 parser.add_argument('-j', '--jobs', type=int, metavar='n', default=1,
3793 help='number of parallel jobs; '
3794 'match CPU count if value is less than 1')
3795 parser.add_argument('-p', '--pep8-passes', metavar='n',
3796 default=-1, type=int,
3797 help='maximum number of additional pep8 passes '
3798 '(default: infinite)')
3799 parser.add_argument('-a', '--aggressive', action='count', default=0,
3800 help='enable non-whitespace changes; '
3801 'multiple -a result in more aggressive changes')
3802 parser.add_argument('--experimental', action='store_true',
3803 help='enable experimental fixes')
3804 parser.add_argument('--exclude', metavar='globs',
3805 help='exclude file/directory names that match these '
3806 'comma-separated globs')
3807 parser.add_argument('--list-fixes', action='store_true',
3808 help='list codes for fixes; '
3809 'used by --ignore and --select')
3810 parser.add_argument('--ignore', metavar='errors', default='',
3811 help='do not fix these errors/warnings '
3812 '(default: {})'.format(DEFAULT_IGNORE))
3813 parser.add_argument('--select', metavar='errors', default='',
3814 help='fix only these errors/warnings (e.g. E4,W)')
3815 parser.add_argument('--max-line-length', metavar='n', default=79, type=int,
3816 help='set maximum allowed line length '
3817 '(default: %(default)s)')
3818 parser.add_argument('--line-range', '--range', metavar='line',
3819 default=None, type=int, nargs=2,
3820 help='only fix errors found within this inclusive '
3821 'range of line numbers (e.g. 1 99); '
3822 'line numbers are indexed at 1')
3823 parser.add_argument('--indent-size', default=DEFAULT_INDENT_SIZE,
3824 type=int, help=argparse.SUPPRESS)
3825 parser.add_argument('--hang-closing', action='store_true',
3826 help='hang-closing option passed to pycodestyle')
3827 parser.add_argument('--exit-code', action='store_true',
3828 help='change to behavior of exit code.'
3829 ' default behavior of return value, 0 is no '
3830 'differences, 1 is error exit. return 2 when'
3831 ' add this option. 2 is exists differences.')
3832 parser.add_argument('files', nargs='*',
3833 help="files to format or '-' for standard in")
3835 return parser
3838def _expand_codes(codes, ignore_codes):
3839 """expand to individual E/W codes"""
3840 ret = set()
3842 is_conflict = False
3843 if all(
3844 any(
3845 conflicting_code.startswith(code)
3846 for code in codes
3847 )
3848 for conflicting_code in CONFLICTING_CODES
3849 ):
3850 is_conflict = True
3852 is_ignore_w503 = "W503" in ignore_codes
3853 is_ignore_w504 = "W504" in ignore_codes
3855 for code in codes:
3856 if code == "W":
3857 if is_ignore_w503 and is_ignore_w504:
3858 ret.update({"W1", "W2", "W3", "W505", "W6"})
3859 elif is_ignore_w503:
3860 ret.update({"W1", "W2", "W3", "W504", "W505", "W6"})
3861 else:
3862 ret.update({"W1", "W2", "W3", "W503", "W505", "W6"})
3863 elif code in ("W5", "W50"):
3864 if is_ignore_w503 and is_ignore_w504:
3865 ret.update({"W505"})
3866 elif is_ignore_w503:
3867 ret.update({"W504", "W505"})
3868 else:
3869 ret.update({"W503", "W505"})
3870 elif not (code in ("W503", "W504") and is_conflict):
3871 ret.add(code)
3873 return ret
3876def _parser_error_with_code(
3877 parser: argparse.ArgumentParser, code: int, msg: str,
3878) -> None:
3879 """wrap parser.error with exit code"""
3880 parser.print_usage(sys.stderr)
3881 parser.exit(code, f"{msg}\n")
3884def parse_args(arguments, apply_config=False):
3885 """Parse command-line options."""
3886 parser = create_parser()
3887 args = parser.parse_args(arguments)
3889 if not args.files and not args.list_fixes:
3890 _parser_error_with_code(
3891 parser, EXIT_CODE_ARGPARSE_ERROR, 'incorrect number of arguments',
3892 )
3894 args.files = [decode_filename(name) for name in args.files]
3896 if apply_config:
3897 parser = read_config(args, parser)
3898 # prioritize settings when exist pyproject.toml's tool.autopep8 section
3899 try:
3900 parser_with_pyproject_toml = read_pyproject_toml(args, parser)
3901 except Exception:
3902 parser_with_pyproject_toml = None
3903 if parser_with_pyproject_toml:
3904 parser = parser_with_pyproject_toml
3905 args = parser.parse_args(arguments)
3906 args.files = [decode_filename(name) for name in args.files]
3908 if '-' in args.files:
3909 if len(args.files) > 1:
3910 _parser_error_with_code(
3911 parser,
3912 EXIT_CODE_ARGPARSE_ERROR,
3913 'cannot mix stdin and regular files',
3914 )
3916 if args.diff:
3917 _parser_error_with_code(
3918 parser,
3919 EXIT_CODE_ARGPARSE_ERROR,
3920 '--diff cannot be used with standard input',
3921 )
3923 if args.in_place:
3924 _parser_error_with_code(
3925 parser,
3926 EXIT_CODE_ARGPARSE_ERROR,
3927 '--in-place cannot be used with standard input',
3928 )
3930 if args.recursive:
3931 _parser_error_with_code(
3932 parser,
3933 EXIT_CODE_ARGPARSE_ERROR,
3934 '--recursive cannot be used with standard input',
3935 )
3937 if len(args.files) > 1 and not (args.in_place or args.diff):
3938 _parser_error_with_code(
3939 parser,
3940 EXIT_CODE_ARGPARSE_ERROR,
3941 'autopep8 only takes one filename as argument '
3942 'unless the "--in-place" or "--diff" args are used',
3943 )
3945 if args.recursive and not (args.in_place or args.diff):
3946 _parser_error_with_code(
3947 parser,
3948 EXIT_CODE_ARGPARSE_ERROR,
3949 '--recursive must be used with --in-place or --diff',
3950 )
3952 if args.in_place and args.diff:
3953 _parser_error_with_code(
3954 parser,
3955 EXIT_CODE_ARGPARSE_ERROR,
3956 '--in-place and --diff are mutually exclusive',
3957 )
3959 if args.max_line_length <= 0:
3960 _parser_error_with_code(
3961 parser,
3962 EXIT_CODE_ARGPARSE_ERROR,
3963 '--max-line-length must be greater than 0',
3964 )
3966 if args.indent_size <= 0:
3967 _parser_error_with_code(
3968 parser,
3969 EXIT_CODE_ARGPARSE_ERROR,
3970 '--indent-size must be greater than 0',
3971 )
3973 if args.select:
3974 args.select = _expand_codes(
3975 _split_comma_separated(args.select),
3976 (_split_comma_separated(args.ignore) if args.ignore else [])
3977 )
3979 if args.ignore:
3980 args.ignore = _split_comma_separated(args.ignore)
3981 if all(
3982 not any(
3983 conflicting_code.startswith(ignore_code)
3984 for ignore_code in args.ignore
3985 )
3986 for conflicting_code in CONFLICTING_CODES
3987 ):
3988 args.ignore.update(CONFLICTING_CODES)
3989 elif not args.select:
3990 if args.aggressive:
3991 # Enable everything by default if aggressive.
3992 args.select = {'E', 'W1', 'W2', 'W3', 'W6'}
3993 else:
3994 args.ignore = _split_comma_separated(DEFAULT_IGNORE)
3996 if args.exclude:
3997 args.exclude = _split_comma_separated(args.exclude)
3998 else:
3999 args.exclude = {}
4001 if args.jobs < 1:
4002 # Do not import multiprocessing globally in case it is not supported
4003 # on the platform.
4004 import multiprocessing
4005 args.jobs = multiprocessing.cpu_count()
4007 if args.jobs > 1 and not (args.in_place or args.diff):
4008 _parser_error_with_code(
4009 parser,
4010 EXIT_CODE_ARGPARSE_ERROR,
4011 'parallel jobs requires --in-place',
4012 )
4014 if args.line_range:
4015 if args.line_range[0] <= 0:
4016 _parser_error_with_code(
4017 parser,
4018 EXIT_CODE_ARGPARSE_ERROR,
4019 '--range must be positive numbers',
4020 )
4021 if args.line_range[0] > args.line_range[1]:
4022 _parser_error_with_code(
4023 parser,
4024 EXIT_CODE_ARGPARSE_ERROR,
4025 'First value of --range should be less than or equal '
4026 'to the second',
4027 )
4029 original_formatwarning = warnings.formatwarning
4030 warnings.formatwarning = _custom_formatwarning
4031 if args.experimental:
4032 warnings.warn(
4033 "`experimental` option is deprecated and will be "
4034 "removed in a future version.",
4035 DeprecationWarning,
4036 )
4037 warnings.formatwarning = original_formatwarning
4039 return args
4042def _get_normalize_options(args, config, section, option_list):
4043 for (k, v) in config.items(section):
4044 norm_opt = k.lstrip('-').replace('-', '_')
4045 if not option_list.get(norm_opt):
4046 continue
4047 opt_type = option_list[norm_opt]
4048 if opt_type is int:
4049 if v.strip() == "auto":
4050 # skip to special case
4051 if args.verbose:
4052 print(f"ignore config: {k}={v}")
4053 continue
4054 value = config.getint(section, k)
4055 elif opt_type is bool:
4056 value = config.getboolean(section, k)
4057 else:
4058 value = config.get(section, k)
4059 yield norm_opt, k, value
4062def read_config(args, parser):
4063 """Read both user configuration and local configuration."""
4064 config = SafeConfigParser()
4066 try:
4067 if args.verbose and os.path.exists(args.global_config):
4068 print("read config path: {}".format(args.global_config))
4069 config.read(args.global_config)
4071 if not args.ignore_local_config:
4072 parent = tail = args.files and os.path.abspath(
4073 os.path.commonprefix(args.files))
4074 while tail:
4075 if config.read([os.path.join(parent, fn)
4076 for fn in PROJECT_CONFIG]):
4077 if args.verbose:
4078 for fn in PROJECT_CONFIG:
4079 config_file = os.path.join(parent, fn)
4080 if not os.path.exists(config_file):
4081 continue
4082 print(
4083 "read config path: {}".format(
4084 os.path.join(parent, fn)
4085 )
4086 )
4087 break
4088 (parent, tail) = os.path.split(parent)
4090 defaults = {}
4091 option_list = {o.dest: o.type or type(o.default)
4092 for o in parser._actions}
4094 for section in ['pep8', 'pycodestyle', 'flake8']:
4095 if not config.has_section(section):
4096 continue
4097 for norm_opt, k, value in _get_normalize_options(
4098 args, config, section, option_list
4099 ):
4100 if args.verbose:
4101 print("enable config: section={}, key={}, value={}".format(
4102 section, k, value))
4103 defaults[norm_opt] = value
4105 parser.set_defaults(**defaults)
4106 except Error:
4107 # Ignore for now.
4108 pass
4110 return parser
4113def read_pyproject_toml(args, parser):
4114 """Read pyproject.toml and load configuration."""
4115 if sys.version_info >= (3, 11):
4116 import tomllib
4117 else:
4118 import tomli as tomllib
4120 config = None
4122 if os.path.exists(args.global_config):
4123 with open(args.global_config, "rb") as fp:
4124 config = tomllib.load(fp)
4126 if not args.ignore_local_config:
4127 parent = tail = args.files and os.path.abspath(
4128 os.path.commonprefix(args.files))
4129 while tail:
4130 pyproject_toml = os.path.join(parent, "pyproject.toml")
4131 if os.path.exists(pyproject_toml):
4132 with open(pyproject_toml, "rb") as fp:
4133 config = tomllib.load(fp)
4134 break
4135 (parent, tail) = os.path.split(parent)
4137 if not config:
4138 return None
4140 if config.get("tool", {}).get("autopep8") is None:
4141 return None
4143 config = config.get("tool", {}).get("autopep8")
4145 defaults = {}
4146 option_list = {o.dest: o.type or type(o.default)
4147 for o in parser._actions}
4149 TUPLED_OPTIONS = ("ignore", "select")
4150 for (k, v) in config.items():
4151 norm_opt = k.lstrip('-').replace('-', '_')
4152 if not option_list.get(norm_opt):
4153 continue
4154 if type(v) in (list, tuple) and norm_opt in TUPLED_OPTIONS:
4155 value = ",".join(v)
4156 else:
4157 value = v
4158 if args.verbose:
4159 print("enable pyproject.toml config: "
4160 "key={}, value={}".format(k, value))
4161 defaults[norm_opt] = value
4163 if defaults:
4164 # set value when exists key-value in defaults dict
4165 parser.set_defaults(**defaults)
4167 return parser
4170def _split_comma_separated(string):
4171 """Return a set of strings."""
4172 return {text.strip() for text in string.split(',') if text.strip()}
4175def decode_filename(filename):
4176 """Return Unicode filename."""
4177 if isinstance(filename, str):
4178 return filename
4180 return filename.decode(sys.getfilesystemencoding())
4183def supported_fixes():
4184 """Yield pep8 error codes that autopep8 fixes.
4186 Each item we yield is a tuple of the code followed by its
4187 description.
4189 """
4190 yield ('E101', docstring_summary(reindent.__doc__))
4192 instance = FixPEP8(filename=None, options=None, contents='')
4193 for attribute in dir(instance):
4194 code = re.match('fix_([ew][0-9][0-9][0-9])', attribute)
4195 if code:
4196 yield (
4197 code.group(1).upper(),
4198 re.sub(r'\s+', ' ',
4199 docstring_summary(getattr(instance, attribute).__doc__))
4200 )
4202 for (code, function) in sorted(global_fixes()):
4203 yield (code.upper() + (4 - len(code)) * ' ',
4204 re.sub(r'\s+', ' ', docstring_summary(function.__doc__)))
4207def docstring_summary(docstring):
4208 """Return summary of docstring."""
4209 return docstring.split('\n')[0] if docstring else ''
4212def line_shortening_rank(candidate, indent_word, max_line_length,
4213 experimental=False):
4214 """Return rank of candidate.
4216 This is for sorting candidates.
4218 """
4219 if not candidate.strip():
4220 return 0
4222 rank = 0
4223 lines = candidate.rstrip().split('\n')
4225 offset = 0
4226 if (
4227 not lines[0].lstrip().startswith('#') and
4228 lines[0].rstrip()[-1] not in '([{'
4229 ):
4230 for (opening, closing) in ('()', '[]', '{}'):
4231 # Don't penalize empty containers that aren't split up. Things like
4232 # this "foo(\n )" aren't particularly good.
4233 opening_loc = lines[0].find(opening)
4234 closing_loc = lines[0].find(closing)
4235 if opening_loc >= 0:
4236 if closing_loc < 0 or closing_loc != opening_loc + 1:
4237 offset = max(offset, 1 + opening_loc)
4239 current_longest = max(offset + len(x.strip()) for x in lines)
4241 rank += 4 * max(0, current_longest - max_line_length)
4243 rank += len(lines)
4245 # Too much variation in line length is ugly.
4246 rank += 2 * standard_deviation(len(line) for line in lines)
4248 bad_staring_symbol = {
4249 '(': ')',
4250 '[': ']',
4251 '{': '}'}.get(lines[0][-1])
4253 if len(lines) > 1:
4254 if (
4255 bad_staring_symbol and
4256 lines[1].lstrip().startswith(bad_staring_symbol)
4257 ):
4258 rank += 20
4260 for lineno, current_line in enumerate(lines):
4261 current_line = current_line.strip()
4263 if current_line.startswith('#'):
4264 continue
4266 for bad_start in ['.', '%', '+', '-', '/']:
4267 if current_line.startswith(bad_start):
4268 rank += 100
4270 # Do not tolerate operators on their own line.
4271 if current_line == bad_start:
4272 rank += 1000
4274 if (
4275 current_line.endswith(('.', '%', '+', '-', '/')) and
4276 "': " in current_line
4277 ):
4278 rank += 1000
4280 if current_line.endswith(('(', '[', '{', '.')):
4281 # Avoid lonely opening. They result in longer lines.
4282 if len(current_line) <= len(indent_word):
4283 rank += 100
4285 # Avoid the ugliness of ", (\n".
4286 if (
4287 current_line.endswith('(') and
4288 current_line[:-1].rstrip().endswith(',')
4289 ):
4290 rank += 100
4292 # Avoid the ugliness of "something[\n" and something[index][\n.
4293 if (
4294 current_line.endswith('[') and
4295 len(current_line) > 1 and
4296 (current_line[-2].isalnum() or current_line[-2] in ']')
4297 ):
4298 rank += 300
4300 # Also avoid the ugliness of "foo.\nbar"
4301 if current_line.endswith('.'):
4302 rank += 100
4304 if has_arithmetic_operator(current_line):
4305 rank += 100
4307 # Avoid breaking at unary operators.
4308 if re.match(r'.*[(\[{]\s*[\-\+~]$', current_line.rstrip('\\ ')):
4309 rank += 1000
4311 if re.match(r'.*lambda\s*\*$', current_line.rstrip('\\ ')):
4312 rank += 1000
4314 if current_line.endswith(('%', '(', '[', '{')):
4315 rank -= 20
4317 # Try to break list comprehensions at the "for".
4318 if current_line.startswith('for '):
4319 rank -= 50
4321 if current_line.endswith('\\'):
4322 # If a line ends in \-newline, it may be part of a
4323 # multiline string. In that case, we would like to know
4324 # how long that line is without the \-newline. If it's
4325 # longer than the maximum, or has comments, then we assume
4326 # that the \-newline is an okay candidate and only
4327 # penalize it a bit.
4328 total_len = len(current_line)
4329 lineno += 1
4330 while lineno < len(lines):
4331 total_len += len(lines[lineno])
4333 if lines[lineno].lstrip().startswith('#'):
4334 total_len = max_line_length
4335 break
4337 if not lines[lineno].endswith('\\'):
4338 break
4340 lineno += 1
4342 if total_len < max_line_length:
4343 rank += 10
4344 else:
4345 rank += 100 if experimental else 1
4347 # Prefer breaking at commas rather than colon.
4348 if ',' in current_line and current_line.endswith(':'):
4349 rank += 10
4351 # Avoid splitting dictionaries between key and value.
4352 if current_line.endswith(':'):
4353 rank += 100
4355 rank += 10 * count_unbalanced_brackets(current_line)
4357 return max(0, rank)
4360def standard_deviation(numbers):
4361 """Return standard deviation."""
4362 numbers = list(numbers)
4363 if not numbers:
4364 return 0
4365 mean = sum(numbers) / len(numbers)
4366 return (sum((n - mean) ** 2 for n in numbers) /
4367 len(numbers)) ** .5
4370def has_arithmetic_operator(line):
4371 """Return True if line contains any arithmetic operators."""
4372 for operator in pycodestyle.ARITHMETIC_OP:
4373 if operator in line:
4374 return True
4376 return False
4379def count_unbalanced_brackets(line):
4380 """Return number of unmatched open/close brackets."""
4381 count = 0
4382 for opening, closing in ['()', '[]', '{}']:
4383 count += abs(line.count(opening) - line.count(closing))
4385 return count
4388def split_at_offsets(line, offsets):
4389 """Split line at offsets.
4391 Return list of strings.
4393 """
4394 result = []
4396 previous_offset = 0
4397 current_offset = 0
4398 for current_offset in sorted(offsets):
4399 if current_offset < len(line) and previous_offset != current_offset:
4400 result.append(line[previous_offset:current_offset].strip())
4401 previous_offset = current_offset
4403 result.append(line[current_offset:])
4405 return result
4408class LineEndingWrapper(object):
4410 r"""Replace line endings to work with sys.stdout.
4412 It seems that sys.stdout expects only '\n' as the line ending, no matter
4413 the platform. Otherwise, we get repeated line endings.
4415 """
4417 def __init__(self, output):
4418 self.__output = output
4420 def write(self, s):
4421 self.__output.write(s.replace('\r\n', '\n').replace('\r', '\n'))
4423 def flush(self):
4424 self.__output.flush()
4427def match_file(filename, exclude):
4428 """Return True if file is okay for modifying/recursing."""
4429 base_name = os.path.basename(filename)
4431 if base_name.startswith('.'):
4432 return False
4434 for pattern in exclude:
4435 if fnmatch.fnmatch(base_name, pattern):
4436 return False
4437 if fnmatch.fnmatch(filename, pattern):
4438 return False
4440 if not os.path.isdir(filename) and not is_python_file(filename):
4441 return False
4443 return True
4446def find_files(filenames, recursive, exclude):
4447 """Yield filenames."""
4448 while filenames:
4449 name = filenames.pop(0)
4450 if recursive and os.path.isdir(name):
4451 for root, directories, children in os.walk(name):
4452 filenames += [os.path.join(root, f) for f in children
4453 if match_file(os.path.join(root, f),
4454 exclude)]
4455 directories[:] = [d for d in directories
4456 if match_file(os.path.join(root, d),
4457 exclude)]
4458 else:
4459 is_exclude_match = False
4460 for pattern in exclude:
4461 if fnmatch.fnmatch(name, pattern):
4462 is_exclude_match = True
4463 break
4464 if not is_exclude_match:
4465 yield name
4468def _fix_file(parameters):
4469 """Helper function for optionally running fix_file() in parallel."""
4470 if parameters[1].verbose:
4471 print('[file:{}]'.format(parameters[0]), file=sys.stderr)
4472 try:
4473 return fix_file(*parameters)
4474 except IOError as error:
4475 print(str(error), file=sys.stderr)
4476 raise error
4479def fix_multiple_files(filenames, options, output=None):
4480 """Fix list of files.
4482 Optionally fix files recursively.
4484 """
4485 results = []
4486 filenames = find_files(filenames, options.recursive, options.exclude)
4487 if options.jobs > 1:
4488 import multiprocessing
4489 pool = multiprocessing.Pool(options.jobs)
4490 rets = []
4491 for name in filenames:
4492 ret = pool.apply_async(_fix_file, ((name, options),))
4493 rets.append(ret)
4494 pool.close()
4495 pool.join()
4496 if options.diff:
4497 for r in rets:
4498 sys.stdout.write(r.get().decode())
4499 sys.stdout.flush()
4500 results.extend([x.get() for x in rets if x is not None])
4501 else:
4502 for name in filenames:
4503 ret = _fix_file((name, options, output))
4504 if ret is None:
4505 continue
4506 if options.diff:
4507 if ret != '':
4508 results.append(ret)
4509 elif options.in_place:
4510 results.append(ret)
4511 else:
4512 original_source = readlines_from_file(name)
4513 if "".join(original_source).splitlines() != ret.splitlines():
4514 results.append(ret)
4515 return results
4518def is_python_file(filename):
4519 """Return True if filename is Python file."""
4520 if filename.endswith('.py'):
4521 return True
4523 try:
4524 with open_with_encoding(
4525 filename,
4526 limit_byte_check=MAX_PYTHON_FILE_DETECTION_BYTES) as f:
4527 text = f.read(MAX_PYTHON_FILE_DETECTION_BYTES)
4528 if not text:
4529 return False
4530 first_line = text.splitlines()[0]
4531 except (IOError, IndexError):
4532 return False
4534 if not PYTHON_SHEBANG_REGEX.match(first_line):
4535 return False
4537 return True
4540def is_probably_part_of_multiline(line):
4541 """Return True if line is likely part of a multiline string.
4543 When multiline strings are involved, pep8 reports the error as being
4544 at the start of the multiline string, which doesn't work for us.
4546 """
4547 return (
4548 '"""' in line or
4549 "'''" in line or
4550 line.rstrip().endswith('\\')
4551 )
4554def wrap_output(output, encoding):
4555 """Return output with specified encoding."""
4556 return codecs.getwriter(encoding)(output.buffer
4557 if hasattr(output, 'buffer')
4558 else output)
4561def get_encoding():
4562 """Return preferred encoding."""
4563 return locale.getpreferredencoding() or sys.getdefaultencoding()
4566def main(argv=None, apply_config=True):
4567 """Command-line entry."""
4568 if argv is None:
4569 argv = sys.argv
4571 try:
4572 # Exit on broken pipe.
4573 signal.signal(signal.SIGPIPE, signal.SIG_DFL)
4574 except AttributeError: # pragma: no cover
4575 # SIGPIPE is not available on Windows.
4576 pass
4578 try:
4579 args = parse_args(argv[1:], apply_config=apply_config)
4581 if args.list_fixes:
4582 for code, description in sorted(supported_fixes()):
4583 print('{code} - {description}'.format(
4584 code=code, description=description))
4585 return EXIT_CODE_OK
4587 if args.files == ['-']:
4588 assert not args.in_place
4590 encoding = sys.stdin.encoding or get_encoding()
4591 read_stdin = sys.stdin.read()
4592 fixed_stdin = fix_code(read_stdin, args, encoding=encoding)
4594 # LineEndingWrapper is unnecessary here due to the symmetry between
4595 # standard in and standard out.
4596 wrap_output(sys.stdout, encoding=encoding).write(fixed_stdin)
4598 if hash(read_stdin) != hash(fixed_stdin):
4599 if args.exit_code:
4600 return EXIT_CODE_EXISTS_DIFF
4601 else:
4602 if args.in_place or args.diff:
4603 args.files = list(set(args.files))
4604 else:
4605 assert len(args.files) == 1
4606 assert not args.recursive
4608 results = fix_multiple_files(args.files, args, sys.stdout)
4609 if args.diff:
4610 ret = any([len(ret) != 0 for ret in results])
4611 else:
4612 # with in-place option
4613 ret = any([ret is not None for ret in results])
4614 if args.exit_code and ret:
4615 return EXIT_CODE_EXISTS_DIFF
4616 except IOError:
4617 return EXIT_CODE_ERROR
4618 except KeyboardInterrupt:
4619 return EXIT_CODE_ERROR # pragma: no cover
4622class CachedTokenizer(object):
4624 """A one-element cache around tokenize.generate_tokens().
4626 Original code written by Ned Batchelder, in coverage.py.
4628 """
4630 def __init__(self):
4631 self.last_text = None
4632 self.last_tokens = None
4634 def generate_tokens(self, text):
4635 """A stand-in for tokenize.generate_tokens()."""
4636 if text != self.last_text:
4637 string_io = io.StringIO(text)
4638 self.last_tokens = list(
4639 tokenize.generate_tokens(string_io.readline)
4640 )
4641 self.last_text = text
4642 return self.last_tokens
4645_cached_tokenizer = CachedTokenizer()
4646generate_tokens = _cached_tokenizer.generate_tokens
4649if __name__ == '__main__':
4650 sys.exit(main())