frozenset([',']),

frozenset(['%']),

frozenset([',', '(', '[', '{']),

frozenset(['%', '(', '[', '{']),

frozenset([',', '(', '[', '{', '%', '+', '-', '*', '/', '//']),

frozenset(['%', '+', '-', '*', '/', '//']),

'E721': ['idioms'],

'W601': ['has_key'],

'W603': ['ne'],

'W604': ['repr'],

'W690': ['apply',

'except',

'exitfunc',

'import',

'numliterals',

'operator',

'paren',

'reduce',

'renames',

'standarderror',

'sys_exc',

'throw',

'tuple_params',

"""Return opened file with a specific encoding."""

if not encoding:

encoding = detect_encoding(filename)

return io.open(filename, mode=mode, encoding=encoding,

"""Return file encoding."""

try:

with open(filename, 'rb') as input_file:

from lib2to3.pgen2 import tokenize as lib2to3_tokenize

encoding = lib2to3_tokenize.detect_encoding(input_file.readline)[0]

# Check for correctness of encoding

with open_with_encoding(filename, encoding) as test_file:

test_file.read()

return encoding

except (LookupError, SyntaxError, UnicodeDecodeError):

blank_lines,

indent_level,

previous_logical):

"""Check for missing blank lines after class declaration."""

if previous_logical.startswith('class '):

if (

logical_line.startswith(('def ', 'class ', '@')) or

pep8.DOCSTRING_REGEX.match(logical_line)

):

if indent_level and not blank_lines:

yield (0, 'E309 expected 1 blank line after class declaration')

elif previous_logical.startswith('def '):

if blank_lines and pep8.DOCSTRING_REGEX.match(logical_line):

yield (0, 'E303 too many blank lines ({0})'.format(blank_lines))

elif pep8.DOCSTRING_REGEX.match(previous_logical):

# Missing blank line between class docstring and method declaration.

if (

indent_level and

not blank_lines and

logical_line.startswith(('def ')) and

'(self' in logical_line

):

"""Override pep8's function to provide indentation information."""

first_row = tokens[0][2][0]

nrows = 1 + tokens[-1][2][0] - first_row

if noqa or nrows == 1:

return

# indent_next tells us whether the next block is indented. Assuming

# that it is indented by 4 spaces, then we should not allow 4-space

# indents on the final continuation line. In turn, some other

# indents are allowed to have an extra 4 spaces.

indent_next = logical_line.endswith(':')

row = depth = 0

# Remember how many brackets were opened on each line.

parens = [0] * nrows

# Relative indents of physical lines.

rel_indent = [0] * nrows

# Visual indents.

indent_chances = {}

last_indent = tokens[0][2]

indent = [last_indent[1]]

last_token_multiline = None

line = None

last_line = ''

last_line_begins_with_multiline = False

for token_type, text, start, end, line in tokens:

newline = row < start[0] - first_row

if newline:

row = start[0] - first_row

newline = (not last_token_multiline and

token_type not in (tokenize.NL, tokenize.NEWLINE))

last_line_begins_with_multiline = last_token_multiline

if newline:

# This is the beginning of a continuation line.

last_indent = start

# Record the initial indent.

rel_indent[row] = pep8.expand_indent(line) - indent_level

if depth:

# A bracket expression in a continuation line.

# Find the line that it was opened on.

for open_row in range(row - 1, -1, -1):

if parens[open_row]:

break

else:

# An unbracketed continuation line (ie, backslash).

open_row = 0

hang = rel_indent[row] - rel_indent[open_row]

close_bracket = (token_type == tokenize.OP and text in ']})')

visual_indent = (not close_bracket and hang > 0 and

indent_chances.get(start[1]))

if close_bracket and indent[depth]:

# Closing bracket for visual indent.

if start[1] != indent[depth]:

yield (start, 'E124 {0}'.format(indent[depth]))

elif close_bracket and not hang:

pass

elif visual_indent is True:

# Visual indent is verified.

if not indent[depth]:

indent[depth] = start[1]

elif visual_indent in (text, unicode):

# Ignore token lined up with matching one from a previous line.

pass

elif indent[depth] and start[1] < indent[depth]:

# Visual indent is broken.

yield (start, 'E128 {0}'.format(indent[depth]))

elif (hang == DEFAULT_INDENT_SIZE or

(indent_next and

rel_indent[row] == 2 * DEFAULT_INDENT_SIZE)):

# Hanging indent is verified.

if close_bracket:

yield (start, 'E123 {0}'.format(indent_level +

rel_indent[open_row]))

else:

one_indented = (indent_level + rel_indent[open_row] +

DEFAULT_INDENT_SIZE)

# Indent is broken.

if hang <= 0:

error = ('E122', one_indented)

elif indent[depth]:

error = ('E127', indent[depth])

elif hang % DEFAULT_INDENT_SIZE:

error = ('E121', one_indented)

else:

error = ('E126', one_indented)

yield (start, '{0} {1}'.format(*error))

# Look for visual indenting.

if (parens[row] and token_type not in (tokenize.NL, tokenize.COMMENT)

and not indent[depth]):

indent[depth] = start[1]

indent_chances[start[1]] = True

# Deal with implicit string concatenation.

elif (token_type in (tokenize.STRING, tokenize.COMMENT) or

text in ('u', 'ur', 'b', 'br')):

indent_chances[start[1]] = unicode

# Special case for the "if" statement because len("if (") is equal to

# 4.

elif not indent_chances and not row and not depth and text == 'if':

indent_chances[end[1] + 1] = True

# Keep track of bracket depth.

if token_type == tokenize.OP:

if text in '([{':

depth += 1

indent.append(0)

parens[row] += 1

elif text in ')]}' and depth > 0:

# Parent indents should not be more than this one.

prev_indent = indent.pop() or last_indent[1]

for d in range(depth):

if indent[d] > prev_indent:

indent[d] = 0

for ind in list(indent_chances):

if ind >= prev_indent:

del indent_chances[ind]

depth -= 1

if depth:

indent_chances[indent[depth]] = True

for idx in range(row, -1, -1):

if parens[idx]:

parens[idx] -= 1

rel_indent[row] = rel_indent[idx]

break

assert len(indent) == depth + 1

if (

start[1] not in indent_chances and

# This is for purposes of speeding up E121 (GitHub #90).

not last_line.rstrip().endswith(',')

):

# Allow to line up tokens.

indent_chances[start[1]] = text

last_token_multiline = (start[0] != end[0])

last_line = line

if (

indent_next and

not last_line_begins_with_multiline and

pep8.expand_indent(line) == indent_level + DEFAULT_INDENT_SIZE

):

yield (last_indent, 'E125 {0}'.format(indent_level +

"""Fix invalid code.

def __init__(self, filename,

options,

contents=None,

long_line_ignore_cache=None):

self.filename = filename

if contents is None:

self.source = readlines_from_file(filename)

else:

sio = io.StringIO(contents)

self.source = sio.readlines()

self.options = options

self.indent_word = _get_indentword(''.join(self.source))

self.long_line_ignore_cache = (

set() if long_line_ignore_cache is None

else long_line_ignore_cache)

# method definition

self.fix_e121 = self._fix_reindent

self.fix_e122 = self._fix_reindent

self.fix_e123 = self._fix_reindent

self.fix_e124 = self._fix_reindent

self.fix_e126 = self._fix_reindent

self.fix_e127 = self._fix_reindent

self.fix_e128 = self._fix_reindent

self.fix_e129 = self._fix_reindent

self.fix_e202 = self.fix_e201

self.fix_e203 = self.fix_e201

self.fix_e211 = self.fix_e201

self.fix_e221 = self.fix_e271

self.fix_e222 = self.fix_e271

self.fix_e223 = self.fix_e271

self.fix_e226 = self.fix_e225

self.fix_e227 = self.fix_e225

self.fix_e228 = self.fix_e225

self.fix_e241 = self.fix_e271

self.fix_e242 = self.fix_e224

self.fix_e261 = self.fix_e262

self.fix_e272 = self.fix_e271

self.fix_e273 = self.fix_e271

self.fix_e274 = self.fix_e271

self.fix_e309 = self.fix_e301

self.fix_e501 = (

self.fix_long_line_logically if

options and (options.aggressive >= 2 or options.experimental) else

self.fix_long_line_physically)

self.fix_e703 = self.fix_e702

self._ws_comma_done = False

def _fix_source(self, results):

try:

(logical_start, logical_end) = _find_logical(self.source)

logical_support = True

except (SyntaxError, tokenize.TokenError): # pragma: no cover

logical_support = False

completed_lines = set()

for result in sorted(results, key=_priority_key):

if result['line'] in completed_lines:

continue

fixed_methodname = 'fix_' + result['id'].lower()

if hasattr(self, fixed_methodname):

fix = getattr(self, fixed_methodname)

line_index = result['line'] - 1

original_line = self.source[line_index]

is_logical_fix = len(inspect.getargspec(fix).args) > 2

if is_logical_fix:

logical = None

if logical_support:

logical = _get_logical(self.source,

result,

logical_start,

logical_end)

if logical and set(range(

logical[0][0] + 1,

logical[1][0] + 1)).intersection(

completed_lines):

continue

modified_lines = fix(result, logical)

else:

modified_lines = fix(result)

if modified_lines is None:

# Force logical fixes to report what they modified.

assert not is_logical_fix

if self.source[line_index] == original_line:

modified_lines = []

if modified_lines:

completed_lines.update(modified_lines)

elif modified_lines == []: # Empty list means no fix

if self.options.verbose >= 2:

print(

'---> Not fixing {f} on line {l}'.format(

f=result['id'], l=result['line']),

file=sys.stderr)

else: # We assume one-line fix when None.

completed_lines.add(result['line'])

else:

if self.options.verbose >= 3:

print(

"---> '{0}' is not defined.".format(fixed_methodname),

file=sys.stderr)

info = result['info'].strip()

print('---> {0}:{1}:{2}:{3}'.format(self.filename,

result['line'],

result['column'],

info),

file=sys.stderr)

def fix(self):

"""Return a version of the source code with PEP 8 violations fixed."""

pep8_options = {

'ignore': self.options.ignore,

'select': self.options.select,

'max_line_length': self.options.max_line_length,

}

results = _execute_pep8(pep8_options, self.source)

if self.options.verbose:

progress = {}

for r in results:

if r['id'] not in progress:

progress[r['id']] = set()

progress[r['id']].add(r['line'])

print('---> {n} issue(s) to fix {progress}'.format(

n=len(results), progress=progress), file=sys.stderr)

if self.options.line_range:

results = [

r for r in results

if self.options.line_range[0] <= r['line'] <=

self.options.line_range[1]]

self._fix_source(filter_results(source=''.join(self.source),

results=results,

aggressive=self.options.aggressive,

indent_size=self.options.indent_size))

return ''.join(self.source)

def _fix_reindent(self, result):

"""Fix a badly indented line.

num_indent_spaces = int(result['info'].split()[1])

line_index = result['line'] - 1

target = self.source[line_index]

self.source[line_index] = ' ' * num_indent_spaces + target.lstrip()

def fix_e125(self, result):

"""Fix indentation undistinguish from the next logical line."""

num_indent_spaces = int(result['info'].split()[1])

line_index = result['line'] - 1

target = self.source[line_index]

spaces_to_add = num_indent_spaces - len(_get_indentation(target))

indent = len(_get_indentation(target))

modified_lines = []

while len(_get_indentation(self.source[line_index])) >= indent:

self.source[line_index] = (' ' * spaces_to_add +

self.source[line_index])

modified_lines.append(1 + line_index) # Line indexed at 1.

line_index -= 1

return modified_lines

def fix_e201(self, result):

"""Remove extraneous whitespace."""

line_index = result['line'] - 1

target = self.source[line_index]

offset = result['column'] - 1

if is_probably_part_of_multiline(target):

return []

fixed = fix_whitespace(target,

offset=offset,

replacement='')

self.source[line_index] = fixed

def fix_e224(self, result):

"""Remove extraneous whitespace around operator."""

target = self.source[result['line'] - 1]

offset = result['column'] - 1

fixed = target[:offset] + target[offset:].replace('\t', ' ')

self.source[result['line'] - 1] = fixed

def fix_e225(self, result):

"""Fix missing whitespace around operator."""

target = self.source[result['line'] - 1]

offset = result['column'] - 1

fixed = target[:offset] + ' ' + target[offset:]

# Only proceed if non-whitespace characters match.

# And make sure we don't break the indentation.

if (

fixed.replace(' ', '') == target.replace(' ', '') and

_get_indentation(fixed) == _get_indentation(target)

):

self.source[result['line'] - 1] = fixed

else:

return []

def fix_e231(self, result):

"""Add missing whitespace."""

# Optimize for comma case. This will fix all commas in the full source

# code in one pass. Don't do this more than once. If it fails the first

# time, there is no point in trying again.

if ',' in result['info'] and not self._ws_comma_done:

self._ws_comma_done = True

original = ''.join(self.source)

new = refactor(original, ['ws_comma'])

if original.strip() != new.strip():

self.source = [new]

return range(1, 1 + len(original))

line_index = result['line'] - 1

target = self.source[line_index]

offset = result['column']

fixed = target[:offset] + ' ' + target[offset:]

self.source[line_index] = fixed

def fix_e251(self, result):

"""Remove whitespace around parameter '=' sign."""

line_index = result['line'] - 1

target = self.source[line_index]

# This is necessary since pep8 sometimes reports columns that goes

# past the end of the physical line. This happens in cases like,

# foo(bar\n=None)

c = min(result['column'] - 1,

len(target) - 1)

if target[c].strip():

fixed = target

else:

fixed = target[:c].rstrip() + target[c:].lstrip()

# There could be an escaped newline

#

# def foo(a=\

# 1)

if fixed.endswith(('=\\\n', '=\\\r\n', '=\\\r')):

self.source[line_index] = fixed.rstrip('\n\r \t\\')

self.source[line_index + 1] = self.source[line_index + 1].lstrip()

return [line_index + 1, line_index + 2] # Line indexed at 1

self.source[result['line'] - 1] = fixed

def fix_e262(self, result):

"""Fix spacing after comment hash."""

target = self.source[result['line'] - 1]

offset = result['column']

code = target[:offset].rstrip(' \t#')

comment = target[offset:].lstrip(' \t#')

fixed = code + (' # ' + comment if comment.strip() else '\n')

self.source[result['line'] - 1] = fixed

def fix_e271(self, result):

"""Fix extraneous whitespace around keywords."""

line_index = result['line'] - 1

target = self.source[line_index]

offset = result['column'] - 1

if is_probably_part_of_multiline(target):

return []

fixed = fix_whitespace(target,

offset=offset,

replacement=' ')

if fixed == target:

return []

else:

self.source[line_index] = fixed

def fix_e301(self, result):

"""Add missing blank line."""

cr = '\n'

self.source[result['line'] - 1] = cr + self.source[result['line'] - 1]

def fix_e302(self, result):

"""Add missing 2 blank lines."""

add_linenum = 2 - int(result['info'].split()[-1])

cr = '\n' * add_linenum

self.source[result['line'] - 1] = cr + self.source[result['line'] - 1]

def fix_e303(self, result):

"""Remove extra blank lines."""

delete_linenum = int(result['info'].split('(')[1].split(')')[0]) - 2

delete_linenum = max(1, delete_linenum)

# We need to count because pep8 reports an offset line number if there

# are comments.

cnt = 0

line = result['line'] - 2

modified_lines = []

while cnt < delete_linenum and line >= 0:

if not self.source[line].strip():

self.source[line] = ''

modified_lines.append(1 + line) # Line indexed at 1

cnt += 1

line -= 1

return modified_lines

def fix_e304(self, result):

"""Remove blank line following function decorator."""

line = result['line'] - 2

if not self.source[line].strip():

self.source[line] = ''

def fix_e401(self, result):

"""Put imports on separate lines."""

line_index = result['line'] - 1

target = self.source[line_index]

offset = result['column'] - 1

if not target.lstrip().startswith('import'):

return []

indentation = re.split(pattern=r'\bimport\b',

string=target, maxsplit=1)[0]

fixed = (target[:offset].rstrip('\t ,') + '\n' +

indentation + 'import ' + target[offset:].lstrip('\t ,'))

self.source[line_index] = fixed

def fix_long_line_logically(self, result, logical):

"""Try to make lines fit within --max-line-length characters."""

if (

not logical or

len(logical[2]) == 1 or

self.source[result['line'] - 1].lstrip().startswith('#')

):

return self.fix_long_line_physically(result)

start_line_index = logical[0][0]

end_line_index = logical[1][0]

logical_lines = logical[2]

previous_line = get_item(self.source, start_line_index - 1, default='')

next_line = get_item(self.source, end_line_index + 1, default='')

single_line = join_logical_line(''.join(logical_lines))

try:

fixed = self.fix_long_line(

target=single_line,

previous_line=previous_line,

next_line=next_line,

original=''.join(logical_lines))

except (SyntaxError, tokenize.TokenError):

return self.fix_long_line_physically(result)

if fixed:

for line_index in range(start_line_index, end_line_index + 1):

self.source[line_index] = ''

self.source[start_line_index] = fixed

return range(start_line_index + 1, end_line_index + 1)

else:

return []

def fix_long_line_physically(self, result):

"""Try to make lines fit within --max-line-length characters."""

line_index = result['line'] - 1

target = self.source[line_index]

previous_line = get_item(self.source, line_index - 1, default='')

next_line = get_item(self.source, line_index + 1, default='')

try:

fixed = self.fix_long_line(

target=target,

previous_line=previous_line,

next_line=next_line,

original=target)

except (SyntaxError, tokenize.TokenError):

return []

if fixed:

self.source[line_index] = fixed

return [line_index + 1]

else:

return []

def fix_long_line(self, target, previous_line,

next_line, original):

cache_entry = (target, previous_line, next_line)

if cache_entry in self.long_line_ignore_cache:

return []

if target.lstrip().startswith('#'):

# Wrap commented lines.

return shorten_comment(

line=target,

max_line_length=self.options.max_line_length,

last_comment=not next_line.lstrip().startswith('#'))

fixed = get_fixed_long_line(

target=target,

previous_line=previous_line,

original=original,

indent_word=self.indent_word,

max_line_length=self.options.max_line_length,

aggressive=self.options.aggressive,

experimental=self.options.experimental,

verbose=self.options.verbose)

if fixed and not code_almost_equal(original, fixed):

return fixed

else:

self.long_line_ignore_cache.add(cache_entry)

return None

def fix_e502(self, result):

"""Remove extraneous escape of newline."""

line_index = result['line'] - 1

target = self.source[line_index]

self.source[line_index] = target.rstrip('\n\r \t\\') + '\n'

def fix_e701(self, result):

"""Put colon-separated compound statement on separate lines."""

line_index = result['line'] - 1

target = self.source[line_index]

c = result['column']

fixed_source = (target[:c] + '\n' +

_get_indentation(target) + self.indent_word +

target[c:].lstrip('\n\r \t\\'))

self.source[result['line'] - 1] = fixed_source

return [result['line'], result['line'] + 1]

def fix_e702(self, result, logical):

"""Put semicolon-separated compound statement on separate lines."""

if not logical:

return [] # pragma: no cover

logical_lines = logical[2]

line_index = result['line'] - 1

target = self.source[line_index]

if target.rstrip().endswith('\\'):

# Normalize '1; \\\n2' into '1; 2'.

self.source[line_index] = target.rstrip('\n \r\t\\')

self.source[line_index + 1] = self.source[line_index + 1].lstrip()

return [line_index + 1, line_index + 2]

if target.rstrip().endswith(';'):

self.source[line_index] = target.rstrip('\n \r\t;') + '\n'

return [line_index + 1]

offset = result['column'] - 1

first = target[:offset].rstrip(';').rstrip()

second = (_get_indentation(logical_lines[0]) +

target[offset:].lstrip(';').lstrip())

self.source[line_index] = first + '\n' + second

return [line_index + 1]

def fix_e711(self, result):

"""Fix comparison with None."""

line_index = result['line'] - 1

target = self.source[line_index]

offset = result['column'] - 1

right_offset = offset + 2

if right_offset >= len(target):

return []

left = target[:offset].rstrip()

center = target[offset:right_offset]

right = target[right_offset:].lstrip()

if not right.startswith('None'):

return []

if center.strip() == '==':

new_center = 'is'

elif center.strip() == '!=':

new_center = 'is not'

else:

return []

self.source[line_index] = ' '.join([left, new_center, right])

def fix_e712(self, result):

"""Fix comparison with boolean."""

line_index = result['line'] - 1

target = self.source[line_index]

offset = result['column'] - 1

# Handle very easy "not" special cases.

if re.match(r'^\s*if \w+ == False:$', target):

self.source[line_index] = re.sub(r'if (\w+) == False:',

r'if not \1:', target, count=1)

elif re.match(r'^\s*if \w+ != True:$', target):

self.source[line_index] = re.sub(r'if (\w+) != True:',

r'if not \1:', target, count=1)

else:

right_offset = offset + 2

if right_offset >= len(target):

return []

left = target[:offset].rstrip()

center = target[offset:right_offset]

right = target[right_offset:].lstrip()

# Handle simple cases only.

new_right = None

if center.strip() == '==':

if re.match(r'\bTrue\b', right):

new_right = re.sub(r'\bTrue\b *', '', right, count=1)

elif center.strip() == '!=':

if re.match(r'\bFalse\b', right):

new_right = re.sub(r'\bFalse\b *', '', right, count=1)

if new_right is None:

return []

if new_right[0].isalnum():

new_right = ' ' + new_right

self.source[line_index] = left + new_right

def fix_w291(self, result):

"""Remove trailing whitespace."""

fixed_line = self.source[result['line'] - 1].rstrip()

indent_word=' ', max_line_length=79,

aggressive=False, experimental=False, verbose=False):

indent = _get_indentation(target)

source = target[len(indent):]

assert source.lstrip() == source

# Check for partial multiline.

tokens = list(generate_tokens(source))

candidates = shorten_line(

tokens, source, indent,

indent_word,

max_line_length,

aggressive=aggressive,

experimental=experimental,

previous_line=previous_line)

# Also sort alphabetically as a tie breaker (for determinism).

candidates = sorted(

sorted(set(candidates).union([target, original])),

key=lambda x: line_shortening_rank(x,

indent_word,

max_line_length))

if verbose >= 4:

print(('-' * 79 + '\n').join([''] + candidates + ['']),

file=codecs.getwriter('utf-8')(sys.stderr.buffer

if hasattr(sys.stderr,

'buffer')

else sys.stderr))

if candidates:

"""Return single line based on logical line input."""

indentation = _get_indentation(logical_line)

return indentation + untokenize_without_newlines(

"""Return source code based on tokens."""

text = ''

last_row = 0

last_column = -1

for t in tokens:

token_string = t[1]

(start_row, start_column) = t[2]

(end_row, end_column) = t[3]

if start_row > last_row:

last_column = 0

if (

(start_column > last_column or token_string == '\n') and

not text.endswith(' ')

):

text += ' '

if token_string != '\n':

text += token_string

last_row = end_row

last_column = end_column

# make a variable which is the index of all the starts of lines

logical_start = []

logical_end = []

last_newline = True

parens = 0

for t in generate_tokens(''.join(source_lines)):

if t[0] in [tokenize.COMMENT, tokenize.DEDENT,

tokenize.INDENT, tokenize.NL,

tokenize.ENDMARKER]:

continue

if not parens and t[0] in [tokenize.NEWLINE, tokenize.SEMI]:

last_newline = True

logical_end.append((t[3][0] - 1, t[2][1]))

continue

if last_newline and not parens:

logical_start.append((t[2][0] - 1, t[2][1]))