This repository was archived by the owner on Oct 23, 2024. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 6
Expand file tree
/
Copy pathcheck_docs.py
More file actions
executable file
·259 lines (206 loc) · 9.64 KB
/
check_docs.py
File metadata and controls
executable file
·259 lines (206 loc) · 9.64 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
#!/usr/bin/env python
# -*- coding: utf8 -*-
"""A script to ensure that our docs are not being utterly neglected."""
import argparse
import os
import sys
IGNORES = {
'pydir': ['tests'],
'pyfile': ['__init__.py'],
'docfile': ['index.rst'],
}
class AddDocIgnores(argparse.Action):
"""Add entries to docfile ignores list."""
def __call__(self, parser, namespace, values, option_string=None):
"""Add entries to docfile ignores list."""
global IGNORES
ignores = values.split(',')
IGNORES['docfile'] += ignores
setattr(namespace, 'doc_ignores', ignores)
class DocParityCheck(object):
"""Ensure proper python module and documentation parity."""
def __init__(self):
self._args = None
@property
def args(self):
"""Parsed command-line arguments."""
if self._args is None:
parser = self._build_parser()
self._args = parser.parse_args()
return self._args
def build_pypackage_basename(self, pytree, base):
"""Build the string representing the parsed package basename.
:param str pytree: The pytree absolute path.
:param str pytree: The absolute path of the pytree sub-package of which determine the
parsed name.
:rtype: str
"""
dirname = os.path.dirname(pytree)
parsed_package_name = base.replace(dirname, '').strip('/')
return parsed_package_name
def _build_parser(self):
"""Build the needed command-line parser."""
parser = argparse.ArgumentParser()
parser.add_argument('--pytree',
required=True,
type=self._valid_directory,
help='This is the path, absolute or relative, of the Python package '
'that is to be parsed.')
parser.add_argument('--doctree',
required=True,
type=self._valid_directory,
help='This is the path, absolute or relative, of the documentation '
'package that is to be parsed.')
parser.add_argument('--no-fail',
action='store_true',
help='Using this option will cause this program to return an exit '
'code of 0 even when the given trees do not match.')
parser.add_argument('--doc-ignores',
action=AddDocIgnores,
help='A comma separated list of additional doc files to ignore')
return parser
def build_rst_name_from_pypath(self, parsed_pypath):
"""Build the expected rst file name based on the parsed Python module path.
:param str parsed_pypath: The parsed Python module path from which to build the expected
rst file name.
:rtype: str
"""
expected_rst_name = parsed_pypath.replace('/', '.').replace('.py', '.rst')
return expected_rst_name
def build_pyfile_path_from_docname(self, docfile):
"""Build the expected Python file name based on the given documentation file name.
:param str docfile: The documentation file name from which to build the Python file name.
:rtype: str
"""
name, ext = os.path.splitext(docfile)
expected_py_name = name.replace('.', '/') + '.py'
return expected_py_name
def calculate_tree_differences(self, pytree, doctree):
"""Calculate the differences between the given trees.
:param dict pytree: The dictionary of the parsed Python tree.
:param dict doctree: The dictionary of the parsed documentation tree.
:rtype: tuple
:returns: A two-tuple of sets, where the first is the missing Python files, and the second
is the missing documentation files.
"""
pykeys = set(pytree.keys())
dockeys = set(doctree.keys())
# Calculate the missing documentation files, if any.
missing_doc_keys = pykeys - dockeys
missing_docs = {pytree[pyfile] for pyfile in missing_doc_keys}
# Calculate the missing Python files, if any.
missing_py_keys = dockeys - pykeys
missing_pys = {docfile for docfile in missing_py_keys}
return missing_pys, missing_docs
def compare_trees(self, parsed_pytree, parsed_doctree):
"""Compare the given parsed trees.
:param dict parsed_pytree: A dictionary representing the parsed Python tree where each
key is a parsed Python file and its key is its expected rst file name.
"""
if parsed_pytree == parsed_doctree:
return 0
missing_pys, missing_docs = self.calculate_tree_differences(pytree=parsed_pytree,
doctree=parsed_doctree)
self.pprint_tree_differences(missing_pys=missing_pys, missing_docs=missing_docs)
return 0 if self.args.no_fail else 1
def _ignore_docfile(self, filename):
"""Test if a documentation filename should be ignored.
:param str filename: The documentation file name to test.
:rtype: bool
"""
if filename in IGNORES['docfile'] or not filename.endswith('.rst'):
return True
return False
def _ignore_pydir(self, basename):
"""Test if a Python directory should be ignored.
:param str filename: The directory name to test.
:rtype: bool
"""
if basename in IGNORES['pydir']:
return True
return False
def _ignore_pyfile(self, filename):
"""Test if a Python filename should be ignored.
:param str filename: The Python file name to test.
:rtype: bool
"""
if filename in IGNORES['pyfile'] or not filename.endswith('.py'):
return True
return False
def parse_doc_tree(self, doctree, pypackages):
"""Parse the given documentation tree.
:param str doctree: The absolute path to the documentation tree which is to be parsed.
:param set pypackages: A set of all Python packages found in the pytree.
:rtype: dict
:returns: A dict where each key is the path of an expected Python module and its value is
the parsed rst module name (relative to the documentation tree).
"""
parsed_doctree = {}
for filename in os.listdir(doctree):
if self._ignore_docfile(filename):
continue
expected_pyfile = self.build_pyfile_path_from_docname(filename)
parsed_doctree[expected_pyfile] = filename
pypackages = {name + '.py' for name in pypackages}
return {elem: parsed_doctree[elem] for elem in parsed_doctree if elem not in pypackages}
def parse_py_tree(self, pytree):
"""Parse the given Python package tree.
:param str pytree: The absolute path to the Python tree which is to be parsed.
:rtype: dict
:returns: A two-tuple. The first element is a dict where each key is the path of a parsed
Python module (relative to the Python tree) and its value is the expected rst module
name. The second element is a set where each element is a Python package or
sub-package.
:rtype: tuple
"""
parsed_pytree = {}
pypackages = set()
for base, dirs, files in os.walk(pytree):
if self._ignore_pydir(os.path.basename(base)):
continue
# TODO(Anthony): If this is being run against a Python 3 package, this needs to be
# adapted to account for namespace packages.
elif '__init__.py' not in files:
continue
package_basename = self.build_pypackage_basename(pytree=pytree, base=base)
pypackages.add(package_basename)
for filename in files:
if self._ignore_pyfile(filename):
continue
parsed_path = os.path.join(package_basename, filename)
parsed_pytree[parsed_path] = self.build_rst_name_from_pypath(parsed_path)
return parsed_pytree, pypackages
def pprint_tree_differences(self, missing_pys, missing_docs):
"""Pprint the missing files of each given set.
:param set missing_pys: The set of missing Python files.
:param set missing_docs: The set of missing documentation files.
:rtype: None
"""
if missing_pys:
print('The following Python files appear to be missing:')
for pyfile in missing_pys:
print(pyfile)
print('\n')
if missing_docs:
print('The following documentation files appear to be missing:')
for docfiile in missing_docs:
print(docfiile)
print('\n')
def _valid_directory(self, path):
"""Ensure that the given path is valid.
:param str path: A valid directory path.
:raises: :py:class:`argparse.ArgumentTypeError`
:returns: An absolute directory path.
"""
abspath = os.path.abspath(path)
if not os.path.isdir(abspath):
raise argparse.ArgumentTypeError('Not a valid directory: {}'.format(abspath))
return abspath
def main(self):
"""Parse package trees and report on any discrepancies."""
args = self.args
parsed_pytree, pypackages = self.parse_py_tree(pytree=args.pytree)
parsed_doctree = self.parse_doc_tree(doctree=args.doctree, pypackages=pypackages)
return self.compare_trees(parsed_pytree=parsed_pytree, parsed_doctree=parsed_doctree)
if __name__ == '__main__':
sys.exit(DocParityCheck().main())