epydoc.docbuilder

1 # epydoc -- Documentation Builder 2 # 3 # Copyright (C) 2005 Edward Loper 4 # Author: Edward Loper <edloper@loper.org> 5 # URL: <http://epydoc.sf.net> 6 # 7 # $Id: docbuilder.py 1683 2008-01-29 22:17:39Z edloper $ 8 9 """ 10 Construct data structures that encode the API documentation for Python 11 objects. These data structures are created using a series of steps: 12 13 1. B{Building docs}: Extract basic information about the objects, 14 and objects that are related to them. This can be done by 15 introspecting the objects' values (with L{epydoc.docintrospecter}; or 16 by parsing their source code (with L{epydoc.docparser}. 17 18 2. B{Merging}: Combine the information obtained from introspection & 19 parsing each object into a single structure. 20 21 3. B{Linking}: Replace any 'pointers' that were created for imported 22 variables by their target (if it's available). 23 24 4. B{Naming}: Chose a unique 'canonical name' for each 25 object. 26 27 5. B{Docstring Parsing}: Parse the docstring of each object, and 28 extract any pertinant information. 29 30 6. B{Inheritance}: Add information about variables that classes 31 inherit from their base classes. 32 33 The documentation information for each individual object is 34 represented using an L{APIDoc}; and the documentation for a collection 35 of objects is represented using a L{DocIndex}. 36 37 The main interface to C{epydoc.docbuilder} consists of two functions: 38 39 - L{build_doc()} -- Builds documentation for a single item, and 40 returns it as an L{APIDoc} object. 41 - L{build_doc_index()} -- Builds documentation for a collection of 42 items, and returns it as a L{DocIndex} object. 43 44 The remaining functions are used by these two main functions to 45 perform individual steps in the creation of the documentation. 46 47 @group Documentation Construction: build_doc, build_doc_index, 48 _get_docs_from_*, _report_valdoc_progress 49 @group Merging: *MERGE*, *merge* 50 @group Linking: link_imports 51 @group Naming: _name_scores, _unreachable_names, assign_canonical_names, 52 _var_shadows_self, _fix_self_shadowing_var, _unreachable_name_for 53 @group Inheritance: inherit_docs, _inherit_info 54 """ 55 __docformat__ = 'epytext en' 56 57 ###################################################################### 58 ## Contents 59 ###################################################################### 60 ## 1. build_doc() & build_doc_index() -- the main interface. 61 ## 2. merge_docs() -- helper, used to merge parse & introspect info 62 ## 3. link_imports() -- helper, used to connect imported vars w/ values 63 ## 4. assign_canonical_names() -- helper, used to set canonical names 64 ## 5. inherit_docs() -- helper, used to inherit docs from base classes 65 66 ###################################################################### 67 ## Imports 68 ###################################################################### 69 70 import sys, os, os.path, __builtin__, imp, re, inspect 71 from epydoc.apidoc import * 72 from epydoc.docintrospecter import introspect_docs 73 from epydoc.docparser import parse_docs, ParseError 74 from epydoc.docstringparser import parse_docstring 75 from epydoc import log 76 from epydoc.util import * 77 from epydoc.compat import * # Backwards compatibility 78 79 ###################################################################### 80 ## 1. build_doc() 81 ###################################################################### 82

83 -class BuildOptions:

84 """ 85 Holds the parameters for a documentation building process. 86 """

87 - def __init__(self, introspect=True, parse=True, 88 exclude_introspect=None, exclude_parse=None, 89 add_submodules=True):

90 self.introspect = introspect 91 self.parse = parse 92 self.exclude_introspect = exclude_introspect 93 self.exclude_parse = exclude_parse 94 self.add_submodules = add_submodules 95 96 # Test for pattern syntax and compile them into pattern objects. 97 try: 98 self._introspect_regexp = (exclude_introspect 99 and re.compile(exclude_introspect) or None) 100 self._parse_regexp = (exclude_parse 101 and re.compile(exclude_parse) or None) 102 except Exception, exc: 103 log.error('Error in regular expression pattern: %s' % exc) 104 raise

105

106 - def must_introspect(self, name):

107 """ 108 Return C{True} if a module is to be introsepcted with the current 109 settings. 110 111 @param name: The name of the module to test 112 @type name: L{DottedName} or C{str} 113 """ 114 return self.introspect \ 115 and not self._matches_filter(name, self._introspect_regexp)

116

117 - def must_parse(self, name):

118 """ 119 Return C{True} if a module is to be parsed with the current settings. 120 121 @param name: The name of the module to test 122 @type name: L{DottedName} or C{str} 123 """ 124 return self.parse \ 125 and not self._matches_filter(name, self._parse_regexp)

126

127 - def _matches_filter(self, name, regexp):

128 """ 129 Test if a module name matches a pattern. 130 131 @param name: The name of the module to test 132 @type name: L{DottedName} or C{str} 133 @param regexp: The pattern object to match C{name} against. 134 If C{None}, return C{False} 135 @type regexp: C{pattern} 136 @return: C{True} if C{name} in dotted format matches C{regexp}, 137 else C{False} 138 @rtype: C{bool} 139 """ 140 if regexp is None: return False 141 142 if isinstance(name, DottedName): 143 name = str(name) 144 145 return bool(regexp.search(name))

146 147

148 -def build_doc(item, introspect=True, parse=True, add_submodules=True, 149 exclude_introspect=None, exclude_parse=None):

150 """ 151 Build API documentation for a given item, and return it as 152 an L{APIDoc} object. 153 154 @rtype: L{APIDoc} 155 @param item: The item to document, specified using any of the 156 following: 157 - A string, naming a python package directory 158 (e.g., C{'epydoc/markup'}) 159 - A string, naming a python file 160 (e.g., C{'epydoc/docparser.py'}) 161 - A string, naming a python object 162 (e.g., C{'epydoc.docparser.DocParser'}) 163 - Any (non-string) python object 164 (e.g., C{list.append}) 165 @param introspect: If true, then use introspection to examine the 166 specified items. Otherwise, just use parsing. 167 @param parse: If true, then use parsing to examine the specified 168 items. Otherwise, just use introspection. 169 """ 170 docindex = build_doc_index([item], introspect, parse, add_submodules, 171 exclude_introspect=exclude_introspect, 172 exclude_parse=exclude_parse) 173 return docindex.root[0]

174

175 -def build_doc_index(items, introspect=True, parse=True, add_submodules=True, 176 exclude_introspect=None, exclude_parse=None):

177 """ 178 Build API documentation for the given list of items, and 179 return it in the form of a L{DocIndex}. 180 181 @rtype: L{DocIndex} 182 @param items: The items to document, specified using any of the 183 following: 184 - A string, naming a python package directory 185 (e.g., C{'epydoc/markup'}) 186 - A string, naming a python file 187 (e.g., C{'epydoc/docparser.py'}) 188 - A string, naming a python object 189 (e.g., C{'epydoc.docparser.DocParser'}) 190 - Any (non-string) python object 191 (e.g., C{list.append}) 192 @param introspect: If true, then use introspection to examine the 193 specified items. Otherwise, just use parsing. 194 @param parse: If true, then use parsing to examine the specified 195 items. Otherwise, just use introspection. 196 """ 197 try: 198 options = BuildOptions(parse=parse, introspect=introspect, 199 exclude_introspect=exclude_introspect, exclude_parse=exclude_parse, 200 add_submodules=add_submodules) 201 except Exception, e: 202 # log.error already reported by constructor. 203 return None 204 205 # Get the basic docs for each item. 206 doc_pairs = _get_docs_from_items(items, options) 207 208 # Merge the introspection & parse docs. 209 if options.parse and options.introspect: 210 log.start_progress('Merging parsed & introspected information') 211 docs = [] 212 for i, (introspect_doc, parse_doc) in enumerate(doc_pairs): 213 if introspect_doc is not None and parse_doc is not None: 214 if introspect_doc.canonical_name not in (None, UNKNOWN): 215 name = introspect_doc.canonical_name 216 else: 217 name = parse_doc.canonical_name 218 log.progress(float(i)/len(doc_pairs), name) 219 docs.append(merge_docs(introspect_doc, parse_doc)) 220 elif introspect_doc is not None: 221 docs.append(introspect_doc) 222 elif parse_doc is not None: 223 docs.append(parse_doc) 224 log.end_progress() 225 elif options.introspect: 226 docs = [doc_pair[0] for doc_pair in doc_pairs if doc_pair[0]] 227 else: 228 docs = [doc_pair[1] for doc_pair in doc_pairs if doc_pair[1]] 229 230 if len(docs) == 0: 231 log.error('Nothing left to document!') 232 return None 233 234 # Collect the docs into a single index. 235 docindex = DocIndex(docs) 236 237 # Replace any proxy valuedocs that we got from importing with 238 # their targets. 239 if options.parse: 240 log.start_progress('Linking imported variables') 241 valdocs = sorted(docindex.reachable_valdocs( 242 imports=False, submodules=False, packages=False, subclasses=False)) 243 for i, val_doc in enumerate(valdocs): 244 _report_valdoc_progress(i, val_doc, valdocs) 245 link_imports(val_doc, docindex) 246 log.end_progress() 247 248 # Assign canonical names. 249 log.start_progress('Indexing documentation') 250 for i, val_doc in enumerate(docindex.root): 251 log.progress(float(i)/len(docindex.root), val_doc.canonical_name) 252 assign_canonical_names(val_doc, val_doc.canonical_name, docindex) 253 log.end_progress() 254 255 # Set overrides pointers 256 log.start_progress('Checking for overridden methods') 257 valdocs = sorted(docindex.reachable_valdocs( 258 imports=False, submodules=False, packages=False, subclasses=False)) 259 for i, val_doc in enumerate(valdocs): 260 if isinstance(val_doc, ClassDoc): 261 percent = float(i)/len(valdocs) 262 log.progress(percent, val_doc.canonical_name) 263 find_overrides(val_doc) 264 log.end_progress() 265 266 # Parse the docstrings for each object. 267 log.start_progress('Parsing docstrings') 268 suppress_warnings = set(valdocs).difference( 269 docindex.reachable_valdocs( 270 imports=False, submodules=False, packages=False, subclasses=False, 271 bases=False, overrides=True)) 272 for i, val_doc in enumerate(valdocs): 273 _report_valdoc_progress(i, val_doc, valdocs) 274 # the value's docstring 275 parse_docstring(val_doc, docindex, suppress_warnings) 276 # the value's variables' docstrings 277 if (isinstance(val_doc, NamespaceDoc) and 278 val_doc.variables not in (None, UNKNOWN)): 279 for var_doc in val_doc.variables.values(): 280 # Now we have a chance to propagate the defining module 281 # to objects for which introspection is not possible, 282 # such as properties. 283 if (isinstance(var_doc.value, ValueDoc) 284 and var_doc.value.defining_module is UNKNOWN): 285 var_doc.value.defining_module = val_doc.defining_module 286 parse_docstring(var_doc, docindex, suppress_warnings) 287 log.end_progress() 288 289 # Take care of inheritance. 290 log.start_progress('Inheriting documentation') 291 for i, val_doc in enumerate(valdocs): 292 if isinstance(val_doc, ClassDoc): 293 percent = float(i)/len(valdocs) 294 log.progress(percent, val_doc.canonical_name) 295 inherit_docs(val_doc) 296 log.end_progress() 297 298 # Initialize the groups & sortedvars attributes. 299 log.start_progress('Sorting & Grouping') 300 for i, val_doc in enumerate(valdocs): 301 if isinstance(val_doc, NamespaceDoc): 302 percent = float(i)/len(valdocs) 303 log.progress(percent, val_doc.canonical_name) 304 val_doc.init_sorted_variables() 305 val_doc.init_variable_groups() 306 if isinstance(val_doc, ModuleDoc): 307 val_doc.init_submodule_groups() 308 val_doc.report_unused_groups() 309 log.end_progress() 310 311 return docindex

312

313 -def _report_valdoc_progress(i, val_doc, val_docs):

314 if (isinstance(val_doc, (ModuleDoc, ClassDoc)) and 315 val_doc.canonical_name is not UNKNOWN and 316 not val_doc.canonical_name[0].startswith('??')): 317 log.progress(float(i)/len(val_docs), val_doc.canonical_name)

318 319 #///////////////////////////////////////////////////////////////// 320 # Documentation Generation 321 #///////////////////////////////////////////////////////////////// 322

323 -def _get_docs_from_items(items, options):

324 325 # Start the progress bar. 326 log.start_progress('Building documentation') 327 progress_estimator = _ProgressEstimator(items) 328 329 # Check for duplicate item names. 330 item_set = set() 331 for item in items[:]: 332 if item in item_set: 333 log.warning("Name %r given multiple times" % item) 334 items.remove(item) 335 item_set.add(item) 336 337 # Keep track of what top-level canonical names we've assigned, to 338 # make sure there are no naming conflicts. This dict maps 339 # canonical names to the item names they came from (so we can print 340 # useful error messages). 341 canonical_names = {} 342 343 # Collect (introspectdoc, parsedoc) pairs for each item. 344 doc_pairs = [] 345 for item in items: 346 if isinstance(item, basestring): 347 if is_module_file(item): 348 doc_pairs.append(_get_docs_from_module_file( 349 item, options, progress_estimator)) 350 elif is_package_dir(item): 351 pkgfile = os.path.abspath(os.path.join(item, '__init__')) 352 doc_pairs.append(_get_docs_from_module_file( 353 pkgfile, options, progress_estimator)) 354 elif os.path.isfile(item): 355 doc_pairs.append(_get_docs_from_pyscript( 356 item, options, progress_estimator)) 357 elif hasattr(__builtin__, item): 358 val = getattr(__builtin__, item) 359 doc_pairs.append(_get_docs_from_pyobject( 360 val, options, progress_estimator)) 361 elif is_pyname(item): 362 doc_pairs.append(_get_docs_from_pyname( 363 item, options, progress_estimator)) 364 elif os.path.isdir(item): 365 log.error("Directory %r is not a package" % item) 366 continue 367 elif os.path.isfile(item): 368 log.error("File %s is not a Python module" % item) 369 continue 370 else: 371 log.error("Could not find a file or object named %s" % 372 item) 373 continue 374 else: 375 doc_pairs.append(_get_docs_from_pyobject( 376 item, options, progress_estimator)) 377 378 # Make sure there are no naming conflicts. 379 name = (getattr(doc_pairs[-1][0], 'canonical_name', None) or 380 getattr(doc_pairs[-1][1], 'canonical_name', None)) 381 if name in canonical_names: 382 log.error( 383 'Two of the specified items, %r and %r, have the same ' 384 'canonical name ("%s"). This may mean that you specified ' 385 'two different files that both use the same module name. ' 386 'Ignoring the second item (%r)' % 387 (canonical_names[name], item, name, canonical_names[name])) 388 doc_pairs.pop() 389 else: 390 canonical_names[name] = item 391 392 # This will only have an effect if doc_pairs[-1] contains a 393 # package's docs. The 'not is_module_file(item)' prevents 394 # us from adding subdirectories if they explicitly specify 395 # a package's __init__.py file. 396 if options.add_submodules and not is_module_file(item): 397 doc_pairs += _get_docs_from_submodules( 398 item, doc_pairs[-1], options, progress_estimator) 399 400 log.end_progress() 401 return doc_pairs

402

403 -def _get_docs_from_pyobject(obj, options, progress_estimator):

404 progress_estimator.complete += 1 405 log.progress(progress_estimator.progress(), repr(obj)) 406 407 if not options.introspect: 408 log.error("Cannot get docs for Python objects without " 409 "introspecting them.") 410 411 introspect_doc = parse_doc = None 412 introspect_error = parse_error = None 413 try: 414 introspect_doc = introspect_docs(value=obj) 415 except ImportError, e: 416 log.error(e) 417 return (None, None) 418 if options.parse: 419 if introspect_doc.canonical_name is not None: 420 prev_introspect = options.introspect 421 options.introspect = False 422 try: 423 _, parse_docs = _get_docs_from_pyname( 424 str(introspect_doc.canonical_name), options, 425 progress_estimator, suppress_warnings=True) 426 finally: 427 options.introspect = prev_introspect 428 429 # We need a name: 430 if introspect_doc.canonical_name in (None, UNKNOWN): 431 if hasattr(obj, '__name__'): 432 introspect_doc.canonical_name = DottedName( 433 DottedName.UNREACHABLE, obj.__name__) 434 else: 435 introspect_doc.canonical_name = DottedName( 436 DottedName.UNREACHABLE) 437 return (introspect_doc, parse_doc)

438

439 -def _get_docs_from_pyname(name, options, progress_estimator, 440 suppress_warnings=False):

441 progress_estimator.complete += 1 442 if options.must_introspect(name) or options.must_parse(name): 443 log.progress(progress_estimator.progress(), name) 444 445 introspect_doc = parse_doc = None 446 introspect_error = parse_error = None 447 if options.must_introspect(name): 448 try: 449 introspect_doc = introspect_docs(name=name) 450 except ImportError, e: 451 introspect_error = str(e) 452 if options.must_parse(name): 453 try: 454 parse_doc = parse_docs(name=name) 455 except ParseError, e: 456 parse_error = str(e) 457 except ImportError, e: 458 # If we get here, then there' probably no python source 459 # available; don't bother to generate a warnining. 460 pass 461 462 # Report any errors we encountered. 463 if not suppress_warnings: 464 _report_errors(name, introspect_doc, parse_doc, 465 introspect_error, parse_error) 466 467 # Return the docs we found. 468 return (introspect_doc, parse_doc)

469

470 -def _get_docs_from_pyscript(filename, options, progress_estimator):

471 # [xx] I should be careful about what names I allow as filenames, 472 # and maybe do some munging to prevent problems. 473 474 introspect_doc = parse_doc = None 475 introspect_error = parse_error = None 476 if options.introspect: 477 try: 478 introspect_doc = introspect_docs(filename=filename, is_script=True) 479 if introspect_doc.canonical_name is UNKNOWN: 480 introspect_doc.canonical_name = munge_script_name(filename) 481 except ImportError, e: 482 introspect_error = str(e) 483 if options.parse: 484 try: 485 parse_doc = parse_docs(filename=filename, is_script=True) 486 except ParseError, e: 487 parse_error = str(e) 488 except ImportError, e: 489 parse_error = str(e) 490 491 # Report any errors we encountered. 492 _report_errors(filename, introspect_doc, parse_doc, 493 introspect_error, parse_error) 494 495 # Return the docs we found. 496 return (introspect_doc, parse_doc)

497

498 -def _get_docs_from_module_file(filename, options, progress_estimator, 499 parent_docs=(None,None)):

500 """ 501 Construct and return the API documentation for the python 502 module with the given filename. 503 504 @param parent_docs: The C{ModuleDoc} of the containing package. 505 If C{parent_docs} is not provided, then this method will 506 check if the given filename is contained in a package; and 507 if so, it will construct a stub C{ModuleDoc} for the 508 containing package(s). C{parent_docs} is a tuple, where 509 the first element is the parent from introspection, and 510 the second element is the parent from parsing. 511 """ 512 # Record our progress. 513 modulename = os.path.splitext(os.path.split(filename)[1])

Source Code for Module epydoc.docbuilder