diff --git a/README.md b/README.md index f098bef..b79aa02 100644 --- a/README.md +++ b/README.md @@ -31,3 +31,12 @@ command in your project to generate the documentation. For an easy introduction into using cldoc, please have a look at the [example project](https://github.com/jessevdk/cldoc/tree/master/example) and corresponding generated [documentation](http://jessevdk.github.com/cldoc/example/). + +# How to run for develpment purposes +cd `realpath ${PROJECT_BUILD}` +LD_LIBRARY_PATH=$(llvm-config-3.6 --libdir) PYTHONPATH=${HOME}/Codes/cldoc \ + python -m cldoc.__init__ generate -- \ + --compile_commands ${PROJECT_BUILD}/compile_commands.json \ + --basedir `realpath ${PROJECT_DIR}` \ + --output /tmp/cldoc + diff --git a/cldoc/clang/cindex.py b/cldoc/clang/cindex.py index e5782e8..706c8dc 100644 --- a/cldoc/clang/cindex.py +++ b/cldoc/clang/cindex.py @@ -1098,6 +1098,18 @@ def __repr__(self): CursorKind.ANNOTATE_ATTR = CursorKind(406) CursorKind.ASM_LABEL_ATTR = CursorKind(407) CursorKind.PACKED_ATTR = CursorKind(408) +CursorKind.PURE_ATTR = CursorKind(409) +CursorKind.CONST_ATTR = CursorKind(410) +CursorKind.NO_DUPLICATE_ATTR = CursorKind(411) +CursorKind.CUDA_CONSTANT_ATTR = CursorKind(412) +CursorKind.CUDA_DEVICE_ATTR = CursorKind(413) +CursorKind.CUDA_GLOBAL_ATTR = CursorKind(414) +CursorKind.CUDA_HOST_ATTR = CursorKind(415) +CursorKind.CUDA_SHARED_ATTR = CursorKind(416) +CursorKind.VISIBILITY_ATTR = CursorKind(417) +CursorKind.DLL_EXPORT = CursorKind(418) +CursorKind.DLL_IMPORT = CursorKind(419) + ### # Preprocessing diff --git a/cldoc/cmdgenerate.py b/cldoc/cmdgenerate.py index cb44df9..01825a4 100644 --- a/cldoc/cmdgenerate.py +++ b/cldoc/cmdgenerate.py @@ -88,7 +88,10 @@ def run(args): parser.add_argument('--custom-css', default=[], metavar='FILES', action='append', help='specify additional css files to be merged into the html (only for when --output is html)') - parser.add_argument('files', nargs='+', help='files to parse') + parser.add_argument('--compile_commands', default=None, + help='Path to compile_commands.json') + + parser.add_argument('files', nargs='*', help='files to parse') restargs = args[sep + 1:] cxxflags = args[:sep] @@ -120,7 +123,7 @@ def run(args): cxxflags.append('-x') cxxflags.append(opts.language) - t = tree.Tree(opts.files, cxxflags) + t = tree.Tree(opts.basedir, opts.files, cxxflags, opts.compile_commands) t.process() diff --git a/cldoc/nodes/method.py b/cldoc/nodes/method.py index f910241..d699b7a 100644 --- a/cldoc/nodes/method.py +++ b/cldoc/nodes/method.py @@ -40,7 +40,10 @@ def override(self): return self._override # Lookup in bases, recursively - bases = list(self.parent.bases) + if hasattr(self.parent, 'bases'): + bases = list(self.parent.bases) + else: + bases = list() mname = self.name self._override = [] diff --git a/cldoc/tree.py b/cldoc/tree.py index 845f8ba..6a0bd43 100644 --- a/cldoc/tree.py +++ b/cldoc/tree.py @@ -21,6 +21,7 @@ import nodes import includepaths import documentmerger +import json from . import example from . import utf8 @@ -50,7 +51,7 @@ if not lname is None: cindex.Config.set_library_file(lname) else: - versions = [None, '3.5', '3.4', '3.3', '3.2'] + versions = [None, '3.6', '3.5', '3.4', '3.3', '3.2'] for v in versions: name = 'clang' @@ -72,8 +73,52 @@ sys.stderr.write("\nFatal: Failed to locate libclang library. cldoc depends on libclang for parsing sources, please make sure you have libclang installed.\n" + str(e) + "\n\n") sys.exit(1) +class ChangeDir(object): + """Provides 'with' semantics for os.chdir(). Will change to the desired directory and then + change back when leaving scope. Example: + + os.chdir(os.expanduser('~')) + print os.getcwd() # e.g. '/home/josh' + with ChangeDir('/tmp'): + print os.getcwd() # '/tmp' + print os.getcwd() # e.g. '/home/josh + """ + # TODO(josh): investigate https://github.com/jaraco/path.py + + def __init__(self, targetdir): + self.targetdir = targetdir + self.old_cwd = '' + + def __enter__(self): + self.old_cwd = os.getcwd() + os.chdir(self.targetdir) + + def __exit__(self, exception_type, exception_value, traceback): + os.chdir(self.old_cwd) + return exception_value is None + + +def strip_flag(flags, flag, nargs=0): + flag_index = flags.index(flag) + if flag_index != -1: + flags.pop(flag_index) + for i in range(nargs): + flags.pop(flag_index) + + +def get_flags(command): + flags = includepaths.flags([]) + command.split()[1:] + strip_flag(flags, '-c', 1) + strip_flag(flags, '-o', 1) + # TODO(josh): remove these hacks + while '-Werror' in flags: + flags.remove('-Werror') + return flags + ['-Wno-mismatched-tags', '-Wno-absolute-value', + '-Wno-ignored-qualifiers', '-Wno-unused-function'] + class Tree(documentmerger.DocumentMerger): - def __init__(self, files, flags): + def __init__(self, basedir, files, flags, command_db_path=None): + self.basedir = basedir self.processed = {} self.files, ok = self.expand_sources([os.path.realpath(f) for f in files]) @@ -81,6 +126,11 @@ def __init__(self, files, flags): sys.exit(1) self.flags = includepaths.flags(flags) + if command_db_path: + with open(command_db_path) as command_db_file: + self.compile_commands = json.load(command_db_file) + else: + self.compile_commands = None # Sort files on sources, then headers self.files.sort(lambda a, b: cmp(self.is_header(a), self.is_header(b))) @@ -154,67 +204,96 @@ def find_node_comment(self, node): return None - def process(self): + def process_file(self, f, directory=None, command=None): """ - process processes all the files with clang and extracts all relevant - nodes from the generated AST + process a single file """ + if f in self.processed: + return - self.index = cindex.Index.create() - self.headers = {} + print('Processing {0}'.format(os.path.basename(f))) - for f in self.files: - if f in self.processed: - continue + if directory: + assert command is not None, "If directory is supplied, command must also be supplied" + with ChangeDir(directory): + tu = self.index.parse(f, get_flags(command)) + else: + tu = self.index.parse(f, self.flags) - print('Processing {0}'.format(os.path.basename(f))) + if len(tu.diagnostics) != 0: + fatal = False - tu = self.index.parse(f, self.flags) + for d in tu.diagnostics: + sys.stderr.write(d.format) + sys.stderr.write("\n") - if len(tu.diagnostics) != 0: - fatal = False + if d.severity == cindex.Diagnostic.Fatal or \ + d.severity == cindex.Diagnostic.Error: + fatal = True - for d in tu.diagnostics: - sys.stderr.write(d.format) - sys.stderr.write("\n") + if fatal: + sys.stderr.write("\nCould not generate documentation for %s due to parser errors\n" % (f,)) + sys.exit(1) + + if not tu: + sys.stderr.write("Could not parse file %s...\n" % (f,)) + sys.exit(1) - if d.severity == cindex.Diagnostic.Fatal or \ - d.severity == cindex.Diagnostic.Error: - fatal = True + # Extract comments from files and included files that we are + # supposed to inspect + extractfiles = [f] - if fatal: - sys.stderr.write("\nCould not generate documentation due to parser errors\n") - sys.exit(1) + for inc in tu.get_includes(): + filename = str(inc.include) + self.headers[filename] = True - if not tu: - sys.stderr.write("Could not parse file %s...\n" % (f,)) - sys.exit(1) + if filename in self.processed or (not filename in self.files) or filename in extractfiles: + continue - # Extract comments from files and included files that we are - # supposed to inspect - extractfiles = [f] + extractfiles.append(filename) - for inc in tu.get_includes(): - filename = str(inc.include) - self.headers[filename] = True + for e in extractfiles: + db = comment.CommentsDatabase(e, tu) - if filename in self.processed or (not filename in self.files) or filename in extractfiles: - continue + self.add_categories(db.category_names) + self.commentsdbs[e] = db + + self.visit(tu.cursor.get_children()) + + for f in self.processing: + self.processed[f] = True + + self.processing = {} + + + def process(self): + """ + process processes all the files with clang and extracts all relevant + nodes from the generated AST + """ - extractfiles.append(filename) + self.index = cindex.Index.create() + self.headers = {} - for e in extractfiles: - db = comment.CommentsDatabase(e, tu) - self.add_categories(db.category_names) - self.commentsdbs[e] = db + num_files = len(self.files) + if self.compile_commands: + num_compile_commands = len(self.compile_commands) + else: + num_compile_commands = 0 - self.visit(tu.cursor.get_children()) + total_to_process = num_files + num_compile_commands + for i, f in enumerate(self.files): + progress_percent = 1e2 * i / total_to_process + sys.stdout.write('[{:6.2f}%] '.format(progress_percent)) + self.process_file(f) - for f in self.processing: - self.processed[f] = True + if self.compile_commands: + for i, entry in enumerate(self.compile_commands): + progress_percent = 1e2 * (i + num_files) / total_to_process + sys.stdout.write('[{:6.2f}%] '.format(progress_percent)) + self.process_file(entry['file'], entry['directory'], entry['command']) - self.processing = {} # Construct hierarchy of nodes. for node in self.all_nodes: @@ -477,6 +556,20 @@ def visit(self, citer, parent=None): # Ignore files other than the ones we are scanning for if not str(item.location.file) in self.files: + # TODO(josh): re-enable this check after figuring out how to + # match files from compilation database + pass + # continue + + realpath_to_file = os.path.realpath(str(item.location.file)) + realpath_to_base = os.path.realpath(self.basedir) + relpath_from_basedir = os.path.relpath(realpath_to_file, + realpath_to_base) + + # Only include files that are a subpath of basedir + # NOTE(josh): item.location.file might be relative to the cwd + # of the command + if not len(relpath_from_basedir) < len(realpath_to_file): continue # Ignore unexposed things