diff options
Diffstat (limited to 'bin/find-unneeded-includes')
-rwxr-xr-x | bin/find-unneeded-includes | 248 |
1 files changed, 233 insertions, 15 deletions
diff --git a/bin/find-unneeded-includes b/bin/find-unneeded-includes index 93257451cce9..509331cd5ff3 100755 --- a/bin/find-unneeded-includes +++ b/bin/find-unneeded-includes @@ -17,7 +17,6 @@ # - no custom configure options required # - no need to generate a dummy library to build a header -import glob import json import multiprocessing import os @@ -28,9 +27,10 @@ import sys import threading import yaml import argparse +import pathlib -def ignoreRemoval(include, toAdd, absFileName, moduleRules): +def ignoreRemoval(include, toAdd, absFileName, moduleRules, noexclude): # global rules # Avoid replacing .hpp with .hdl in the com::sun::star and ooo::vba namespaces. @@ -75,7 +75,6 @@ def ignoreRemoval(include, toAdd, absFileName, moduleRules): o3tl = { "o3tl/typed_flags_set.hxx" : "namespace o3tl { template <typename T> struct typed_flags; }", "o3tl/deleter.hxx" : "namespace o3tl { template <typename T> struct default_delete; }", - "o3tl/span.hxx" : "namespace o3tl { template <typename T> class span; }", } for k, v, in o3tl.items(): if include == k and v in toAdd: @@ -138,9 +137,9 @@ def ignoreRemoval(include, toAdd, absFileName, moduleRules): if include.endswith(".hpp"): return True - # yaml rules + # yaml rules, except when --noexclude is given - if "excludelist" in moduleRules.keys(): + if "excludelist" in moduleRules.keys() and not noexclude: excludelistRules = moduleRules["excludelist"] if fileName in excludelistRules.keys(): if include in excludelistRules[fileName]: @@ -154,11 +153,12 @@ def unwrapInclude(include): return include[1:-1] -def processIWYUOutput(iwyuOutput, moduleRules, fileName): +def processIWYUOutput(iwyuOutput, moduleRules, fileName, noexclude, checknamespaces): inAdd = False toAdd = [] inRemove = False toRemove = [] + inFull = False currentFileName = None for line in iwyuOutput: @@ -175,6 +175,9 @@ def processIWYUOutput(iwyuOutput, moduleRules, fileName): if inAdd: inAdd = False continue + if inFull: + inFull = False + continue shouldAdd = fileName + " should add these lines:" match = re.match(shouldAdd, line) @@ -190,6 +193,12 @@ def processIWYUOutput(iwyuOutput, moduleRules, fileName): inRemove = True continue + if checknamespaces: + match = re.match("The full include-list for " + fileName, line) + if match: + inFull = True + continue + if inAdd: match = re.match('#include ([^ ]+)', line) if match: @@ -199,7 +208,7 @@ def processIWYUOutput(iwyuOutput, moduleRules, fileName): # Forward declaration. toAdd.append(line) - if inRemove: + if inRemove and not checknamespaces: match = re.match("- #include (.*) // lines (.*)-.*", line) if match: # Only suggest removals for now. Removing fwd decls is more complex: they may be @@ -207,28 +216,206 @@ def processIWYUOutput(iwyuOutput, moduleRules, fileName): # avoid the later. include = unwrapInclude(match.group(1)) lineno = match.group(2) - if not ignoreRemoval(include, toAdd, currentFileName, moduleRules): + if not ignoreRemoval(include, toAdd, currentFileName, moduleRules, noexclude): toRemove.append("%s:%s: %s" % (currentFileName, lineno, include)) + if inFull: + if checknamespaces: + # match for all possible URE/UNO namespaces, created with: + # find udkapi/com/sun/star/ -type d | sort| xargs basename -a | tr '\012' '|' + # find offapi/com/sun/star/ -type d | sort | xargs basename -a | tr '\012' '|' + # and ooo::vba namespaces + # plus a few popular ones about other modules + ns = re.compile( + '.*for\ (' + # URE namespaces + 'beans|' + 'bridge|oleautomation|' + 'connection|' + 'container|' + 'io|' + 'java|' + 'lang|' + 'loader|' + 'reflection|' + 'registry|' + 'script|' + 'security|' + 'task|' + 'uno|' + 'uri|' + 'util|' + # UNO namespaces + 'accessibility|' + 'animations|' + 'auth|' + 'awt|tab|tree|grid|' + 'chart|' + 'chart2|data|' + 'configuration|bootstrap|backend|xml|' + 'cui|' + 'datatransfer|clipboard|dnd|' + 'deployment|test|ui|' + 'document|' + 'drawing|framework|' + 'embed|' + 'form|binding|runtime|control|inspection|submission|component|validation|' + 'formula|' + 'frame|status|' + 'gallery|' + 'geometry|' + 'graphic|' + 'i18n|' + 'image|' + 'inspection|' + 'ldap|' + 'linguistic2|' + 'logging|' + 'mail|' + 'media|' + 'mozilla|' + 'office|' + 'packages|zip|manifest|' + 'presentation|textfield|' + 'qa|' + 'rdf|' + 'rendering|' + 'report|inspection|meta|' + 'resource|' + 'scanner|' + 'script|vba|browse|provider|' + 'sdb|application|tools|' + 'sdbc|' + 'sdbcx|' + 'security|' + 'setup|' + 'sheet|opencl|' + 'smarttags|' + 'style|' + 'svg|' + 'system|windows|' + 'table|' + 'task|' + 'text|textfield|docinfo|fieldmaster|' + 'tiledrendering|' + 'ucb|' + 'ui|dialogs|test|' + 'util|' + 'view|' + 'xforms|' + 'xml|xslt|wrapper|csax|sax|input|xpath|dom|views|events|crypto|sax|' + 'xsd|' + # ooo::vba and its namespaces + 'ooo|vba|excel|powerpoint|adodb|access|office|word|stdole|msforms|dao|' + # use of module namespaces, as spotted in the code + 'analysis|pricing' # sca internals + 'apphelper|CloneHelper|DataSeriesProperties|SceneProperties|wrapper|' # for chart internals + 'basegfx|utils|' + 'boost|posix_time|gregorian' + 'cairo|' + 'canvas|' + 'chelp|' + 'comphelper|' + 'connectivity|' + 'cpp|java|' # for codemaker:: + 'cppu|' + 'dbaccess|dbahsql|dbaui|dbtools|' + 'desktop|dp_misc|' + 'drawinglayer|attribute|geometry|primitive2d|processor2d|' + 'editeng|' + 'emscripten|' + 'formula|' + 'framework|' + 'frm|' + 'http_dav_ucp|tdoc_ucp|package_ucp|hierarchy_ucp|gio|fileaccess|ucb_impl|hcp_impl|ucb_cmdenv|' # for ucb internal + 'i18npool|' + 'internal|ColorComponentTag|' # for slideshow internals + 'jfw_plugin|' + 'jni_uno|' + 'librevenge|' + 'linguistic|' + 'lok|' + 'mtv|' # for mdds::mtv + 'nsSwDocInfoSubType|SWUnoHelper|nsHdFtFlags|' # sw internal + 'o3tl|' + 'odfflatxml|' # filter internal + 'oox|core|drawingml|ole|vml|' + 'OpenStormBento|' + 'osl|' + 'PackageKit|' + 'pdfi|pdfparse|' + 'ppt|' + 'pyuno|' + 'reportdesign|' + 'rptui|' + 'rtl|math|textenc|' + 'salhelper|' + 'sax_fastparser|' + 'sax|' # for xml::sax + 'sc|' + 'SchXMLTools|' # for xmloff + 'sd|slidesorter|cache|controller|model|view|' + 'sf_misc|' + 'sfx2|DocTempl|' + 'sidebar|' # for sfx2::sidebar + 'skeletonmaker|' + 'std|chrono_literals|literals|' + 'stoc_sec|' + 'store|' + 'svl|impl|' + 'svt|' + 'svtools|' + 'svx|sdr|contact|table|' + 'sw|access|annotation|mark|types|util|' + 'toolkit|' + 'treeview|' + 'ucbhelper|' + 'unodevtools' + 'unopkg|' + 'util|db|qe|' # for xmlsearch:: + 'utl|' + 'vcl|' + 'writerfilter|' + 'xforms|' + 'xmloff|token|EnhancedCustomShapeToken' # for xmloff:: + 'ZipUtils' + ')$', re.VERBOSE + ) + + reason = re.match(ns, line) + if reason: + # Warn about namespaces: if a header is suggested only '// for $namespace', then the namespace is not used + # otherwise the used classes name would show up after the '// for' + # Cleaning out the respective header (if there is any + # - which is not always the case) is for the next run! + nameSpace = reason.group(1).split(' ')[0] + print("WARNING:", fileName, "This 'using namespace' is likely unnecessary:", nameSpace) + + # Get the row number, normal IWYU output does not contain this info + subprocess.run(["git", "grep", "-n", "using namespace.*"+nameSpace+";", fileName]) + for remove in sorted(toRemove): print("ERROR: %s: remove not needed include" % remove) return len(toRemove) -def run_tool(task_queue, failed_files, dontstop): +def run_tool(task_queue, failed_files, dontstop, noexclude, checknamespaces): while True: invocation, moduleRules = task_queue.get() if not len(failed_files): print("[IWYU] " + invocation.split(' ')[-1]) p = subprocess.Popen(invocation, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - retcode = processIWYUOutput(p.communicate()[0].decode('utf-8').splitlines(), moduleRules, invocation.split(' ')[-1]) - if retcode == -1: + retcode = processIWYUOutput(p.communicate()[0].decode('utf-8').splitlines(), moduleRules, invocation.split(' ')[-1], noexclude, checknamespaces) + if retcode == -1 and not checknamespaces: print("ERROR: A file is probably not self contained, check this commands output:\n" + invocation) elif retcode > 0: print("ERROR: The following command found unused includes:\n" + invocation) if not dontstop: failed_files.append(invocation) task_queue.task_done() + if checknamespaces: + # Workaround: sometimes running git grep makes the letters typed into the terminal disappear after the script is finished + os.system('stty sane') def isInUnoIncludeFile(path): @@ -244,7 +431,7 @@ def isInUnoIncludeFile(path): or path.startswith("include/uno/") -def tidy(compileCommands, paths, dontstop): +def tidy(compileCommands, paths, dontstop, noexclude,checknamespaces): return_code = 0 try: @@ -252,7 +439,7 @@ def tidy(compileCommands, paths, dontstop): task_queue = queue.Queue(max_task) failed_files = [] for _ in range(max_task): - t = threading.Thread(target=run_tool, args=(task_queue, failed_files, dontstop)) + t = threading.Thread(target=run_tool, args=(task_queue, failed_files, dontstop, noexclude,checknamespaces)) t.daemon = True t.start() @@ -260,6 +447,10 @@ def tidy(compileCommands, paths, dontstop): if isInUnoIncludeFile(path): continue + # IWYU fails on these with #error: don't use this in new code + if path.startswith("include/vcl/toolkit"): + continue + moduleName = path.split("/")[0] rulePath = os.path.join(moduleName, "IwyuFilter_" + moduleName + ".yaml") @@ -306,7 +497,7 @@ def tidy(compileCommands, paths, dontstop): def main(argv): parser = argparse.ArgumentParser(description='Check source files for unneeded includes.') - parser.add_argument('--dontstop', action='store_true', + parser.add_argument('--continue', action='store_true', help='Don\'t stop on errors. Useful for periodic re-check of large amount of files') parser.add_argument('Files' , nargs='*', help='The files to be checked') @@ -314,6 +505,12 @@ def main(argv): help='Recursively search a directory for source files to check') parser.add_argument('--headers', action='store_true', help='Check header files. If omitted, check source files. Use with --recursive.') + parser.add_argument('--noexclude', action='store_true', + help='Ignore excludelist. Useful to check whether its exclusions are still all valid.') + parser.add_argument('--ns', action='store_true', + help='Warn about unused "using namespace" statements. ' + 'Removing these may uncover more removable headers ' + 'in a subsequent normal run') args = parser.parse_args() @@ -341,7 +538,28 @@ def main(argv): print ("File 'compile_commands.json' does not exist, please run:\nmake vim-ide-integration") sys.exit(-1) - tidy(compileCommands, paths=list_of_files, dontstop=args.dontstop) + # quickly sanity check whether files with exceptions in yaml still exists + # only check for the module of the very first filename passed + + # Verify there are files selected for checking, with --recursive it + # may happen that there are in fact no C/C++ files in a module directory + if not list_of_files: + print("No files found to check!") + sys.exit(-2) + + moduleName = sorted(list_of_files)[0].split("/")[0] + rulePath = os.path.join(moduleName, "IwyuFilter_" + moduleName + ".yaml") + moduleRules = {} + if os.path.exists(rulePath): + moduleRules = yaml.full_load(open(rulePath)) + if "excludelist" in moduleRules.keys(): + excludelistRules = moduleRules["excludelist"] + for pathname in excludelistRules.keys(): + file = pathlib.Path(pathname) + if not file.exists(): + print("WARNING: File listed in " + rulePath + " no longer exists: " + pathname) + + tidy(compileCommands, paths=list_of_files, dontstop=vars(args)["continue"], noexclude=args.noexclude, checknamespaces=args.ns) if __name__ == '__main__': main(sys.argv[1:]) |