summaryrefslogtreecommitdiffstats
path: root/bin/find-unneeded-includes
diff options
context:
space:
mode:
Diffstat (limited to 'bin/find-unneeded-includes')
-rwxr-xr-xbin/find-unneeded-includes248
1 files changed, 233 insertions, 15 deletions
diff --git a/bin/find-unneeded-includes b/bin/find-unneeded-includes
index 93257451cce9..509331cd5ff3 100755
--- a/bin/find-unneeded-includes
+++ b/bin/find-unneeded-includes
@@ -17,7 +17,6 @@
# - no custom configure options required
# - no need to generate a dummy library to build a header
-import glob
import json
import multiprocessing
import os
@@ -28,9 +27,10 @@ import sys
import threading
import yaml
import argparse
+import pathlib
-def ignoreRemoval(include, toAdd, absFileName, moduleRules):
+def ignoreRemoval(include, toAdd, absFileName, moduleRules, noexclude):
# global rules
# Avoid replacing .hpp with .hdl in the com::sun::star and ooo::vba namespaces.
@@ -75,7 +75,6 @@ def ignoreRemoval(include, toAdd, absFileName, moduleRules):
o3tl = {
"o3tl/typed_flags_set.hxx" : "namespace o3tl { template <typename T> struct typed_flags; }",
"o3tl/deleter.hxx" : "namespace o3tl { template <typename T> struct default_delete; }",
- "o3tl/span.hxx" : "namespace o3tl { template <typename T> class span; }",
}
for k, v, in o3tl.items():
if include == k and v in toAdd:
@@ -138,9 +137,9 @@ def ignoreRemoval(include, toAdd, absFileName, moduleRules):
if include.endswith(".hpp"):
return True
- # yaml rules
+ # yaml rules, except when --noexclude is given
- if "excludelist" in moduleRules.keys():
+ if "excludelist" in moduleRules.keys() and not noexclude:
excludelistRules = moduleRules["excludelist"]
if fileName in excludelistRules.keys():
if include in excludelistRules[fileName]:
@@ -154,11 +153,12 @@ def unwrapInclude(include):
return include[1:-1]
-def processIWYUOutput(iwyuOutput, moduleRules, fileName):
+def processIWYUOutput(iwyuOutput, moduleRules, fileName, noexclude, checknamespaces):
inAdd = False
toAdd = []
inRemove = False
toRemove = []
+ inFull = False
currentFileName = None
for line in iwyuOutput:
@@ -175,6 +175,9 @@ def processIWYUOutput(iwyuOutput, moduleRules, fileName):
if inAdd:
inAdd = False
continue
+ if inFull:
+ inFull = False
+ continue
shouldAdd = fileName + " should add these lines:"
match = re.match(shouldAdd, line)
@@ -190,6 +193,12 @@ def processIWYUOutput(iwyuOutput, moduleRules, fileName):
inRemove = True
continue
+ if checknamespaces:
+ match = re.match("The full include-list for " + fileName, line)
+ if match:
+ inFull = True
+ continue
+
if inAdd:
match = re.match('#include ([^ ]+)', line)
if match:
@@ -199,7 +208,7 @@ def processIWYUOutput(iwyuOutput, moduleRules, fileName):
# Forward declaration.
toAdd.append(line)
- if inRemove:
+ if inRemove and not checknamespaces:
match = re.match("- #include (.*) // lines (.*)-.*", line)
if match:
# Only suggest removals for now. Removing fwd decls is more complex: they may be
@@ -207,28 +216,206 @@ def processIWYUOutput(iwyuOutput, moduleRules, fileName):
# avoid the later.
include = unwrapInclude(match.group(1))
lineno = match.group(2)
- if not ignoreRemoval(include, toAdd, currentFileName, moduleRules):
+ if not ignoreRemoval(include, toAdd, currentFileName, moduleRules, noexclude):
toRemove.append("%s:%s: %s" % (currentFileName, lineno, include))
+ if inFull:
+ if checknamespaces:
+ # match for all possible URE/UNO namespaces, created with:
+ # find udkapi/com/sun/star/ -type d | sort| xargs basename -a | tr '\012' '|'
+ # find offapi/com/sun/star/ -type d | sort | xargs basename -a | tr '\012' '|'
+ # and ooo::vba namespaces
+ # plus a few popular ones about other modules
+ ns = re.compile(
+ '.*for\ ('
+ # URE namespaces
+ 'beans|'
+ 'bridge|oleautomation|'
+ 'connection|'
+ 'container|'
+ 'io|'
+ 'java|'
+ 'lang|'
+ 'loader|'
+ 'reflection|'
+ 'registry|'
+ 'script|'
+ 'security|'
+ 'task|'
+ 'uno|'
+ 'uri|'
+ 'util|'
+ # UNO namespaces
+ 'accessibility|'
+ 'animations|'
+ 'auth|'
+ 'awt|tab|tree|grid|'
+ 'chart|'
+ 'chart2|data|'
+ 'configuration|bootstrap|backend|xml|'
+ 'cui|'
+ 'datatransfer|clipboard|dnd|'
+ 'deployment|test|ui|'
+ 'document|'
+ 'drawing|framework|'
+ 'embed|'
+ 'form|binding|runtime|control|inspection|submission|component|validation|'
+ 'formula|'
+ 'frame|status|'
+ 'gallery|'
+ 'geometry|'
+ 'graphic|'
+ 'i18n|'
+ 'image|'
+ 'inspection|'
+ 'ldap|'
+ 'linguistic2|'
+ 'logging|'
+ 'mail|'
+ 'media|'
+ 'mozilla|'
+ 'office|'
+ 'packages|zip|manifest|'
+ 'presentation|textfield|'
+ 'qa|'
+ 'rdf|'
+ 'rendering|'
+ 'report|inspection|meta|'
+ 'resource|'
+ 'scanner|'
+ 'script|vba|browse|provider|'
+ 'sdb|application|tools|'
+ 'sdbc|'
+ 'sdbcx|'
+ 'security|'
+ 'setup|'
+ 'sheet|opencl|'
+ 'smarttags|'
+ 'style|'
+ 'svg|'
+ 'system|windows|'
+ 'table|'
+ 'task|'
+ 'text|textfield|docinfo|fieldmaster|'
+ 'tiledrendering|'
+ 'ucb|'
+ 'ui|dialogs|test|'
+ 'util|'
+ 'view|'
+ 'xforms|'
+ 'xml|xslt|wrapper|csax|sax|input|xpath|dom|views|events|crypto|sax|'
+ 'xsd|'
+ # ooo::vba and its namespaces
+ 'ooo|vba|excel|powerpoint|adodb|access|office|word|stdole|msforms|dao|'
+ # use of module namespaces, as spotted in the code
+ 'analysis|pricing' # sca internals
+ 'apphelper|CloneHelper|DataSeriesProperties|SceneProperties|wrapper|' # for chart internals
+ 'basegfx|utils|'
+ 'boost|posix_time|gregorian'
+ 'cairo|'
+ 'canvas|'
+ 'chelp|'
+ 'comphelper|'
+ 'connectivity|'
+ 'cpp|java|' # for codemaker::
+ 'cppu|'
+ 'dbaccess|dbahsql|dbaui|dbtools|'
+ 'desktop|dp_misc|'
+ 'drawinglayer|attribute|geometry|primitive2d|processor2d|'
+ 'editeng|'
+ 'emscripten|'
+ 'formula|'
+ 'framework|'
+ 'frm|'
+ 'http_dav_ucp|tdoc_ucp|package_ucp|hierarchy_ucp|gio|fileaccess|ucb_impl|hcp_impl|ucb_cmdenv|' # for ucb internal
+ 'i18npool|'
+ 'internal|ColorComponentTag|' # for slideshow internals
+ 'jfw_plugin|'
+ 'jni_uno|'
+ 'librevenge|'
+ 'linguistic|'
+ 'lok|'
+ 'mtv|' # for mdds::mtv
+ 'nsSwDocInfoSubType|SWUnoHelper|nsHdFtFlags|' # sw internal
+ 'o3tl|'
+ 'odfflatxml|' # filter internal
+ 'oox|core|drawingml|ole|vml|'
+ 'OpenStormBento|'
+ 'osl|'
+ 'PackageKit|'
+ 'pdfi|pdfparse|'
+ 'ppt|'
+ 'pyuno|'
+ 'reportdesign|'
+ 'rptui|'
+ 'rtl|math|textenc|'
+ 'salhelper|'
+ 'sax_fastparser|'
+ 'sax|' # for xml::sax
+ 'sc|'
+ 'SchXMLTools|' # for xmloff
+ 'sd|slidesorter|cache|controller|model|view|'
+ 'sf_misc|'
+ 'sfx2|DocTempl|'
+ 'sidebar|' # for sfx2::sidebar
+ 'skeletonmaker|'
+ 'std|chrono_literals|literals|'
+ 'stoc_sec|'
+ 'store|'
+ 'svl|impl|'
+ 'svt|'
+ 'svtools|'
+ 'svx|sdr|contact|table|'
+ 'sw|access|annotation|mark|types|util|'
+ 'toolkit|'
+ 'treeview|'
+ 'ucbhelper|'
+ 'unodevtools'
+ 'unopkg|'
+ 'util|db|qe|' # for xmlsearch::
+ 'utl|'
+ 'vcl|'
+ 'writerfilter|'
+ 'xforms|'
+ 'xmloff|token|EnhancedCustomShapeToken' # for xmloff::
+ 'ZipUtils'
+ ')$', re.VERBOSE
+ )
+
+ reason = re.match(ns, line)
+ if reason:
+ # Warn about namespaces: if a header is suggested only '// for $namespace', then the namespace is not used
+ # otherwise the used classes name would show up after the '// for'
+ # Cleaning out the respective header (if there is any
+ # - which is not always the case) is for the next run!
+ nameSpace = reason.group(1).split(' ')[0]
+ print("WARNING:", fileName, "This 'using namespace' is likely unnecessary:", nameSpace)
+
+ # Get the row number, normal IWYU output does not contain this info
+ subprocess.run(["git", "grep", "-n", "using namespace.*"+nameSpace+";", fileName])
+
for remove in sorted(toRemove):
print("ERROR: %s: remove not needed include" % remove)
return len(toRemove)
-def run_tool(task_queue, failed_files, dontstop):
+def run_tool(task_queue, failed_files, dontstop, noexclude, checknamespaces):
while True:
invocation, moduleRules = task_queue.get()
if not len(failed_files):
print("[IWYU] " + invocation.split(' ')[-1])
p = subprocess.Popen(invocation, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
- retcode = processIWYUOutput(p.communicate()[0].decode('utf-8').splitlines(), moduleRules, invocation.split(' ')[-1])
- if retcode == -1:
+ retcode = processIWYUOutput(p.communicate()[0].decode('utf-8').splitlines(), moduleRules, invocation.split(' ')[-1], noexclude, checknamespaces)
+ if retcode == -1 and not checknamespaces:
print("ERROR: A file is probably not self contained, check this commands output:\n" + invocation)
elif retcode > 0:
print("ERROR: The following command found unused includes:\n" + invocation)
if not dontstop:
failed_files.append(invocation)
task_queue.task_done()
+ if checknamespaces:
+ # Workaround: sometimes running git grep makes the letters typed into the terminal disappear after the script is finished
+ os.system('stty sane')
def isInUnoIncludeFile(path):
@@ -244,7 +431,7 @@ def isInUnoIncludeFile(path):
or path.startswith("include/uno/")
-def tidy(compileCommands, paths, dontstop):
+def tidy(compileCommands, paths, dontstop, noexclude,checknamespaces):
return_code = 0
try:
@@ -252,7 +439,7 @@ def tidy(compileCommands, paths, dontstop):
task_queue = queue.Queue(max_task)
failed_files = []
for _ in range(max_task):
- t = threading.Thread(target=run_tool, args=(task_queue, failed_files, dontstop))
+ t = threading.Thread(target=run_tool, args=(task_queue, failed_files, dontstop, noexclude,checknamespaces))
t.daemon = True
t.start()
@@ -260,6 +447,10 @@ def tidy(compileCommands, paths, dontstop):
if isInUnoIncludeFile(path):
continue
+ # IWYU fails on these with #error: don't use this in new code
+ if path.startswith("include/vcl/toolkit"):
+ continue
+
moduleName = path.split("/")[0]
rulePath = os.path.join(moduleName, "IwyuFilter_" + moduleName + ".yaml")
@@ -306,7 +497,7 @@ def tidy(compileCommands, paths, dontstop):
def main(argv):
parser = argparse.ArgumentParser(description='Check source files for unneeded includes.')
- parser.add_argument('--dontstop', action='store_true',
+ parser.add_argument('--continue', action='store_true',
help='Don\'t stop on errors. Useful for periodic re-check of large amount of files')
parser.add_argument('Files' , nargs='*',
help='The files to be checked')
@@ -314,6 +505,12 @@ def main(argv):
help='Recursively search a directory for source files to check')
parser.add_argument('--headers', action='store_true',
help='Check header files. If omitted, check source files. Use with --recursive.')
+ parser.add_argument('--noexclude', action='store_true',
+ help='Ignore excludelist. Useful to check whether its exclusions are still all valid.')
+ parser.add_argument('--ns', action='store_true',
+ help='Warn about unused "using namespace" statements. '
+ 'Removing these may uncover more removable headers '
+ 'in a subsequent normal run')
args = parser.parse_args()
@@ -341,7 +538,28 @@ def main(argv):
print ("File 'compile_commands.json' does not exist, please run:\nmake vim-ide-integration")
sys.exit(-1)
- tidy(compileCommands, paths=list_of_files, dontstop=args.dontstop)
+ # quickly sanity check whether files with exceptions in yaml still exists
+ # only check for the module of the very first filename passed
+
+ # Verify there are files selected for checking, with --recursive it
+ # may happen that there are in fact no C/C++ files in a module directory
+ if not list_of_files:
+ print("No files found to check!")
+ sys.exit(-2)
+
+ moduleName = sorted(list_of_files)[0].split("/")[0]
+ rulePath = os.path.join(moduleName, "IwyuFilter_" + moduleName + ".yaml")
+ moduleRules = {}
+ if os.path.exists(rulePath):
+ moduleRules = yaml.full_load(open(rulePath))
+ if "excludelist" in moduleRules.keys():
+ excludelistRules = moduleRules["excludelist"]
+ for pathname in excludelistRules.keys():
+ file = pathlib.Path(pathname)
+ if not file.exists():
+ print("WARNING: File listed in " + rulePath + " no longer exists: " + pathname)
+
+ tidy(compileCommands, paths=list_of_files, dontstop=vars(args)["continue"], noexclude=args.noexclude, checknamespaces=args.ns)
if __name__ == '__main__':
main(sys.argv[1:])