Auto merge of #680 - puremourning:compilation-database, r=micbou

[READY] Automatically load a compilation database if found

Alternative implementation to : #679

See #679 for explanation and rationale.

Fixes: #489
Fixes: ycm-core/YouCompleteMe#2310 (sort of)
Fixes: #26
Fixes: ycm-core/YouCompleteMe#1981
Fixes: ycm-core/YouCompleteMe#1623 (sort of)
Fixes: ycm-core/YouCompleteMe#1622
Partly implements: ycm-core/YouCompleteMe#927
Fixes: ycm-core/YouCompleteMe#664
Fixes: ycm-core/YouCompleteMe#487
Fixes (discussion on): ycm-core/YouCompleteMe#174

This implementation is almost identical to the default-extra-conf version, except:

- the code lives in `flags.py`
- it re-uses `INCLUDE_FLAGS` from `flags.py`
- it provides the directory of the compilation database in the `clang_completer` debug info instead of the name of the default extra conf module.

On reflection, I think I might prefer this implementation.

For now, the tests are identical to the other PR (you can now see why I added that set of tests!)

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/valloric/ycmd/680)
<!-- Reviewable:end -->

cpp/ycm/ClangCompleter/CompilationDatabase.cpp

@@ -41,11 +41,11 @@ remove_pointer< CXCompileCommands >::type > CompileCommandsWrap;
 
 CompilationDatabase::CompilationDatabase(
   const boost::python::object &path_to_directory )
-  : is_loaded_( false ) {
+  : is_loaded_( false )
+  , path_to_directory_( GetUtf8String( path_to_directory ) ) {
   CXCompilationDatabase_Error status;
-  std::string path_to_directory_string = GetUtf8String( path_to_directory );
   compilation_database_ = clang_CompilationDatabase_fromDirectory(
-                            path_to_directory_string.c_str(),
+                            path_to_directory_.c_str(),
                             &status );
   is_loaded_ = status == CXCompilationDatabase_NoError;
 }

cpp/ycm/ClangCompleter/CompilationDatabase.h

@@ -54,9 +54,14 @@ class CompilationDatabase : boost::noncopyable {
   CompilationInfoForFile GetCompilationInfoForFile(
     const boost::python::object &path_to_file );
 
+  std::string GetDatabaseDirectory() {
+    return path_to_directory_;
+  }
+
 private:
 
   bool is_loaded_;
+  std::string path_to_directory_;
   CXCompilationDatabase compilation_database_;
   boost::mutex compilation_database_mutex_;
 };

cpp/ycm/ycm_core.cpp

@@ -197,7 +197,9 @@ BOOST_PYTHON_MODULE(ycm_core)
     .def( "AlreadyGettingFlags",
           &CompilationDatabase::AlreadyGettingFlags )
     .def( "GetCompilationInfoForFile",
-          &CompilationDatabase::GetCompilationInfoForFile );
+          &CompilationDatabase::GetCompilationInfoForFile )
+    .def_readonly( "database_directory",
+                   &CompilationDatabase::GetDatabaseDirectory );
 
   class_< CompilationInfoForFile,
       boost::shared_ptr< CompilationInfoForFile > >(

ycmd/completers/cpp/clang_completer.py

@@ -34,7 +34,8 @@
 from ycmd.utils import ToCppStringCompatible, ToUnicode
 from ycmd.completers.completer import Completer
 from ycmd.completers.completer_utils import GetIncludeStatementValue
-from ycmd.completers.cpp.flags import Flags, PrepareFlagsForClang
+from ycmd.completers.cpp.flags import ( Flags, PrepareFlagsForClang,
+                                        NoCompilationDatabase )
 from ycmd.completers.cpp.ephemeral_values_set import EphemeralValuesSet
 from ycmd.responses import NoExtraConfDetected, UnknownExtraConf
 
@@ -373,22 +374,49 @@ def DebugInfo( self, request_data ):
     filename = request_data[ 'filepath' ]
     try:
       extra_conf = extra_conf_store.ModuleFileForSourceFile( filename )
-      flags = self._FlagsForRequest( request_data ) or []
-    except NoExtraConfDetected:
-      return ( 'C-family completer debug information:\n'
-               '  No configuration file found' )
     except UnknownExtraConf as error:
       return ( 'C-family completer debug information:\n'
                '  Configuration file found but not loaded\n'
                '  Configuration path: {0}'.format(
                  error.extra_conf_file ) )
-    if not extra_conf:
+
+    try:
+      # Note that it only raises NoExtraConfDetected:
+      #  - when extra_conf is None and,
+      #  - there is no compilation database
+      flags = self._FlagsForRequest( request_data )
+    except NoExtraConfDetected:
+      # No flags
+      return ( 'C-family completer debug information:\n'
+               '  No configuration file found\n'
+               '  No compilation database found' )
+
+    # If _FlagsForRequest returns None or raises, we use an empty list in
+    # practice.
+    flags = flags or []
+
+    if extra_conf:
+      # We got the flags from the extra conf file
+      return ( 'C-family completer debug information:\n'
+               '  Configuration file found and loaded\n'
+               '  Configuration path: {0}\n'
+               '  Flags: {1}'.format( extra_conf, list( flags ) ) )
+
+    try:
+      database = self._flags.FindCompilationDatabase(
+          os.path.dirname( filename ) )
+    except NoCompilationDatabase:
+      # No flags
       return ( 'C-family completer debug information:\n'
-               '  No configuration file found' )
+               '  No configuration file found\n'
+               '  No compilation database found' )
+
+    # We got the flags from the compilation database
     return ( 'C-family completer debug information:\n'
-             '  Configuration file found and loaded\n'
-             '  Configuration path: {0}\n'
-             '  Flags: {1}'.format( extra_conf, list( flags ) ) )
+             '  No configuration file found\n'
+             '  Using compilation database from: {0}\n'
+             '  Flags: {1}'.format( database.database_directory,
+                                    list( flags ) ) )
 
 
   def _FlagsForRequest( self, request_data ):

ycmd/completers/cpp/flags.py

@@ -30,13 +30,18 @@
 from future.utils import PY2, native
 from ycmd import extra_conf_store
 from ycmd.utils import ( ToCppStringCompatible, OnMac, OnWindows, ToUnicode,
-                         ToBytes )
+                         ToBytes, PathsToAllParentFolders )
 from ycmd.responses import NoExtraConfDetected
 
+
 INCLUDE_FLAGS = [ '-isystem', '-I', '-iquote', '-isysroot', '--sysroot',
                   '-gcc-toolchain', '-include', '-include-pch', '-iframework',
                   '-F', '-imacros' ]
 
+# --sysroot= must be first (or at least, before --sysroot) because the latter is
+# a prefix of the former (and the algorithm checks prefixes)
+PATH_FLAGS =  [ '--sysroot=' ] + INCLUDE_FLAGS
+
 # We need to remove --fcolor-diagnostics because it will cause shell escape
 # sequences to show up in editors, which is bad. See Valloric/YouCompleteMe#1421
 STATE_FLAGS_TO_SKIP = set( [ '-c',
@@ -59,6 +64,23 @@
 # See Valloric/ycmd#266
 CPP_COMPILER_REGEX = re.compile( r'\+\+(-\d+(\.\d+){0,2})?$' )
 
+# List of file extensions to be considered "header" files and thus not present
+# in the compilation database. The logic will try and find an associated
+# "source" file (see SOURCE_EXTENSIONS below) and use the flags for that.
+HEADER_EXTENSIONS = [ '.h', '.hxx', '.hpp', '.hh' ]
+
+# List of file extensions which are considered "source" files for the purposes
+# of heuristically locating the flags for a header file.
+SOURCE_EXTENSIONS = [ '.cpp', '.cxx', '.cc', '.c', '.m', '.mm' ]
+
+EMPTY_FLAGS = {
+  'flags': [],
+}
+
+
+class NoCompilationDatabase( Exception ):
+  pass
+
 
 class Flags( object ):
   """Keeps track of the flags necessary to compile a file.
@@ -71,42 +93,70 @@ def __init__( self ):
     self.extra_clang_flags = _ExtraClangFlags()
     self.no_extra_conf_file_warning_posted = False
 
+    # We cache the compilation database for any given source directory
+    # Keys are directory names and values are ycm_core.CompilationDatabase
+    # instances or None. Value is None when it is known there is no compilation
+    # database to be found for the directory.
+    self.compilation_database_dir_map = dict()
+
+    # Sometimes we don't actually know what the flags to use are. Rather than
+    # returning no flags, if we've previously found flags for a file in a
+    # particular directory, return them. These will probably work in a high
+    # percentage of cases and allow new files (which are not yet in the
+    # compilation database) to receive at least some flags.
+    # Keys are directory names and values are ycm_core.CompilationInfo
+    # instances. Values may not be None.
+    self.file_directory_heuristic_map = dict()
+
 
   def FlagsForFile( self,
                     filename,
                     add_extra_clang_flags = True,
                     client_data = None ):
+
+    # The try-catch here is to avoid a synchronisation primitive. This method
+    # may be called from multiple threads, and python gives us
+    # 1-python-statement synchronisation for "free" (via the GIL)
     try:
       return self.flags_for_file[ filename ]
     except KeyError:
-      module = extra_conf_store.ModuleForSourceFile( filename )
-      if not module:
-        if not self.no_extra_conf_file_warning_posted:
-          self.no_extra_conf_file_warning_posted = True
-          raise NoExtraConfDetected
-        return None
-
-      results = _CallExtraConfFlagsForFile( module,
-                                            filename,
-                                            client_data )
+      pass
 
-      if not results or not results.get( 'flags_ready', True ):
-        return None
+    module = extra_conf_store.ModuleForSourceFile( filename )
+    try:
+      results = self._GetFlagsFromExtraConfOrDatabase( module,
+                                                       filename,
+                                                       client_data )
+    except NoCompilationDatabase:
+      if not self.no_extra_conf_file_warning_posted:
+        self.no_extra_conf_file_warning_posted = True
+        raise NoExtraConfDetected
+      return None
+
+    if not results or not results.get( 'flags_ready', True ):
+      return None
+
+    flags = _ExtractFlagsList( results )
+    if not flags:
+      return None
+
+    if add_extra_clang_flags:
+      flags += self.extra_clang_flags
+
+    sanitized_flags = PrepareFlagsForClang( flags,
+                                            filename,
+                                            add_extra_clang_flags )
 
-      flags = _ExtractFlagsList( results )
-      if not flags:
-        return None
+    if results.get( 'do_cache', True ):
+      self.flags_for_file[ filename ] = sanitized_flags
+    return sanitized_flags
 
-      if add_extra_clang_flags:
-        flags += self.extra_clang_flags
 
-      sanitized_flags = PrepareFlagsForClang( flags,
-                                              filename,
-                                              add_extra_clang_flags )
+  def _GetFlagsFromExtraConfOrDatabase( self, module, filename, client_data ):
+    if not module:
+      return self._GetFlagsFromCompilationDatabase( filename )
 
-      if results.get( 'do_cache', True ):
-        self.flags_for_file[ filename ] = sanitized_flags
-      return sanitized_flags
+    return _CallExtraConfFlagsForFile( module, filename, client_data )
 
 
   def UserIncludePaths( self, filename, client_data ):
@@ -148,6 +198,77 @@ def UserIncludePaths( self, filename, client_data ):
 
   def Clear( self ):
     self.flags_for_file.clear()
+    self.compilation_database_dir_map.clear()
+    self.file_directory_heuristic_map.clear()
+
+
+  def _GetFlagsFromCompilationDatabase( self, file_name ):
+    file_dir = os.path.dirname( file_name )
+    file_root, file_extension = os.path.splitext( file_name )
+
+    database = self.FindCompilationDatabase( file_dir )
+    compilation_info = _GetCompilationInfoForFile( database,
+                                                   file_name,
+                                                   file_extension )
+
+    if not compilation_info:
+      # Note: Try-catch here synchronises access to the cache (as this can be
+      # called from multiple threads).
+      try:
+        # We previously saw a file in this directory. As a guess, just
+        # return the flags for that file. Hopefully this will at least give some
+        # meaningful compilation.
+        compilation_info = self.file_directory_heuristic_map[ file_dir ]
+      except KeyError:
+        # No cache for this directory and there are no flags for this file in
+        # the database.
+        return EMPTY_FLAGS
+
+    # If this is the first file we've seen in path file_dir, cache the
+    # compilation_info for it in case we see a file in the same dir with no
+    # flags available.
+    # The following updates file_directory_heuristic_map if and only if file_dir
+    # isn't already there. This works around a race condition where 2 threads
+    # could be executing this method in parallel.
+    self.file_directory_heuristic_map.setdefault( file_dir, compilation_info )
+
+    return {
+      'flags': _MakeRelativePathsInFlagsAbsolute(
+        compilation_info.compiler_flags_,
+        compilation_info.compiler_working_dir_ ),
+    }
+
+
+  # Return a compilation database object for the supplied path. Raises
+  # NoCompilationDatabase if no compilation database can be found.
+  def FindCompilationDatabase( self, file_dir ):
+    # We search up the directory hierarchy, to first see if we have a
+    # compilation database already for that path, or if a compile_commands.json
+    # file exists in that directory.
+    for folder in PathsToAllParentFolders( file_dir ):
+      # Try/catch to syncronise access to cache
+      try:
+        database = self.compilation_database_dir_map[ folder ]
+        if database:
+          return database
+
+        raise NoCompilationDatabase
+      except KeyError:
+        pass
+
+      compile_commands = os.path.join( folder, 'compile_commands.json' )
+      if os.path.exists( compile_commands ):
+        database = ycm_core.CompilationDatabase( folder )
+
+        if database.DatabaseSuccessfullyLoaded():
+          self.compilation_database_dir_map[ folder ] = database
+          return database
+
+    # Nothing was found. No compilation flags are available.
+    # Note: we cache the fact that none was found for this folder to speed up
+    # subsequent searches.
+    self.compilation_database_dir_map[ file_dir ] = None
+    raise NoCompilationDatabase
 
 
 def _ExtractFlagsList( flags_for_file_output ):
@@ -417,3 +538,67 @@ def _SpecialClangIncludes():
   libclang_dir = os.path.dirname( ycm_core.__file__ )
   path_to_includes = os.path.join( libclang_dir, 'clang_includes' )
   return [ '-resource-dir=' + path_to_includes ]
+
+
+def _MakeRelativePathsInFlagsAbsolute( flags, working_directory ):
+  if not working_directory:
+    return list( flags )
+  new_flags = []
+  make_next_absolute = False
+  for flag in flags:
+    new_flag = flag
+
+    if make_next_absolute:
+      make_next_absolute = False
+      if not os.path.isabs( new_flag ):
+        new_flag = os.path.join( working_directory, flag )
+      new_flag = os.path.normpath( new_flag )
+    else:
+      for path_flag in PATH_FLAGS:
+        # Single dash argument alone, e.g. -isysroot <path>
+        if flag == path_flag:
+          make_next_absolute = True
+          break
+
+        # Single dash argument with inbuilt path, e.g. -isysroot<path>
+        # or double-dash argument, e.g. --isysroot=<path>
+        if flag.startswith( path_flag ):
+          path = flag[ len( path_flag ): ]
+          if not os.path.isabs( path ):
+            path = os.path.join( working_directory, path )
+          path = os.path.normpath( path )
+
+          new_flag = '{0}{1}'.format( path_flag, path )
+          break
+
+    if new_flag:
+      new_flags.append( new_flag )
+  return new_flags
+
+
+# Find the compilation info structure from the supplied database for the
+# supplied file. If the source file is a header, try and find an appropriate
+# source file and return the compilation_info for that.
+def _GetCompilationInfoForFile( database, file_name, file_extension ):
+  # The compilation_commands.json file generated by CMake does not have entries
+  # for header files. So we do our best by asking the db for flags for a
+  # corresponding source file, if any. If one exists, the flags for that file
+  # should be good enough.
+  if file_extension in HEADER_EXTENSIONS:
+    for extension in SOURCE_EXTENSIONS:
+      replacement_file = os.path.splitext( file_name )[ 0 ] + extension
+      compilation_info = database.GetCompilationInfoForFile(
+        replacement_file )
+      if compilation_info and compilation_info.compiler_flags_:
+        return compilation_info
+
+    # No corresponding source file was found, so we can't generate any flags for
+    # this header file.
+    return None
+
+  # It's a source file. Just ask the database for the flags.
+  compilation_info = database.GetCompilationInfoForFile( file_name )
+  if compilation_info.compiler_flags_:
+    return compilation_info
+
+  return None