Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add recipe for the LLVM core libraries #3375

Merged
merged 37 commits into from
May 24, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
6c6d87b
Added recipe for the LLVM core libraries
newlawrence Nov 1, 2020
8e4e4d0
Reuse cmake configuration
newlawrence Nov 1, 2020
daa072b
Fix shared builds for 10x and 11x
newlawrence Nov 1, 2020
b735998
Parse dependencies from graphviz output
newlawrence Nov 2, 2020
edf9abc
Beware of dummy targets
newlawrence Nov 2, 2020
aebb740
Keep only latest version for the moment
newlawrence Nov 2, 2020
6fdd6a6
Beware of Windows line endings
newlawrence Nov 3, 2020
00c52c8
Relax cmake version requirement
newlawrence Nov 3, 2020
41562aa
Add list of unsupported compiler configurations
newlawrence Nov 3, 2020
d876521
Commit suggestion
newlawrence Nov 3, 2020
9cb7274
Cannot discriminate Visual Studio by its minor version
newlawrence Nov 4, 2020
d8286db
Disable object targets properly
newlawrence Nov 4, 2020
b9347da
Use external dependencies properly
newlawrence Nov 5, 2020
881ce17
Enable building with libffi in Windows
newlawrence Nov 5, 2020
d0c2201
Do not force libc++ in clang builds
newlawrence Nov 6, 2020
329d69e
Wrong value for a cmake flag
newlawrence Nov 6, 2020
d876b34
Building native tools were making some configurations fail
newlawrence Nov 6, 2020
c9a784c
Allow Visual Studio generators now that native tools are disabled
newlawrence Nov 6, 2020
cf7b3c0
Disable relative paths for the sources in the debug info
newlawrence Nov 6, 2020
8ce9dec
Update supported compilers
newlawrence Nov 7, 2020
01e6a26
String comparisons are dangerous
newlawrence Nov 7, 2020
33c36f2
Remove iconv patch comment
newlawrence Nov 7, 2020
bb7baec
Commit suggestion
newlawrence Nov 8, 2020
59d8248
More meaningful exception messages
newlawrence Nov 8, 2020
1ef186a
Remove unnecessary readme
newlawrence Nov 8, 2020
d30f900
Support older Python versions
newlawrence Nov 11, 2020
7c9f6ef
Empty commit to trigger the CI
newlawrence Nov 22, 2020
304a14e
Remove unnecessary status message
newlawrence Jan 4, 2021
563e571
Less strict cmake minimum required version
newlawrence Jan 4, 2021
e79842e
LLVM version bumped to 11.0.1
newlawrence Jan 16, 2021
c0ab4e7
LLVM version bumped to 11.1.0
newlawrence Apr 7, 2021
194cf21
Deactivate fPIC if shared
newlawrence Apr 9, 2021
e021639
Fix fPIC flag for shared builds
newlawrence Apr 12, 2021
d99091d
Install LLVMTableGenGlobalISel manually
newlawrence Apr 12, 2021
c4ae50c
Add option use_sanitizer
newlawrence May 21, 2021
e5b2dee
Disable all shared builds
newlawrence May 21, 2021
f6a023b
Dedent conandata.yml
newlawrence May 22, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions recipes/llvm-core/all/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
cmake_minimum_required(VERSION 3.13.4)
project(conanllvm)

include(${CMAKE_CURRENT_BINARY_DIR}/conanbuildinfo.cmake)
conan_basic_setup()

add_subdirectory("source")
13 changes: 13 additions & 0 deletions recipes/llvm-core/all/conandata.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
sources:
"11.1.0":
url: https://github.com/llvm/llvm-project/releases/download/llvmorg-11.1.0/llvm-11.1.0.src.tar.xz
sha256: ce8508e318a01a63d4e8b3090ab2ded3c598a50258cc49e2625b9120d4c03ea5

patches:
"11.1.0":
- base_path: "source"
patch_file: "patches/11.1.0-vs2019.patch"
- base_path: "source"
patch_file: "patches/11.1.0-cmake.patch"
- base_path: "source"
patch_file: "patches/11.1.0-native.patch"
319 changes: 319 additions & 0 deletions recipes/llvm-core/all/conanfile.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,319 @@
from conans.errors import ConanInvalidConfiguration
from conans import ConanFile, CMake, tools

from collections import defaultdict
import json
import re
import os.path
import os


class LLVMCoreConan(ConanFile):
name = 'llvm-core'
description = (
'A toolkit for the construction of highly optimized compilers,'
'optimizers, and runtime environments.'
)
license = 'Apache-2.0 WITH LLVM-exception'
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As a remark for all reviewers (and my future self), this is actually the correct SPDX annotation 👍

topics = ('conan', 'llvm')
homepage = 'https://github.com/llvm/llvm-project/tree/master/llvm'
url = 'https://github.com/conan-io/conan-center-index'

settings = ('os', 'arch', 'compiler', 'build_type')
options = {
'shared': [True, False],
'fPIC': [True, False],
'components': 'ANY',
'targets': 'ANY',
'exceptions': [True, False],
'rtti': [True, False],
'threads': [True, False],
'lto': ['On', 'Off', 'Full', 'Thin'],
'static_stdlib': [True, False],
'unwind_tables': [True, False],
'expensive_checks': [True, False],
'use_perf': [True, False],
'use_sanitizer': [
'Address',
'Memory',
'MemoryWithOrigins',
'Undefined',
'Thread',
'DataFlow',
'Address;Undefined',
'None'
],
'with_ffi': [True, False],
'with_zlib': [True, False],
'with_xml2': [True, False]
}
default_options = {
'shared': False,
'fPIC': True,
'components': 'all',
'targets': 'all',
'exceptions': True,
'rtti': True,
'threads': True,
'lto': 'Off',
'static_stdlib': False,
'unwind_tables': True,
'expensive_checks': False,
'use_perf': False,
'use_sanitizer': 'None',
'with_ffi': False,
'with_zlib': True,
'with_xml2': True
}

exports_sources = ['CMakeLists.txt', 'patches/*']
generators = ['cmake', 'cmake_find_package']
no_copy_source = True

@property
def _source_subfolder(self):
return 'source'

def _supports_compiler(self):
compiler = self.settings.compiler.value
version = tools.Version(self.settings.compiler.version)
major_rev, minor_rev = int(version.major), int(version.minor)

unsupported_combinations = [
[compiler == 'gcc', major_rev == 5, minor_rev < 1],
[compiler == 'gcc', major_rev < 5],
[compiler == 'clang', major_rev < 4],
[compiler == 'apple-clang', major_rev < 9],
[compiler == 'Visual Studio', major_rev < 15]
]
if any(all(combination) for combination in unsupported_combinations):
message = 'unsupported compiler: "{}", version "{}"'
raise ConanInvalidConfiguration(message.format(compiler, version))

def _patch_sources(self):
for patch in self.conan_data.get('patches', {}).get(self.version, []):
tools.patch(**patch)

def _patch_build(self):
if os.path.exists('FindIconv.cmake'):
tools.replace_in_file('FindIconv.cmake', 'iconv charset', 'iconv')

def _configure_cmake(self):
cmake = CMake(self)
cmake.definitions['BUILD_SHARED_LIBS'] = False
cmake.definitions['CMAKE_SKIP_RPATH'] = True
cmake.definitions['CMAKE_POSITION_INDEPENDENT_CODE'] = \
self.options.get_safe('fPIC', default=False) or self.options.shared

if not self.options.shared:
cmake.definitions['DISABLE_LLVM_LINK_LLVM_DYLIB'] = True
# cmake.definitions['LLVM_LINK_DYLIB'] = self.options.shared

cmake.definitions['LLVM_TARGET_ARCH'] = 'host'
cmake.definitions['LLVM_TARGETS_TO_BUILD'] = self.options.targets
cmake.definitions['LLVM_BUILD_LLVM_DYLIB'] = self.options.shared
cmake.definitions['LLVM_DYLIB_COMPONENTS'] = self.options.components
cmake.definitions['LLVM_ENABLE_PIC'] = \
self.options.get_safe('fPIC', default=False)

cmake.definitions['LLVM_ABI_BREAKING_CHECKS'] = 'WITH_ASSERTS'
cmake.definitions['LLVM_ENABLE_WARNINGS'] = True
cmake.definitions['LLVM_ENABLE_PEDANTIC'] = True
cmake.definitions['LLVM_ENABLE_WERROR'] = False

cmake.definitions['LLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN'] = True
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

While we wait for the CI confirmation we have all green, this is the only thing left in my opinion. This flag is to avoid failure on the Visual Studio 2019 builds with the version we currently have (16.4) because of this issue.

I'd like to disable it before merging to master as I'm not sure what "Visual Studio version 16.4 and lower is known to miscompile part of LLVM" implies, so better to wait until we have an updated version in the CI. A hard error won't pass unnoticed to someone trying to build a package using an old version.

cmake.definitions['LLVM_USE_RELATIVE_PATHS_IN_DEBUG_INFO'] = False
cmake.definitions['LLVM_BUILD_INSTRUMENTED_COVERAGE'] = False
cmake.definitions['LLVM_OPTIMIZED_TABLEGEN'] = True
cmake.definitions['LLVM_REVERSE_ITERATION'] = False
cmake.definitions['LLVM_ENABLE_BINDINGS'] = False
cmake.definitions['LLVM_CCACHE_BUILD'] = False

cmake.definitions['LLVM_INCLUDE_TOOLS'] = self.options.shared
cmake.definitions['LLVM_INCLUDE_EXAMPLES'] = False
cmake.definitions['LLVM_INCLUDE_TESTS'] = False
cmake.definitions['LLVM_INCLUDE_BENCHMARKS'] = False
cmake.definitions['LLVM_APPEND_VC_REV'] = False
cmake.definitions['LLVM_BUILD_DOCS'] = False
cmake.definitions['LLVM_ENABLE_IDE'] = False

cmake.definitions['LLVM_ENABLE_EH'] = self.options.exceptions
cmake.definitions['LLVM_ENABLE_RTTI'] = self.options.rtti
cmake.definitions['LLVM_ENABLE_THREADS'] = self.options.threads
cmake.definitions['LLVM_ENABLE_LTO'] = self.options.lto
cmake.definitions['LLVM_STATIC_LINK_CXX_STDLIB'] = \
self.options.static_stdlib
cmake.definitions['LLVM_ENABLE_UNWIND_TABLES'] = \
self.options.unwind_tables
cmake.definitions['LLVM_ENABLE_EXPENSIVE_CHECKS'] = \
self.options.expensive_checks
cmake.definitions['LLVM_ENABLE_ASSERTIONS'] = \
self.settings.build_type == 'Debug'

cmake.definitions['LLVM_USE_NEWPM'] = False
cmake.definitions['LLVM_USE_OPROFILE'] = False
cmake.definitions['LLVM_USE_PERF'] = self.options.use_perf
if self.options.use_sanitizer == 'None':
cmake.definitions['LLVM_USE_SANITIZER'] = ''
else:
cmake.definitions['LLVM_USE_SANITIZER'] = \
self.options.use_sanitizer

cmake.definitions['LLVM_ENABLE_Z3_SOLVER'] = False
cmake.definitions['LLVM_ENABLE_LIBPFM'] = False
cmake.definitions['LLVM_ENABLE_LIBEDIT'] = False
cmake.definitions['LLVM_ENABLE_FFI'] = self.options.with_ffi
cmake.definitions['LLVM_ENABLE_ZLIB'] = \
self.options.get_safe('with_zlib', False)
cmake.definitions['LLVM_ENABLE_LIBXML2'] = \
self.options.get_safe('with_xml2', False)
return cmake

def config_options(self):
if self.settings.os == 'Windows':
del self.options.fPIC
del self.options.with_zlib
del self.options.with_xml2

def requirements(self):
if self.options.with_ffi:
self.requires('libffi/3.3')
if self.options.get_safe('with_zlib', False):
self.requires('zlib/1.2.11')
if self.options.get_safe('with_xml2', False):
self.requires('libxml2/2.9.10')

def configure(self):
if self.options.shared: # Shared builds disabled just due to the CI
message = 'Shared builds not currently supported'
raise ConanInvalidConfiguration(message)
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I didn't remove the code to build the package as a shared library but to raise an invalid configuration exception in all cases instead. In case someone wants to tweak the package to recover the lost functionality or, for us in the future, it is as easy as to remove this two lines and uncomment the following ones.

# del self.options.fPIC
# if self.settings.os == 'Windows' and self.options.shared:
# message = 'Shared builds not supported on Windows'
# raise ConanInvalidConfiguration(message)
if self.options.exceptions and not self.options.rtti:
message = 'Cannot enable exceptions without rtti support'
raise ConanInvalidConfiguration(message)
self._supports_compiler()

def source(self):
tools.get(**self.conan_data['sources'][self.version])
os.rename('llvm-{}.src'.format(self.version), self._source_subfolder)
self._patch_sources()
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Patches should be applied in build method.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Even in this case where we are setting no_copy_source to True?

The patches applied here work for all the different build configurations unconditionally, therefore they can be applied only once. If we were to apply the patches in the build method we'd have no other choice but to copy the sources, which I wouldn't recommend given their size.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it is always better to copy the sources. However, given the size of the package and the artifacts, and the limitation of the build machines, an exception is necessary.


def build(self):
self._patch_build()
cmake = self._configure_cmake()
cmake.configure()
cmake.build()

def package(self):
self.copy('LICENSE.TXT', dst='licenses', src=self._source_subfolder)
lib_path = os.path.join(self.package_folder, 'lib')

cmake = self._configure_cmake()
cmake.install()

if not self.options.shared:
for ext in ['.a', '.lib']:
lib = '*LLVMTableGenGlobalISel{}'.format(ext)
self.copy(lib, dst='lib', src='lib')

self.run('cmake --graphviz=graph/llvm.dot .')
with tools.chdir('graph'):
dot_text = tools.load('llvm.dot').replace('\r\n', '\n')

dep_regex = re.compile(r'//\s(.+)\s->\s(.+)$', re.MULTILINE)
deps = re.findall(dep_regex, dot_text)

dummy_targets = defaultdict(list)
for target, dep in deps:
if not target.startswith('LLVM'):
dummy_targets[target].append(dep)

cmake_targets = {
'libffi::libffi': 'ffi',
'ZLIB::ZLIB': 'z',
'Iconv::Iconv': 'iconv',
'LibXml2::LibXml2': 'xml2'
}

components = defaultdict(list)
for lib, dep in deps:
if not lib.startswith('LLVM'):
continue
elif dep.startswith('-delayload:'):
continue
elif dep.startswith('LLVM'):
components[dep]
elif dep in cmake_targets:
dep = cmake_targets[dep]
elif os.path.exists(dep):
dep = os.path.splitext(os.path.basename(dep))[0]
dep = dep.replace('lib', '')
dep = dep.replace('-l', '')

if dep in dummy_targets.keys():
components[lib].extend(dummy_targets[dep])
components[lib] = list(set(components[lib]))
else:
components[lib].append(dep)

tools.rmdir(os.path.join(self.package_folder, 'bin'))
tools.rmdir(os.path.join(self.package_folder, 'lib', 'cmake'))
tools.rmdir(os.path.join(self.package_folder, 'share'))

for name in os.listdir(lib_path):
if 'LLVM' not in name:
os.remove(os.path.join(lib_path, name))

if not self.options.shared:
components_path = \
os.path.join(self.package_folder, 'lib', 'components.json')
with open(components_path, 'w') as components_file:
json.dump(components, components_file, indent=4)
else:
suffixes = ['.dylib', '.so']
for name in os.listdir(lib_path):
if not any(suffix in name for suffix in suffixes):
os.remove(os.path.join(lib_path, name))

def package_info(self):
if self.options.shared:
self.cpp_info.libs = tools.collect_libs(self)
if self.settings.os == 'Linux':
self.cpp_info.system_libs = ['tinfo', 'pthread']
self.cpp_info.system_libs.extend(['rt', 'dl', 'm'])
elif self.settings.os == 'Macos':
self.cpp_info.system_libs = ['curses', 'm']
return

components_path = \
os.path.join(self.package_folder, 'lib', 'components.json')
with open(components_path, 'r') as components_file:
components = json.load(components_file)

dependencies = ['ffi', 'z', 'iconv', 'xml2']
targets = {
'ffi': 'libffi::libffi',
'z': 'zlib::zlib',
'xml2': 'libxml2::libxml2'
}

for lib, deps in components.items():
component = lib[4:].replace('LLVM', '').lower()

self.cpp_info.components[component].libs = [lib]

self.cpp_info.components[component].requires = [
dep[4:].replace('LLVM', '').lower()
for dep in deps if dep.startswith('LLVM')
]
for lib, target in targets.items():
if lib in deps:
self.cpp_info.components[component].requires.append(target)

self.cpp_info.components[component].system_libs = [
dep for dep in deps
if not dep.startswith('LLVM') and dep not in dependencies
]
57 changes: 57 additions & 0 deletions recipes/llvm-core/all/patches/11.1.0-cmake.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
--- cmake/config-ix.cmake
+++ cmake/config-ix.cmake
@@ -159,6 +159,9 @@
set(LIBXML2_FOUND 0)
if((LLVM_ENABLE_LIBXML2) AND ((CMAKE_SYSTEM_NAME MATCHES "Linux") AND (ICONV_LIBRARY_PATH) OR APPLE))
find_package(LibXml2)
+ set(LIBXML2_FOUND 1)
+ list(GET LibXml2_INCLUDE_DIRS -1 LIBXML2_INCLUDE_DIR)
+ set(LIBXML2_LIBRARIES ${LibXml2_LIBRARIES})
if (LIBXML2_FOUND)
set(LLVM_LIBXML2_ENABLED 1)
if ((CMAKE_OSX_SYSROOT) AND (EXISTS ${CMAKE_OSX_SYSROOT}/${LIBXML2_INCLUDE_DIR}))
@@ -321,7 +321,7 @@
message(FATAL_ERROR "libffi includes are not found.")
endif()

- find_library(FFI_LIBRARY_PATH ffi PATHS ${FFI_LIBRARY_DIR})
+ find_library(FFI_LIBRARY_PATH NAMES ffi libffi PATHS ${FFI_LIBRARY_DIR})
if( NOT FFI_LIBRARY_PATH )
message(FATAL_ERROR "libffi is not found.")
endif()
--- cmake/modules/TableGen.cmake
+++ cmake/modules/TableGen.cmake
@@ -138,12 +138,7 @@
macro(add_tablegen target project)
set(${target}_OLD_LLVM_LINK_COMPONENTS ${LLVM_LINK_COMPONENTS})
set(LLVM_LINK_COMPONENTS ${LLVM_LINK_COMPONENTS} TableGen)
-
- # CMake-3.9 doesn't let compilation units depend on their dependent libraries.
- if(NOT (CMAKE_GENERATOR STREQUAL "Ninja" AND NOT CMAKE_VERSION VERSION_LESS 3.9) AND NOT XCODE)
- # FIXME: It leaks to user, callee of add_tablegen.
- set(LLVM_ENABLE_OBJLIB ON)
- endif()
+ set(LLVM_ENABLE_OBJLIB OFF)

add_llvm_executable(${target} DISABLE_LLVM_LINK_LLVM_DYLIB ${ARGN})
set(LLVM_LINK_COMPONENTS ${${target}_OLD_LLVM_LINK_COMPONENTS})
--- lib/WindowsManifest/CMakeLists.txt
+++ lib/WindowsManifest/CMakeLists.txt
@@ -8,16 +8,6 @@
if(LIBXML2_LIBRARIES)
target_link_libraries(LLVMWindowsManifest PUBLIC ${LIBXML2_LIBRARIES})

- get_filename_component(xml2_library ${LIBXML2_LIBRARIES} NAME)
- if (CMAKE_STATIC_LIBRARY_PREFIX AND
- xml2_library MATCHES "^${CMAKE_STATIC_LIBRARY_PREFIX}.*${CMAKE_STATIC_LIBRARY_SUFFIX}$")
- string(REGEX REPLACE "^${CMAKE_STATIC_LIBRARY_PREFIX}" "" xml2_library ${xml2_library})
- string(REGEX REPLACE "${CMAKE_STATIC_LIBRARY_SUFFIX}$" "" xml2_library ${xml2_library})
- elseif (CMAKE_SHARED_LIBRARY_PREFIX AND
- xml2_library MATCHES "^${CMAKE_SHARED_LIBRARY_PREFIX}.*${CMAKE_SHARED_LIBRARY_SUFFIX}$")
- string(REGEX REPLACE "^${CMAKE_SHARED_LIBRARY_PREFIX}" "" xml2_library ${xml2_library})
- string(REGEX REPLACE "${CMAKE_SHARED_LIBRARY_SUFFIX}$" "" xml2_library ${xml2_library})
- endif()
set_property(TARGET LLVMWindowsManifest PROPERTY
- LLVM_SYSTEM_LIBS ${xml2_library})
+ LLVM_SYSTEM_LIBS ${LIBXML2_LIBRARIES})
endif()
Loading