Fix how we locate Clang runtime library directory (#182)

Also add a branch.txt file and logic in build_and_release.sh to use it, as specified in https://github.com/yugabyte/yugabyte-db-thirdparty#setting-up-a-new-release-version-branch

Author: mbautin

branch.txt

@@ -0,0 +1 @@
+2.17.3

build_and_release.sh

@@ -155,7 +155,17 @@ fi
 
 original_repo_dir=$PWD
 git_sha1=$( git rev-parse HEAD )
-tag=v$( date +%Y%m%d%H%M%S )-${git_sha1:0:10}
+
+branch_file_path="$YB_THIRDPARTY_DIR/branch.txt"
+branch_name=""
+if [[ -f ${branch_file_path} ]]; then
+  branch_name=$(<"${branch_file_path}")
+fi
+tag=v
+if [[ -n ${branch_name} ]]; then
+  tag+="${branch_name}-"
+fi
+tag+=$( date +%Y%m%d%H%M%S )-${git_sha1:0:10}
 
 archive_dir_name=yugabyte-db-thirdparty-$tag
 if [[ -z ${YB_THIRDPARTY_ARCHIVE_NAME_SUFFIX:-} ]]; then

python/yugabyte_db_thirdparty/builder.py

@@ -985,13 +985,21 @@ def init_linux_clang_flags(self, dep: Dependency) -> None:
                 # parts of the runtime library and C++ standard libraries are present.
 
             assert self.compiler_choice.cc is not None
-            compiler_rt_lib_dir = get_clang_library_dir(self.compiler_choice.get_c_compiler())
-            self.add_lib_dir_and_rpath(compiler_rt_lib_dir)
             ubsan_lib_candidates = []
             ubsan_lib_found = False
             for ubsan_lib_arch_suffix in ['', f'-{platform.processor()}']:
                 ubsan_lib_name = f'clang_rt.ubsan_minimal{ubsan_lib_arch_suffix}'
-                ubsan_lib_so_path = os.path.join(compiler_rt_lib_dir, f'lib{ubsan_lib_name}.so')
+                ubsan_lib_file_name = f'lib{ubsan_lib_name}.so'
+                compiler_rt_lib_dir_as_list = get_clang_library_dir(
+                    self.compiler_choice.get_c_compiler(),
+                    look_for_file=ubsan_lib_file_name)
+                if not compiler_rt_lib_dir_as_list:
+                    continue
+                assert len(compiler_rt_lib_dir_as_list) == 1
+                compiler_rt_lib_dir = compiler_rt_lib_dir_as_list[0]
+                self.add_lib_dir_and_rpath(compiler_rt_lib_dir)
+
+                ubsan_lib_so_path = os.path.join(compiler_rt_lib_dir, ubsan_lib_file_name)
                 ubsan_lib_candidates.append(ubsan_lib_so_path)
                 if os.path.exists(ubsan_lib_so_path):
                     self.ld_flags.append(f'-l{ubsan_lib_name}')

python/yugabyte_db_thirdparty/clang_util.py

@@ -39,26 +39,63 @@ def get_clang_library_dirs(clang_executable_path: str) -> List[str]:
     return library_dirs
 
 
-def get_clang_library_dir(clang_executable_path: str) -> str:
+def get_clang_library_dir(
+        clang_executable_path: str,
+        look_for_file: Optional[str] = None,
+        all_dirs: bool = False) -> List[str]:
+    """
+    Finds and returns the Clang runtime library directory using the provided Clang executable path.
+    For each of the library directories returned by get_clang_library_dirs(), we will look for a
+    a lib/linux or lib/<arch>-unknown-linux-gnu subdirectory. If we find such a subdirectory, we
+    will consider returning it (but only if it contains the given file if specified).
+
+    :param clang_executable_path: The path to the Clang executable.
+    :param look_for_file: An optional file to look for in the candidate directory. If this file does
+                          not exist in the candidate directory, we will continue looking for another
+                          candidate directory.
+    :param all_dirs: to return all possible directories. This set of directories is filtered by the
+                     presence of the look_for_file if specified.
+    :return: the Clang runtime library directory, or an empty list if not found, or all directories
+             if all_dirs is specified.
+    """
     library_dirs = get_clang_library_dirs(clang_executable_path)
     candidate_dirs: List[str] = []
 
     arch = platform.machine()
     arch_specific_subdir_name = f'{arch}-unknown-linux-gnu'
+    subdir_names = ['linux', arch_specific_subdir_name]
+
+    found_dirs: List[str] = []
 
     for library_dir in library_dirs:
-        for subdir_name in ['linux', arch_specific_subdir_name]:
+        for subdir_name in subdir_names:
             candidate_dir = os.path.join(library_dir, 'lib', subdir_name)
-            if os.path.isdir(candidate_dir):
-                return candidate_dir
+            if os.path.isdir(candidate_dir) and (
+                    look_for_file is None or
+                    os.path.exists(os.path.join(candidate_dir, look_for_file))):
+                if all_dirs:
+                    found_dirs.append(candidate_dir)
+                else:
+                    # Return the first directory we found satisfying the condition.
+                    return [candidate_dir]
             candidate_dirs.append(candidate_dir)
 
+    if (all_dirs and found_dirs) or look_for_file is not None:
+        # If we are looking for all directories, return all directories we found. But make sure
+        # we found at least one.
+        #
+        # If we are looking for a specific file, allow returning an empty list if we did not find
+        # a directory with that particular file.
+        return found_dirs
+
     for candidate_dir in candidate_dirs:
         log(f"Considered candidate directory: {candidate_dir}")
     raise ValueError(
         "Could not find the Clang runtime library directory by appending lib/... suffixes to "
         "any of the directories returned by 'clang -print-search-dirs' "
-        f"(clang path: {clang_executable_path}): {library_dirs}")
+        f"(clang path: {clang_executable_path}, subdir names: {subdir_names}, "
+        f"file name that must exist in the directory: {look_for_file}): {library_dirs}"
+    )
 
 
 def get_clang_include_dir(clang_executable_path: str) -> str:

python/yugabyte_db_thirdparty/yb_build_thirdparty_main.py

@@ -72,9 +72,12 @@ def main() -> None:
         lib_tester = get_lib_tester(fs_layout=builder.fs_layout)
         lib_tester.add_allowed_shared_lib_paths(builder.additional_allowed_shared_lib_paths)
         if builder.compiler_choice.is_linux_clang():
-            lib_tester.add_allowed_shared_lib_paths({
-                get_clang_library_dir(builder.compiler_choice.get_c_compiler())
-            })
+            clang_library_dirs: List[str] = get_clang_library_dir(
+                builder.compiler_choice.get_c_compiler(),
+                all_dirs=True
+            )
+            assert len(clang_library_dirs) > 0
+            lib_tester.add_allowed_shared_lib_paths(set(clang_library_dirs))
         lib_tester.configure_for_compiler(builder.compiler_choice)
 
         lib_tester.run()