Library building for Windows compilers

bin/lib/amazon_properties.py

@@ -3,6 +3,7 @@
 import urllib.parse
 from collections import defaultdict
 from typing import Dict, Any
+from lib.library_platform import LibraryPlatform
 
 import requests
 
@@ -20,12 +21,21 @@ def get_specific_library_version_details(libraries, libid, library_version):
 COMPILEROPT_RE = re.compile(r"(\w*)\.(.*)\.(\w*)")
 
 
-def get_properties_compilers_and_libraries(language, logger, filter_binary_support: bool = True):
+def get_properties_compilers_and_libraries(
+    language, logger, platform: LibraryPlatform, filter_binary_support: bool = True
+):
     _compilers: Dict[str, Dict[str, Any]] = defaultdict(lambda: {})
     _libraries: Dict[str, Dict[str, Any]] = defaultdict(lambda: {})
 
     encoded_language = urllib.parse.quote(language)
-    url = f"https://raw.githubusercontent.com/compiler-explorer/compiler-explorer/main/etc/config/{encoded_language}.amazon.properties"
+
+    if platform == LibraryPlatform.Linux:
+        url = f"https://raw.githubusercontent.com/compiler-explorer/compiler-explorer/main/etc/config/{encoded_language}.amazon.properties"
+    elif platform == LibraryPlatform.Windows:
+        url = f"https://raw.githubusercontent.com/compiler-explorer/compiler-explorer/main/etc/config/{encoded_language}.amazonwin.properties"
+    else:
+        raise RuntimeError("Unsupported platform")
+
     request = requests.get(url, timeout=30)
     if not request.ok:
         raise RuntimeError(f"Fetch failure for {url}: {request}")

bin/lib/binary_info.py

@@ -3,10 +3,12 @@
 from collections import defaultdict
 from pathlib import Path
 from typing import Dict, Any, Iterable
+from lib.library_platform import LibraryPlatform
 
 SYMBOLLINE_RE = re.compile(
     r"^\s*(\d*):\s[0-9a-f]*\s*(\d*)\s(\w*)\s*(\w*)\s*(\w*)\s*([\w|\d]*)\s?([\w\.]*)?$", re.MULTILINE
 )
+SYMBOLLINE_NM_RE = re.compile(r"^[0-9a-f ]*\s(\w)\s(.*)\r$", re.MULTILINE)
 SO_STRANGE_SYMLINK = re.compile(r"INPUT \((\S*)\)")
 
 ELF_CLASS_RE = re.compile(r"^\s*Class:\s*(.*)$", re.MULTILINE)
@@ -21,17 +23,22 @@
 sym_grp_ndx = 5
 sym_grp_name = 6
 
+nm_sym_grp_ndx = 0
+nm_sym_grp_name = 1
+
 
 class BinaryInfo:
-    def __init__(self, logger, buildfolder, filepath):
+    def __init__(self, logger, buildfolder: str, filepath: str, platform: LibraryPlatform):
         self.logger = logger
 
         self.buildfolder = Path(buildfolder)
         self.filepath = Path(filepath)
+        self.platform = platform
 
         self.readelf_header_details = ""
         self.readelf_symbols_details = ""
         self.ldd_details = ""
+        self.nm_used = False
 
         self._follow_and_readelf()
         self._read_symbols_from_binary()
@@ -50,15 +57,24 @@ def _follow_and_readelf(self) -> None:
             self.logger.debug("Was symlink -> readelf on %s", self.filepath)
 
         try:
-            self.readelf_header_details = self._debug_check_output(["readelf", "-h", str(self.filepath)])
-            self.readelf_symbols_details = self._debug_check_output(["readelf", "-W", "-s", str(self.filepath)])
-            if ".so" in self.filepath.name:
-                # pylint: disable=W0702
-                try:
-                    self.ldd_details = self._debug_check_output(["ldd", str(self.filepath)])
-                except:
-                    # some C++ SO's are stubborn and ldd can't read them for some reason, readelf -d sort of gives us the same info
-                    self.ldd_details = self._debug_check_output(["readelf", "-d", str(self.filepath)])
+            if self.platform == LibraryPlatform.Linux:
+                self.readelf_header_details = self._debug_check_output(["readelf", "-h", str(self.filepath)])
+                self.readelf_symbols_details = self._debug_check_output(["readelf", "-W", "-s", str(self.filepath)])
+                if ".so" in self.filepath.name:
+                    # pylint: disable=W0702
+                    try:
+                        self.ldd_details = self._debug_check_output(["ldd", str(self.filepath)])
+                    except:
+                        # some C++ SO's are stubborn and ldd can't read them for some reason, readelf -d sort of gives us the same info
+                        self.ldd_details = self._debug_check_output(["readelf", "-d", str(self.filepath)])
+            elif self.platform == LibraryPlatform.Windows:
+                if str(self.filepath).endswith(".a"):
+                    self.readelf_symbols_details = self._debug_check_output(["nm", str(self.filepath)])
+                    self.nm_used = True
+                else:
+                    self.readelf_symbols_details = self._debug_check_output(["nm", str(self.filepath)])
+                    self.nm_used = True
+
         except subprocess.CalledProcessError:
             try:
                 match = SO_STRANGE_SYMLINK.match(Path(self.filepath).read_text(encoding="utf-8"))
@@ -72,14 +88,24 @@ def _read_symbols_from_binary(self) -> None:
         self.required_symbols = set()
         self.implemented_symbols = set()
 
-        symbollinematches = SYMBOLLINE_RE.findall(self.readelf_symbols_details)
-        if symbollinematches:
-            for line in symbollinematches:
-                if len(line) == 7 and line[sym_grp_name]:
-                    if line[sym_grp_ndx] == "UND":
-                        self.required_symbols.add(line[sym_grp_name])
-                    else:
-                        self.implemented_symbols.add(line[sym_grp_name])
+        if self.nm_used:
+            symbollinematches = SYMBOLLINE_NM_RE.findall(self.readelf_symbols_details)
+            if symbollinematches:
+                for line in symbollinematches:
+                    if line[nm_sym_grp_name]:
+                        if line[nm_sym_grp_ndx] == "U":
+                            self.required_symbols.add(line[nm_sym_grp_name])
+                        else:
+                            self.implemented_symbols.add(line[nm_sym_grp_name])
+        else:
+            symbollinematches = SYMBOLLINE_RE.findall(self.readelf_symbols_details)
+            if symbollinematches:
+                for line in symbollinematches:
+                    if len(line) == 7 and line[sym_grp_name]:
+                        if line[sym_grp_ndx] == "UND":
+                            self.required_symbols.add(line[sym_grp_name])
+                        else:
+                            self.implemented_symbols.add(line[sym_grp_name])
 
     @staticmethod
     def symbol_maybe_cxx11abi(symbol: str) -> bool:

bin/lib/ce_install.py

@@ -17,6 +17,7 @@
 import yaml
 
 from lib.amazon_properties import get_properties_compilers_and_libraries
+from lib.library_platform import LibraryPlatform
 from lib.config_safe_loader import ConfigSafeLoader
 from lib.installation import installers_for
 from lib.installation_context import InstallationContext
@@ -304,7 +305,7 @@ def amazon_check():
 
     for language in languages:
         _LOGGER.info("Checking %s libraries", language)
-        [_, libraries] = get_properties_compilers_and_libraries(language, _LOGGER)
+        [_, libraries] = get_properties_compilers_and_libraries(language, _LOGGER, LibraryPlatform.Linux)
 
         for libraryid in libraries:
             _LOGGER.debug("Checking %s", libraryid)
@@ -465,23 +466,33 @@ def build(context: CliContext, filter_: List[str], force: bool, buildfor: str, p
     num_installed = 0
     num_skipped = 0
     num_failed = 0
+
+    platform = LibraryPlatform.Linux
+
+    if "windows" in context.enabled:
+        platform = LibraryPlatform.Windows
+
     for installable in context.get_installables(filter_):
         if buildfor:
-            print(f"Building {installable.name} just for {buildfor}")
+            print(f"Building {installable.name} ({platform.value}) just for {buildfor}")
         else:
-            print(f"Building {installable.name} for all")
+            print(f"Building {installable.name} ({platform.value}) for all")
 
         if force or installable.should_build():
             if not installable.is_installed():
                 _LOGGER.info("%s is not installed, unable to build", installable.name)
                 num_skipped += 1
             else:
                 try:
-                    [num_installed, num_skipped, num_failed] = installable.build(buildfor, popular_compilers_only)
+                    [num_installed, num_skipped, num_failed] = installable.build(
+                        buildfor, popular_compilers_only, platform
+                    )
                     if num_installed > 0:
                         _LOGGER.info("%s built OK", installable.name)
                     elif num_failed:
                         _LOGGER.info("%s failed to build", installable.name)
+                    else:
+                        _LOGGER.info("%s hit a BUG", installable.name)
                 except RuntimeError as e:
                     if buildfor:
                         raise e

bin/lib/fortran_library_builder.py

@@ -20,6 +20,7 @@
 
 from lib.amazon import get_ssm_param
 from lib.amazon_properties import get_specific_library_version_details, get_properties_compilers_and_libraries
+from lib.library_platform import LibraryPlatform
 from lib.library_build_config import LibraryBuildConfig
 from lib.staging import StagingDir
 
@@ -97,7 +98,9 @@ def __init__(
         if self.language in _propsandlibs:
             [self.compilerprops, self.libraryprops] = _propsandlibs[self.language]
         else:
-            [self.compilerprops, self.libraryprops] = get_properties_compilers_and_libraries(self.language, self.logger)
+            [self.compilerprops, self.libraryprops] = get_properties_compilers_and_libraries(
+                self.language, self.logger, LibraryPlatform.Linux
+            )
             _propsandlibs[self.language] = [self.compilerprops, self.libraryprops]
 
         self.check_compiler_popularity = popular_compilers_only

bin/lib/installable/installable.py

@@ -10,6 +10,7 @@
 from typing import Optional, Callable, Dict, List, Any, Union
 from lib.nightly_versions import NightlyVersions
 
+from lib.library_platform import LibraryPlatform
 from lib.installation_context import InstallationContext
 from lib.library_build_config import LibraryBuildConfig
 from lib.library_builder import LibraryBuilder
@@ -38,7 +39,7 @@ def __init__(self, install_context: InstallationContext, config: Dict[str, Any])
         self.context = self.config_get("context", [])
         self.name = f'{"/".join(self.context)} {self.target_name}'
         self.is_library = False
-        self.language = False
+        self.language = ""
         if len(self.context) > 0:
             self.is_library = self.context[0] == "libraries"
         if len(self.context) > 1:
@@ -231,7 +232,7 @@ def sort_key(self):
     def nightly_like(self) -> bool:
         return self.install_always or self.target_name in ["nightly", "trunk", "master", "main"]
 
-    def build(self, buildfor, popular_compilers_only):
+    def build(self, buildfor: str, popular_compilers_only: bool, platform: LibraryPlatform):
         if not self.is_library:
             raise RuntimeError("Nothing to build")
 
@@ -240,7 +241,7 @@ def build(self, buildfor, popular_compilers_only):
 
         if self.build_config.build_type in ["cmake", "make"]:
             sourcefolder = os.path.join(self.install_context.destination, self.install_path)
-            builder = LibraryBuilder(
+            cppbuilder = LibraryBuilder(
                 _LOGGER,
                 self.language,
                 self.context[-1],
@@ -249,14 +250,15 @@ def build(self, buildfor, popular_compilers_only):
                 self.install_context,
                 self.build_config,
                 popular_compilers_only,
+                platform,
             )
             if self.build_config.build_type == "cmake":
-                return builder.makebuild(buildfor)
+                return cppbuilder.makebuild(buildfor)
             elif self.build_config.build_type == "make":
-                return builder.makebuild(buildfor)
+                return cppbuilder.makebuild(buildfor)
         elif self.build_config.build_type == "fpm":
             sourcefolder = os.path.join(self.install_context.destination, self.install_path)
-            builder = FortranLibraryBuilder(
+            fbuilder = FortranLibraryBuilder(
                 _LOGGER,
                 self.language,
                 self.context[-1],
@@ -266,12 +268,12 @@ def build(self, buildfor, popular_compilers_only):
                 self.build_config,
                 popular_compilers_only,
             )
-            return builder.makebuild(buildfor)
+            return fbuilder.makebuild(buildfor)
         elif self.build_config.build_type == "cargo":
-            builder = RustLibraryBuilder(
+            rbuilder = RustLibraryBuilder(
                 _LOGGER, self.language, self.context[-1], self.target_name, self.install_context, self.build_config
             )
-            return builder.makebuild(buildfor)
+            return rbuilder.makebuild(buildfor)
         else:
             raise RuntimeError("Unsupported build_type")