diff --git a/Makefile.am b/Makefile.am index bf14f8d49708c..35c753966912d 100644 --- a/Makefile.am +++ b/Makefile.am @@ -39,12 +39,9 @@ OSX_TEMP_ISO = $(OSX_DMG:.dmg=).temp.iso OSX_BACKGROUND_SVG=background.svg OSX_BACKGROUND_IMAGE=background.tiff OSX_BACKGROUND_IMAGE_DPIS=36 72 -OSX_DSSTORE_GEN=$(top_srcdir)/contrib/macdeploy/custom_dsstore.py OSX_DEPLOY_SCRIPT=$(top_srcdir)/contrib/macdeploy/macdeployqtplus -OSX_FANCY_PLIST=$(top_srcdir)/contrib/macdeploy/fancy.plist OSX_INSTALLER_ICONS=$(top_srcdir)/src/qt/res/icons/dash.icns OSX_PLIST=$(top_builddir)/share/qt/Info.plist #not installed -OSX_QT_TRANSLATIONS = ar,bg,ca,cs,da,de,es,fa,fi,fr,gd,gl,he,hu,it,ja,ko,lt,lv,pl,pt,ru,sk,sl,sv,uk,zh_CN,zh_TW DIST_CONTRIB = \ $(top_srcdir)/contrib/debian/copyright \ @@ -57,16 +54,15 @@ DIST_SHARE = \ BIN_CHECKS=$(top_srcdir)/contrib/devtools/symbol-check.py \ $(top_srcdir)/contrib/devtools/security-check.py \ - $(top_srcdir)/contrib/devtools/pixie.py + $(top_srcdir)/contrib/devtools/utils.py WINDOWS_PACKAGING = $(top_srcdir)/share/pixmaps/dash.ico \ $(top_srcdir)/share/pixmaps/nsis-header.bmp \ $(top_srcdir)/share/pixmaps/nsis-wizard.bmp \ $(top_srcdir)/doc/README_windows.txt -OSX_PACKAGING = $(OSX_DEPLOY_SCRIPT) $(OSX_FANCY_PLIST) $(OSX_INSTALLER_ICONS) \ +OSX_PACKAGING = $(OSX_DEPLOY_SCRIPT) $(OSX_INSTALLER_ICONS) \ $(top_srcdir)/contrib/macdeploy/$(OSX_BACKGROUND_SVG) \ - $(OSX_DSSTORE_GEN) \ $(top_srcdir)/contrib/macdeploy/detached-sig-apply.sh \ $(top_srcdir)/contrib/macdeploy/detached-sig-create.sh @@ -127,7 +123,7 @@ osx_volname: if BUILD_DARWIN $(OSX_DMG): $(OSX_APP_BUILT) $(OSX_PACKAGING) $(OSX_BACKGROUND_IMAGE) - $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) -add-qt-tr $(OSX_QT_TRANSLATIONS) -translations-dir=$(QT_TRANSLATION_DIR) -dmg -fancy $(OSX_FANCY_PLIST) -verbose 2 -volname $(OSX_VOLNAME) + $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) $(OSX_VOLNAME) -translations-dir=$(QT_TRANSLATION_DIR) -dmg $(OSX_BACKGROUND_IMAGE).png: contrib/macdeploy/$(OSX_BACKGROUND_SVG) sed 's/PACKAGE_NAME/$(PACKAGE_NAME)/' < "$<" | $(RSVG_CONVERT) -f png -d 36 -p 36 -o $@ @@ -160,11 +156,8 @@ $(APP_DIST_DIR)/.background/$(OSX_BACKGROUND_IMAGE): $(OSX_BACKGROUND_IMAGE_DPIF $(MKDIR_P) $(@D) $(TIFFCP) -c none $(OSX_BACKGROUND_IMAGE_DPIFILES) $@ -$(APP_DIST_DIR)/.DS_Store: $(OSX_DSSTORE_GEN) - $(PYTHON) $< "$@" "$(OSX_VOLNAME)" - $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Dash-Qt: $(OSX_APP_BUILT) $(OSX_PACKAGING) - INSTALLNAMETOOL=$(INSTALLNAMETOOL) OTOOL=$(OTOOL) STRIP=$(STRIP) $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) -translations-dir=$(QT_TRANSLATION_DIR) -add-qt-tr $(OSX_QT_TRANSLATIONS) -verbose 2 + INSTALLNAMETOOL=$(INSTALLNAMETOOL) OTOOL=$(OTOOL) STRIP=$(STRIP) $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) $(OSX_VOLNAME) -translations-dir=$(QT_TRANSLATION_DIR) deploydir: $(APP_DIST_EXTRAS) endif !BUILD_DARWIN @@ -342,8 +335,14 @@ clean-local: clean-docs test-security-check: if TARGET_DARWIN - $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_MACHO + $(AM_V_at) CC='$(CC)' CFLAGS='$(CFLAGS)' CPPFLAGS='$(CPPFLAGS)' LDFLAGS='$(LDFLAGS)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_MACHO + $(AM_V_at) CC='$(CC)' CFLAGS='$(CFLAGS)' CPPFLAGS='$(CPPFLAGS)' LDFLAGS='$(LDFLAGS)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_MACHO +endif +if TARGET_WINDOWS + $(AM_V_at) CC='$(CC)' CFLAGS='$(CFLAGS)' CPPFLAGS='$(CPPFLAGS)' LDFLAGS='$(LDFLAGS)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_PE + $(AM_V_at) CC='$(CC)' CFLAGS='$(CFLAGS)' CPPFLAGS='$(CPPFLAGS)' LDFLAGS='$(LDFLAGS)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_PE endif if TARGET_LINUX - $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_ELF + $(AM_V_at) CC='$(CC)' CFLAGS='$(CFLAGS)' CPPFLAGS='$(CPPFLAGS)' LDFLAGS='$(LDFLAGS)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_ELF + $(AM_V_at) CC='$(CC)' CFLAGS='$(CFLAGS)' CPPFLAGS='$(CPPFLAGS)' LDFLAGS='$(LDFLAGS)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_ELF endif diff --git a/ci/dash/build_src.sh b/ci/dash/build_src.sh index 7dc5a3a03d0a9..a0bd392ea2235 100755 --- a/ci/dash/build_src.sh +++ b/ci/dash/build_src.sh @@ -61,6 +61,10 @@ if [ -n "$USE_VALGRIND" ]; then ${BASE_ROOT_DIR}/ci/test/wrap-valgrind.sh fi +if [ "$RUN_SECURITY_TESTS" = "true" ]; then + make test-security-check +fi + if [ "$RUN_SYMBOL_TESTS" = "true" ]; then make $MAKEJOBS -C src check-symbols fi diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index e7b3dc43c745c..9c0c33c8e0126 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -38,6 +38,7 @@ export HOST=${HOST:-$("$BASE_ROOT_DIR/depends/config.guess")} export USE_BUSY_BOX=${USE_BUSY_BOX:-false} export RUN_UNIT_TESTS=${RUN_UNIT_TESTS:-true} export RUN_INTEGRATION_TESTS=${RUN_INTEGRATION_TESTS:-true} +export RUN_SECURITY_TESTS=${RUN_SECURITY_TESTS:-false} export RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-false} export RUN_SYMBOL_TESTS=${RUN_SYMBOL_TESTS:-true} export CONTAINER_NAME=${CONTAINER_NAME:-ci_unnamed} diff --git a/ci/test/00_setup_env_mac_host.sh b/ci/test/00_setup_env_mac_host.sh index dea7925e851a7..a5459ea0a95e2 100755 --- a/ci/test/00_setup_env_mac_host.sh +++ b/ci/test/00_setup_env_mac_host.sh @@ -8,9 +8,10 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_macos export HOST=x86_64-apple-darwin19 -export PIP_PACKAGES="zmq" +export PIP_PACKAGES="zmq lief" export RUN_UNIT_TESTS=true export RUN_INTEGRATION_TESTS=false +export RUN_SECURITY_TESTS="true" export GOAL="install" export BITCOIN_CONFIG="--enable-gui --enable-reduce-exports --disable-miner --enable-werror" # Run without depends diff --git a/ci/test/00_setup_env_win64.sh b/ci/test/00_setup_env_win64.sh index b7cc19acff2b6..b19876bece7d0 100755 --- a/ci/test/00_setup_env_win64.sh +++ b/ci/test/00_setup_env_win64.sh @@ -11,6 +11,7 @@ export HOST=x86_64-w64-mingw32 export PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine64" export DPKG_ADD_ARCH="i386" export RUN_INTEGRATION_TESTS=false +export RUN_SECURITY_TESTS="false" export GOAL="deploy" export BITCOIN_CONFIG="--enable-gui --enable-reduce-exports --disable-miner" export DIRECT_WINE_EXEC_TESTS=true diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index 92ef17c302f4e..68c48dfae47d2 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -68,6 +68,9 @@ if [[ $DOCKER_NAME_TAG == centos* ]]; then elif [ "$CI_USE_APT_INSTALL" != "no" ]; then ${CI_RETRY_EXE} DOCKER_EXEC apt-get update ${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -y $PACKAGES $DOCKER_PACKAGES + if [ -n "$PIP_PACKAGES" ]; then + ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES + fi fi if [ "$TRAVIS_OS_NAME" == "osx" ]; then diff --git a/configure.ac b/configure.ac index 5e46881183ab4..fa3b03ae1c24a 100644 --- a/configure.ac +++ b/configure.ac @@ -106,7 +106,6 @@ AC_PATH_PROG([GIT], [git]) AC_PATH_PROG(CCACHE,ccache) AC_PATH_PROG(XGETTEXT,xgettext) AC_PATH_PROG(HEXDUMP,hexdump) -AC_PATH_TOOL(CPPFILT, c++filt) AC_PATH_TOOL(OBJCOPY, objcopy) AC_PATH_TOOL(DSYMUTIL, dsymutil) AC_PATH_PROG(DOXYGEN, doxygen) @@ -942,6 +941,7 @@ if test x$use_hardening != xno; then ]) fi + AX_CHECK_LINK_FLAG([[-Wl,--enable-reloc-section]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--enable-reloc-section"],, [[$LDFLAG_WERROR]]) AX_CHECK_LINK_FLAG([[-Wl,--dynamicbase]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--dynamicbase"]) AX_CHECK_LINK_FLAG([[-Wl,--nxcompat]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--nxcompat"]) AX_CHECK_LINK_FLAG([[-Wl,--high-entropy-va]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--high-entropy-va"]) @@ -1823,6 +1823,8 @@ AC_SUBST(ANDROID_ARCH) AC_CONFIG_FILES([Makefile src/Makefile doc/man/Makefile share/setup.nsi share/qt/Info.plist test/config.ini]) AC_CONFIG_FILES([contrib/devtools/split-debug.sh],[chmod +x contrib/devtools/split-debug.sh]) AM_COND_IF([HAVE_DOXYGEN], [AC_CONFIG_FILES([doc/Doxyfile])]) +AC_CONFIG_LINKS([contrib/devtools/security-check.py:contrib/devtools/security-check.py]) +AC_CONFIG_LINKS([contrib/devtools/test-security-check.py:contrib/devtools/test-security-check.py]) AC_CONFIG_LINKS([contrib/devtools/symbol-check.py:contrib/devtools/symbol-check.py]) AC_CONFIG_LINKS([contrib/devtools/test-symbol-check.py:contrib/devtools/test-symbol-check.py]) AC_CONFIG_LINKS([contrib/filter-lcov.py:contrib/filter-lcov.py]) diff --git a/contrib/containers/ci/Dockerfile b/contrib/containers/ci/Dockerfile index f07a79699323c..ba2a61972107b 100644 --- a/contrib/containers/ci/Dockerfile +++ b/contrib/containers/ci/Dockerfile @@ -37,6 +37,7 @@ RUN pip3 install \ codespell==1.17.1 \ flake8==3.8.3 \ jinja2 \ + lief==0.12.0 \ pyzmq \ vulture==2.3 \ yq \ diff --git a/contrib/devtools/pixie.py b/contrib/devtools/pixie.py deleted file mode 100644 index 64660968ad255..0000000000000 --- a/contrib/devtools/pixie.py +++ /dev/null @@ -1,323 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2020 Wladimir J. van der Laan -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -''' -Compact, self-contained ELF implementation for bitcoin-core security checks. -''' -import struct -import types -from typing import Dict, List, Optional, Union, Tuple - -# you can find all these values in elf.h -EI_NIDENT = 16 - -# Byte indices in e_ident -EI_CLASS = 4 # ELFCLASSxx -EI_DATA = 5 # ELFDATAxxxx - -ELFCLASS32 = 1 # 32-bit -ELFCLASS64 = 2 # 64-bit - -ELFDATA2LSB = 1 # little endian -ELFDATA2MSB = 2 # big endian - -# relevant values for e_machine -EM_386 = 3 -EM_PPC64 = 21 -EM_ARM = 40 -EM_AARCH64 = 183 -EM_X86_64 = 62 -EM_RISCV = 243 - -# relevant values for e_type -ET_DYN = 3 - -# relevant values for sh_type -SHT_PROGBITS = 1 -SHT_STRTAB = 3 -SHT_DYNAMIC = 6 -SHT_DYNSYM = 11 -SHT_GNU_verneed = 0x6ffffffe -SHT_GNU_versym = 0x6fffffff - -# relevant values for p_type -PT_LOAD = 1 -PT_GNU_STACK = 0x6474e551 -PT_GNU_RELRO = 0x6474e552 - -# relevant values for p_flags -PF_X = (1 << 0) -PF_W = (1 << 1) -PF_R = (1 << 2) - -# relevant values for d_tag -DT_NEEDED = 1 -DT_FLAGS = 30 - -# relevant values of `d_un.d_val' in the DT_FLAGS entry -DF_BIND_NOW = 0x00000008 - -# relevant d_tags with string payload -STRING_TAGS = {DT_NEEDED} - -# rrlevant values for ST_BIND subfield of st_info (symbol binding) -STB_LOCAL = 0 - -class ELFRecord(types.SimpleNamespace): - '''Unified parsing for ELF records.''' - def __init__(self, data: bytes, offset: int, eh: 'ELFHeader', total_size: Optional[int]) -> None: - hdr_struct = self.STRUCT[eh.ei_class][0][eh.ei_data] - if total_size is not None and hdr_struct.size > total_size: - raise ValueError(f'{self.__class__.__name__} header size too small ({total_size} < {hdr_struct.size})') - for field, value in zip(self.STRUCT[eh.ei_class][1], hdr_struct.unpack(data[offset:offset + hdr_struct.size])): - setattr(self, field, value) - -def BiStruct(chars: str) -> Dict[int, struct.Struct]: - '''Compile a struct parser for both endians.''' - return { - ELFDATA2LSB: struct.Struct('<' + chars), - ELFDATA2MSB: struct.Struct('>' + chars), - } - -class ELFHeader(ELFRecord): - FIELDS = ['e_type', 'e_machine', 'e_version', 'e_entry', 'e_phoff', 'e_shoff', 'e_flags', 'e_ehsize', 'e_phentsize', 'e_phnum', 'e_shentsize', 'e_shnum', 'e_shstrndx'] - STRUCT = { - ELFCLASS32: (BiStruct('HHIIIIIHHHHHH'), FIELDS), - ELFCLASS64: (BiStruct('HHIQQQIHHHHHH'), FIELDS), - } - - def __init__(self, data: bytes, offset: int) -> None: - self.e_ident = data[offset:offset + EI_NIDENT] - if self.e_ident[0:4] != b'\x7fELF': - raise ValueError('invalid ELF magic') - self.ei_class = self.e_ident[EI_CLASS] - self.ei_data = self.e_ident[EI_DATA] - - super().__init__(data, offset + EI_NIDENT, self, None) - - def __repr__(self) -> str: - return f'Header(e_ident={self.e_ident!r}, e_type={self.e_type}, e_machine={self.e_machine}, e_version={self.e_version}, e_entry={self.e_entry}, e_phoff={self.e_phoff}, e_shoff={self.e_shoff}, e_flags={self.e_flags}, e_ehsize={self.e_ehsize}, e_phentsize={self.e_phentsize}, e_phnum={self.e_phnum}, e_shentsize={self.e_shentsize}, e_shnum={self.e_shnum}, e_shstrndx={self.e_shstrndx})' - -class Section(ELFRecord): - name: Optional[bytes] = None - FIELDS = ['sh_name', 'sh_type', 'sh_flags', 'sh_addr', 'sh_offset', 'sh_size', 'sh_link', 'sh_info', 'sh_addralign', 'sh_entsize'] - STRUCT = { - ELFCLASS32: (BiStruct('IIIIIIIIII'), FIELDS), - ELFCLASS64: (BiStruct('IIQQQQIIQQ'), FIELDS), - } - - def __init__(self, data: bytes, offset: int, eh: ELFHeader) -> None: - super().__init__(data, offset, eh, eh.e_shentsize) - self._data = data - - def __repr__(self) -> str: - return f'Section(sh_name={self.sh_name}({self.name!r}), sh_type=0x{self.sh_type:x}, sh_flags={self.sh_flags}, sh_addr=0x{self.sh_addr:x}, sh_offset=0x{self.sh_offset:x}, sh_size={self.sh_size}, sh_link={self.sh_link}, sh_info={self.sh_info}, sh_addralign={self.sh_addralign}, sh_entsize={self.sh_entsize})' - - def contents(self) -> bytes: - '''Return section contents.''' - return self._data[self.sh_offset:self.sh_offset + self.sh_size] - -class ProgramHeader(ELFRecord): - STRUCT = { - # different ELF classes have the same fields, but in a different order to optimize space versus alignment - ELFCLASS32: (BiStruct('IIIIIIII'), ['p_type', 'p_offset', 'p_vaddr', 'p_paddr', 'p_filesz', 'p_memsz', 'p_flags', 'p_align']), - ELFCLASS64: (BiStruct('IIQQQQQQ'), ['p_type', 'p_flags', 'p_offset', 'p_vaddr', 'p_paddr', 'p_filesz', 'p_memsz', 'p_align']), - } - - def __init__(self, data: bytes, offset: int, eh: ELFHeader) -> None: - super().__init__(data, offset, eh, eh.e_phentsize) - - def __repr__(self) -> str: - return f'ProgramHeader(p_type={self.p_type}, p_offset={self.p_offset}, p_vaddr={self.p_vaddr}, p_paddr={self.p_paddr}, p_filesz={self.p_filesz}, p_memsz={self.p_memsz}, p_flags={self.p_flags}, p_align={self.p_align})' - -class Symbol(ELFRecord): - STRUCT = { - # different ELF classes have the same fields, but in a different order to optimize space versus alignment - ELFCLASS32: (BiStruct('IIIBBH'), ['st_name', 'st_value', 'st_size', 'st_info', 'st_other', 'st_shndx']), - ELFCLASS64: (BiStruct('IBBHQQ'), ['st_name', 'st_info', 'st_other', 'st_shndx', 'st_value', 'st_size']), - } - - def __init__(self, data: bytes, offset: int, eh: ELFHeader, symtab: Section, strings: bytes, version: Optional[bytes]) -> None: - super().__init__(data, offset, eh, symtab.sh_entsize) - self.name = _lookup_string(strings, self.st_name) - self.version = version - - def __repr__(self) -> str: - return f'Symbol(st_name={self.st_name}({self.name!r}), st_value={self.st_value}, st_size={self.st_size}, st_info={self.st_info}, st_other={self.st_other}, st_shndx={self.st_shndx}, version={self.version!r})' - - @property - def is_import(self) -> bool: - '''Returns whether the symbol is an imported symbol.''' - return self.st_bind != STB_LOCAL and self.st_shndx == 0 - - @property - def is_export(self) -> bool: - '''Returns whether the symbol is an exported symbol.''' - return self.st_bind != STB_LOCAL and self.st_shndx != 0 - - @property - def st_bind(self) -> int: - '''Returns STB_*.''' - return self.st_info >> 4 - -class Verneed(ELFRecord): - DEF = (BiStruct('HHIII'), ['vn_version', 'vn_cnt', 'vn_file', 'vn_aux', 'vn_next']) - STRUCT = { ELFCLASS32: DEF, ELFCLASS64: DEF } - - def __init__(self, data: bytes, offset: int, eh: ELFHeader) -> None: - super().__init__(data, offset, eh, None) - - def __repr__(self) -> str: - return f'Verneed(vn_version={self.vn_version}, vn_cnt={self.vn_cnt}, vn_file={self.vn_file}, vn_aux={self.vn_aux}, vn_next={self.vn_next})' - -class Vernaux(ELFRecord): - DEF = (BiStruct('IHHII'), ['vna_hash', 'vna_flags', 'vna_other', 'vna_name', 'vna_next']) - STRUCT = { ELFCLASS32: DEF, ELFCLASS64: DEF } - - def __init__(self, data: bytes, offset: int, eh: ELFHeader, strings: bytes) -> None: - super().__init__(data, offset, eh, None) - self.name = _lookup_string(strings, self.vna_name) - - def __repr__(self) -> str: - return f'Veraux(vna_hash={self.vna_hash}, vna_flags={self.vna_flags}, vna_other={self.vna_other}, vna_name={self.vna_name}({self.name!r}), vna_next={self.vna_next})' - -class DynTag(ELFRecord): - STRUCT = { - ELFCLASS32: (BiStruct('II'), ['d_tag', 'd_val']), - ELFCLASS64: (BiStruct('QQ'), ['d_tag', 'd_val']), - } - - def __init__(self, data: bytes, offset: int, eh: ELFHeader, section: Section) -> None: - super().__init__(data, offset, eh, section.sh_entsize) - - def __repr__(self) -> str: - return f'DynTag(d_tag={self.d_tag}, d_val={self.d_val})' - -def _lookup_string(data: bytes, index: int) -> bytes: - '''Look up string by offset in ELF string table.''' - endx = data.find(b'\x00', index) - assert endx != -1 - return data[index:endx] - -VERSYM_S = BiStruct('H') # .gnu_version section has a single 16-bit integer per symbol in the linked section -def _parse_symbol_table(section: Section, strings: bytes, eh: ELFHeader, versym: bytes, verneed: Dict[int, bytes]) -> List[Symbol]: - '''Parse symbol table, return a list of symbols.''' - data = section.contents() - symbols = [] - versym_iter = (verneed.get(v[0]) for v in VERSYM_S[eh.ei_data].iter_unpack(versym)) - for ofs, version in zip(range(0, len(data), section.sh_entsize), versym_iter): - symbols.append(Symbol(data, ofs, eh, section, strings, version)) - return symbols - -def _parse_verneed(section: Section, strings: bytes, eh: ELFHeader) -> Dict[int, bytes]: - '''Parse .gnu.version_r section, return a dictionary of {versym: 'GLIBC_...'}.''' - data = section.contents() - ofs = 0 - result = {} - while True: - verneed = Verneed(data, ofs, eh) - aofs = ofs + verneed.vn_aux - while True: - vernaux = Vernaux(data, aofs, eh, strings) - result[vernaux.vna_other] = vernaux.name - if not vernaux.vna_next: - break - aofs += vernaux.vna_next - - if not verneed.vn_next: - break - ofs += verneed.vn_next - - return result - -def _parse_dyn_tags(section: Section, strings: bytes, eh: ELFHeader) -> List[Tuple[int, Union[bytes, int]]]: - '''Parse dynamic tags. Return array of tuples.''' - data = section.contents() - ofs = 0 - result = [] - for ofs in range(0, len(data), section.sh_entsize): - tag = DynTag(data, ofs, eh, section) - val = _lookup_string(strings, tag.d_val) if tag.d_tag in STRING_TAGS else tag.d_val - result.append((tag.d_tag, val)) - - return result - -class ELFFile: - sections: List[Section] - program_headers: List[ProgramHeader] - dyn_symbols: List[Symbol] - dyn_tags: List[Tuple[int, Union[bytes, int]]] - - def __init__(self, data: bytes) -> None: - self.data = data - self.hdr = ELFHeader(self.data, 0) - self._load_sections() - self._load_program_headers() - self._load_dyn_symbols() - self._load_dyn_tags() - self._section_to_segment_mapping() - - def _load_sections(self) -> None: - self.sections = [] - for idx in range(self.hdr.e_shnum): - offset = self.hdr.e_shoff + idx * self.hdr.e_shentsize - self.sections.append(Section(self.data, offset, self.hdr)) - - shstr = self.sections[self.hdr.e_shstrndx].contents() - for section in self.sections: - section.name = _lookup_string(shstr, section.sh_name) - - def _load_program_headers(self) -> None: - self.program_headers = [] - for idx in range(self.hdr.e_phnum): - offset = self.hdr.e_phoff + idx * self.hdr.e_phentsize - self.program_headers.append(ProgramHeader(self.data, offset, self.hdr)) - - def _load_dyn_symbols(self) -> None: - # first, load 'verneed' section - verneed = None - for section in self.sections: - if section.sh_type == SHT_GNU_verneed: - strtab = self.sections[section.sh_link].contents() # associated string table - assert verneed is None # only one section of this kind please - verneed = _parse_verneed(section, strtab, self.hdr) - assert verneed is not None - - # then, correlate GNU versym sections with dynamic symbol sections - versym = {} - for section in self.sections: - if section.sh_type == SHT_GNU_versym: - versym[section.sh_link] = section - - # finally, load dynsym sections - self.dyn_symbols = [] - for idx, section in enumerate(self.sections): - if section.sh_type == SHT_DYNSYM: # find dynamic symbol tables - strtab_data = self.sections[section.sh_link].contents() # associated string table - versym_data = versym[idx].contents() # associated symbol version table - self.dyn_symbols += _parse_symbol_table(section, strtab_data, self.hdr, versym_data, verneed) - - def _load_dyn_tags(self) -> None: - self.dyn_tags = [] - for idx, section in enumerate(self.sections): - if section.sh_type == SHT_DYNAMIC: # find dynamic tag tables - strtab = self.sections[section.sh_link].contents() # associated string table - self.dyn_tags += _parse_dyn_tags(section, strtab, self.hdr) - - def _section_to_segment_mapping(self) -> None: - for ph in self.program_headers: - ph.sections = [] - for section in self.sections: - if ph.p_vaddr <= section.sh_addr < (ph.p_vaddr + ph.p_memsz): - ph.sections.append(section) - - def query_dyn_tags(self, tag_in: int) -> List[Union[int, bytes]]: - '''Return the values of all dyn tags with the specified tag.''' - return [val for (tag, val) in self.dyn_tags if tag == tag_in] - - -def load(filename: str) -> ELFFile: - with open(filename, 'rb') as f: - data = f.read() - return ELFFile(data) diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py index 7b09c42fdec23..563029ca7c2a3 100755 --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -6,303 +6,258 @@ Perform basic security checks on a series of executables. Exit status will be 0 if successful, and the program will be silent. Otherwise the exit status will be 1 and it will log which executables failed which checks. -Needs `objdump` (for PE) and `otool` (for MACHO). ''' -import subprocess import sys -import os -from typing import List, Optional +from typing import List -import pixie +import lief -OBJDUMP_CMD = os.getenv('OBJDUMP', '/usr/bin/objdump') -OTOOL_CMD = os.getenv('OTOOL', '/usr/bin/otool') - -def run_command(command) -> str: - p = subprocess.run(command, stdout=subprocess.PIPE, check=True, universal_newlines=True) - return p.stdout - -def check_ELF_PIE(executable) -> bool: - ''' - Check for position independent executable (PIE), allowing for address space randomization. - ''' - elf = pixie.load(executable) - return elf.hdr.e_type == pixie.ET_DYN - -def check_ELF_NX(executable) -> bool: - ''' - Check that no sections are writable and executable (including the stack) - ''' - elf = pixie.load(executable) - have_wx = False - have_gnu_stack = False - for ph in elf.program_headers: - if ph.p_type == pixie.PT_GNU_STACK: - have_gnu_stack = True - if (ph.p_flags & pixie.PF_W) != 0 and (ph.p_flags & pixie.PF_X) != 0: # section is both writable and executable - have_wx = True - return have_gnu_stack and not have_wx - -def check_ELF_RELRO(executable) -> bool: +def check_ELF_RELRO(binary) -> bool: ''' Check for read-only relocations. GNU_RELRO program header must exist Dynamic section must have BIND_NOW flag ''' - elf = pixie.load(executable) have_gnu_relro = False - for ph in elf.program_headers: + for segment in binary.segments: # Note: not checking p_flags == PF_R: here as linkers set the permission differently # This does not affect security: the permission flags of the GNU_RELRO program # header are ignored, the PT_LOAD header determines the effective permissions. # However, the dynamic linker need to write to this area so these are RW. # Glibc itself takes care of mprotecting this area R after relocations are finished. # See also https://marc.info/?l=binutils&m=1498883354122353 - if ph.p_type == pixie.PT_GNU_RELRO: + if segment.type == lief.ELF.SEGMENT_TYPES.GNU_RELRO: have_gnu_relro = True have_bindnow = False - for flags in elf.query_dyn_tags(pixie.DT_FLAGS): - assert isinstance(flags, int) - if flags & pixie.DF_BIND_NOW: + try: + flags = binary.get(lief.ELF.DYNAMIC_TAGS.FLAGS) + if flags.value & lief.ELF.DYNAMIC_FLAGS.BIND_NOW: have_bindnow = True + except: + have_bindnow = False return have_gnu_relro and have_bindnow -def check_ELF_Canary(executable) -> bool: +def check_ELF_Canary(binary) -> bool: ''' Check for use of stack canary ''' - elf = pixie.load(executable) - ok = False - for symbol in elf.dyn_symbols: - if symbol.name == b'__stack_chk_fail': - ok = True - return ok + return binary.has_symbol('__stack_chk_fail') -def check_ELF_separate_code(executable): +def check_ELF_separate_code(binary): ''' Check that sections are appropriately separated in virtual memory, based on their permissions. This checks for missing -Wl,-z,separate-code and potentially other problems. ''' - elf = pixie.load(executable) - R = pixie.PF_R - W = pixie.PF_W - E = pixie.PF_X + R = lief.ELF.SEGMENT_FLAGS.R + W = lief.ELF.SEGMENT_FLAGS.W + E = lief.ELF.SEGMENT_FLAGS.X EXPECTED_FLAGS = { # Read + execute - b'.init': R | E, - b'.plt': R | E, - b'.plt.got': R | E, - b'.plt.sec': R | E, - b'.text': R | E, - b'.fini': R | E, + '.init': R | E, + '.plt': R | E, + '.plt.got': R | E, + '.plt.sec': R | E, + '.text': R | E, + '.fini': R | E, # Read-only data - b'.interp': R, - b'.note.gnu.property': R, - b'.note.gnu.build-id': R, - b'.note.ABI-tag': R, - b'.gnu.hash': R, - b'.dynsym': R, - b'.dynstr': R, - b'.gnu.version': R, - b'.gnu.version_r': R, - b'.rela.dyn': R, - b'.rela.plt': R, - b'.rodata': R, - b'.eh_frame_hdr': R, - b'.eh_frame': R, - b'.qtmetadata': R, - b'.gcc_except_table': R, - b'.stapsdt.base': R, + '.interp': R, + '.note.gnu.property': R, + '.note.gnu.build-id': R, + '.note.ABI-tag': R, + '.gnu.hash': R, + '.dynsym': R, + '.dynstr': R, + '.gnu.version': R, + '.gnu.version_r': R, + '.rela.dyn': R, + '.rela.plt': R, + '.rodata': R, + '.eh_frame_hdr': R, + '.eh_frame': R, + '.qtmetadata': R, + '.gcc_except_table': R, + '.stapsdt.base': R, # Writable data - b'.init_array': R | W, - b'.fini_array': R | W, - b'.dynamic': R | W, - b'.got': R | W, - b'.data': R | W, - b'.bss': R | W, + '.init_array': R | W, + '.fini_array': R | W, + '.dynamic': R | W, + '.got': R | W, + '.data': R | W, + '.bss': R | W, } - if elf.hdr.e_machine == pixie.EM_PPC64: + if binary.header.machine_type == lief.ELF.ARCH.PPC64: # .plt is RW on ppc64 even with separate-code - EXPECTED_FLAGS[b'.plt'] = R | W + EXPECTED_FLAGS['.plt'] = R | W # For all LOAD program headers get mapping to the list of sections, # and for each section, remember the flags of the associated program header. flags_per_section = {} - for ph in elf.program_headers: - if ph.p_type == pixie.PT_LOAD: - for section in ph.sections: - assert(section.name not in flags_per_section) - flags_per_section[section.name] = ph.p_flags + for segment in binary.segments: + if segment.type == lief.ELF.SEGMENT_TYPES.LOAD: + for section in segment.sections: + flags_per_section[section.name] = segment.flags # Spot-check ELF LOAD program header flags per section # If these sections exist, check them against the expected R/W/E flags for (section, flags) in flags_per_section.items(): if section in EXPECTED_FLAGS: - if EXPECTED_FLAGS[section] != flags: + if int(EXPECTED_FLAGS[section]) != int(flags): return False return True -def get_PE_dll_characteristics(executable) -> int: - '''Get PE DllCharacteristics bits''' - stdout = run_command([OBJDUMP_CMD, '-x', executable]) - - bits = 0 - for line in stdout.splitlines(): - tokens = line.split() - if len(tokens)>=2 and tokens[0] == 'DllCharacteristics': - bits = int(tokens[1],16) - return bits +def check_ELF_control_flow(binary) -> bool: + ''' + Check for control flow instrumentation + ''' + main = binary.get_function_address('main') + content = binary.get_content_from_virtual_address(main, 4, lief.Binary.VA_TYPES.AUTO) -IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA = 0x0020 -IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE = 0x0040 -IMAGE_DLL_CHARACTERISTICS_NX_COMPAT = 0x0100 + if content == [243, 15, 30, 250]: # endbr64 + return True + return False -def check_PE_DYNAMIC_BASE(executable) -> bool: +def check_PE_DYNAMIC_BASE(binary) -> bool: '''PIE: DllCharacteristics bit 0x40 signifies dynamicbase (ASLR)''' - bits = get_PE_dll_characteristics(executable) - return (bits & IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE) == IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE + return lief.PE.DLL_CHARACTERISTICS.DYNAMIC_BASE in binary.optional_header.dll_characteristics_lists # Must support high-entropy 64-bit address space layout randomization # in addition to DYNAMIC_BASE to have secure ASLR. -def check_PE_HIGH_ENTROPY_VA(executable) -> bool: +def check_PE_HIGH_ENTROPY_VA(binary) -> bool: '''PIE: DllCharacteristics bit 0x20 signifies high-entropy ASLR''' - bits = get_PE_dll_characteristics(executable) - return (bits & IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA) == IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA + return lief.PE.DLL_CHARACTERISTICS.HIGH_ENTROPY_VA in binary.optional_header.dll_characteristics_lists -def check_PE_RELOC_SECTION(executable) -> bool: +def check_PE_RELOC_SECTION(binary) -> bool: '''Check for a reloc section. This is required for functional ASLR.''' - stdout = run_command([OBJDUMP_CMD, '-h', executable]) + return binary.has_relocations - for line in stdout.splitlines(): - if '.reloc' in line: - return True - return False - -def check_PE_NX(executable) -> bool: - '''NX: DllCharacteristics bit 0x100 signifies nxcompat (DEP)''' - bits = get_PE_dll_characteristics(executable) - return (bits & IMAGE_DLL_CHARACTERISTICS_NX_COMPAT) == IMAGE_DLL_CHARACTERISTICS_NX_COMPAT +def check_PE_control_flow(binary) -> bool: + ''' + Check for control flow instrumentation + ''' + main = binary.get_symbol('main').value -def get_MACHO_executable_flags(executable) -> List[str]: - stdout = run_command([OTOOL_CMD, '-vh', executable]) + section_addr = binary.section_from_rva(main).virtual_address + virtual_address = binary.optional_header.imagebase + section_addr + main - flags: List[str] = [] - for line in stdout.splitlines(): - tokens = line.split() - # filter first two header lines - if 'magic' in tokens or 'Mach' in tokens: - continue - # filter ncmds and sizeofcmds values - flags += [t for t in tokens if not t.isdigit()] - return flags + content = binary.get_content_from_virtual_address(virtual_address, 4, lief.Binary.VA_TYPES.VA) -def check_MACHO_PIE(executable) -> bool: - ''' - Check for position independent executable (PIE), allowing for address space randomization. - ''' - flags = get_MACHO_executable_flags(executable) - if 'PIE' in flags: + if content == [243, 15, 30, 250]: # endbr64 return True return False -def check_MACHO_NOUNDEFS(executable) -> bool: +def check_MACHO_NOUNDEFS(binary) -> bool: ''' Check for no undefined references. ''' - flags = get_MACHO_executable_flags(executable) - if 'NOUNDEFS' in flags: - return True - return False + return binary.header.has(lief.MachO.HEADER_FLAGS.NOUNDEFS) -def check_MACHO_NX(executable) -> bool: +def check_MACHO_LAZY_BINDINGS(binary) -> bool: ''' - Check for no stack execution + Check for no lazy bindings. + We don't use or check for MH_BINDATLOAD. See #18295. ''' - flags = get_MACHO_executable_flags(executable) - if 'ALLOW_STACK_EXECUTION' in flags: - return False - return True + return binary.dyld_info.lazy_bind == (0,0) -def check_MACHO_LAZY_BINDINGS(executable) -> bool: +def check_MACHO_Canary(binary) -> bool: ''' - Check for no lazy bindings. - We don't use or check for MH_BINDATLOAD. See #18295. + Check for use of stack canary ''' - stdout = run_command([OTOOL_CMD, '-l', executable]) + return binary.has_symbol('___stack_chk_fail') - for line in stdout.splitlines(): - tokens = line.split() - if 'lazy_bind_off' in tokens or 'lazy_bind_size' in tokens: - if tokens[1] != '0': - return False - return True +def check_PIE(binary) -> bool: + ''' + Check for position independent executable (PIE), + allowing for address space randomization. + ''' + return binary.is_pie -def check_MACHO_Canary(executable) -> bool: +def check_NX(binary) -> bool: ''' - Check for use of stack canary + Check for no stack execution ''' - stdout = run_command([OTOOL_CMD, '-Iv', executable]) + return binary.has_nx - ok = False - for line in stdout.splitlines(): - if '___stack_chk_fail' in line: - ok = True - return ok +def check_MACHO_control_flow(binary) -> bool: + ''' + Check for control flow instrumentation + ''' + content = binary.get_content_from_virtual_address(binary.entrypoint, 4, lief.Binary.VA_TYPES.AUTO) -CHECKS = { -'ELF': [ - ('PIE', check_ELF_PIE), - ('NX', check_ELF_NX), + if content == [243, 15, 30, 250]: # endbr64 + return True + return False + +BASE_ELF = [ + ('PIE', check_PIE), + ('NX', check_NX), ('RELRO', check_ELF_RELRO), ('Canary', check_ELF_Canary), ('separate_code', check_ELF_separate_code), -], -'PE': [ +] + +BASE_PE = [ + ('PIE', check_PIE), ('DYNAMIC_BASE', check_PE_DYNAMIC_BASE), ('HIGH_ENTROPY_VA', check_PE_HIGH_ENTROPY_VA), - ('NX', check_PE_NX), - ('RELOC_SECTION', check_PE_RELOC_SECTION) -], -'MACHO': [ - ('PIE', check_MACHO_PIE), + ('NX', check_NX), + ('RELOC_SECTION', check_PE_RELOC_SECTION), + ('CONTROL_FLOW', check_PE_control_flow), +] + +BASE_MACHO = [ + ('PIE', check_PIE), ('NOUNDEFS', check_MACHO_NOUNDEFS), - ('NX', check_MACHO_NX), + ('NX', check_NX), ('LAZY_BINDINGS', check_MACHO_LAZY_BINDINGS), - ('Canary', check_MACHO_Canary) + ('Canary', check_MACHO_Canary), + ('CONTROL_FLOW', check_MACHO_control_flow), ] -} -def identify_executable(executable) -> Optional[str]: - with open(filename, 'rb') as f: - magic = f.read(4) - if magic.startswith(b'MZ'): - return 'PE' - elif magic.startswith(b'\x7fELF'): - return 'ELF' - elif magic.startswith(b'\xcf\xfa'): - return 'MACHO' - return None +CHECKS = { + lief.EXE_FORMATS.ELF: { + lief.ARCHITECTURES.X86: BASE_ELF + [('CONTROL_FLOW', check_ELF_control_flow)], + lief.ARCHITECTURES.ARM: BASE_ELF, + lief.ARCHITECTURES.ARM64: BASE_ELF, + lief.ARCHITECTURES.PPC: BASE_ELF, + lief.ARCHITECTURES.RISCV: BASE_ELF, + }, + lief.EXE_FORMATS.PE: { + lief.ARCHITECTURES.X86: BASE_PE, + }, + lief.EXE_FORMATS.MACHO: { + lief.ARCHITECTURES.X86: BASE_MACHO, + } +} if __name__ == '__main__': - retval = 0 + retval: int = 0 for filename in sys.argv[1:]: try: - etype = identify_executable(filename) - if etype is None: - print('%s: unknown format' % filename) + binary = lief.parse(filename) + etype = binary.format + arch = binary.abstract.header.architecture + binary.concrete + + if etype == lief.EXE_FORMATS.UNKNOWN: + print(f'{filename}: unknown executable format') + retval = 1 + continue + + if arch == lief.ARCHITECTURES.NONE: + print(f'{filename}: unknown architecture') retval = 1 continue - failed = [] - for (name, func) in CHECKS[etype]: - if not func(filename): + failed: List[str] = [] + for (name, func) in CHECKS[etype][arch]: + if not func(binary): failed.append(name) if failed: - print('%s: failed %s' % (filename, ' '.join(failed))) + print(f'{filename}: failed {" ".join(failed)}') retval = 1 except IOError: - print('%s: cannot open' % filename) + print(f'{filename}: cannot open') retval = 1 sys.exit(retval) diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index 136c122901f18..d1a7cfffb538c 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -10,12 +10,10 @@ find ../path/to/binaries -type f -executable | xargs python3 contrib/devtools/symbol-check.py ''' -import subprocess import sys -import os -from typing import List, Optional +from typing import Dict -import pixie +import lief # Debian 9 (Stretch) EOL: 2022. https://wiki.debian.org/DebianReleases#Production_Releases # @@ -43,29 +41,45 @@ MAX_VERSIONS = { 'GCC': (4,8,0), 'GLIBC': { - pixie.EM_386: (2,28), - pixie.EM_X86_64: (2,18), - pixie.EM_ARM: (2,28), - pixie.EM_AARCH64:(2,18), - pixie.EM_PPC64: (2,18), - pixie.EM_RISCV: (2,27), + lief.ELF.ARCH.x86_64: (2,18), + lief.ELF.ARCH.ARM: (2,28), + lief.ELF.ARCH.AARCH64:(2,18), + lief.ELF.ARCH.PPC64: (2,18), + lief.ELF.ARCH.RISCV: (2,27), }, 'LIBATOMIC': (1,0), 'V': (0,5,0), # xkb (bitcoin-qt only) } -# See here for a description of _IO_stdin_used: -# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=634261#109 # Ignore symbols that are exported as part of every executable IGNORE_EXPORTS = { -'_edata', '_end', '__end__', '_init', '__bss_start', '__bss_start__', '_bss_end__', '__bss_end__', '_fini', '_IO_stdin_used', 'stdin', 'stdout', 'stderr', +'_edata', '_end', '__end__', '_init', '__bss_start', '__bss_start__', '_bss_end__', +'__bss_end__', '_fini', '_IO_stdin_used', 'stdin', 'stdout', 'stderr', 'environ', '_environ', '__environ', # Used in stacktraces.cpp '__cxa_demangle' } -CPPFILT_CMD = os.getenv('CPPFILT', '/usr/bin/c++filt') -OBJDUMP_CMD = os.getenv('OBJDUMP', '/usr/bin/objdump') -OTOOL_CMD = os.getenv('OTOOL', '/usr/bin/otool') + +# Expected linker-loader names can be found here: +# https://sourceware.org/glibc/wiki/ABIList?action=recall&rev=16 +ELF_INTERPRETER_NAMES: Dict[lief.ELF.ARCH, Dict[lief.ENDIANNESS, str]] = { + lief.ELF.ARCH.x86_64: { + lief.ENDIANNESS.LITTLE: "/lib64/ld-linux-x86-64.so.2", + }, + lief.ELF.ARCH.ARM: { + lief.ENDIANNESS.LITTLE: "/lib/ld-linux-armhf.so.3", + }, + lief.ELF.ARCH.AARCH64: { + lief.ENDIANNESS.LITTLE: "/lib/ld-linux-aarch64.so.1", + }, + lief.ELF.ARCH.PPC64: { + lief.ENDIANNESS.BIG: "/lib64/ld64.so.1", + lief.ENDIANNESS.LITTLE: "/lib64/ld64.so.2", + }, + lief.ELF.ARCH.RISCV: { + lief.ENDIANNESS.LITTLE: "/lib/ld-linux-riscv64-lp64d.so.1", + }, +} # Allowed NEEDED libraries ELF_ALLOWED_LIBRARIES = { @@ -141,31 +155,8 @@ 'WTSAPI32.dll', } -class CPPFilt(object): - ''' - Demangle C++ symbol names. - - Use a pipe to the 'c++filt' command. - ''' - def __init__(self): - self.proc = subprocess.Popen(CPPFILT_CMD, stdin=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) - - def __call__(self, mangled): - self.proc.stdin.write(mangled + '\n') - self.proc.stdin.flush() - return self.proc.stdout.readline().rstrip() - - def close(self): - self.proc.stdin.close() - self.proc.stdout.close() - self.proc.wait() - def check_version(max_versions, version, arch) -> bool: - if '_' in version: - (lib, _, ver) = version.rpartition('_') - else: - lib = version - ver = '0' + (lib, _, ver) = version.rpartition('_') ver = tuple([int(x) for x in ver.split('.')]) if not lib in max_versions: return False @@ -174,129 +165,119 @@ def check_version(max_versions, version, arch) -> bool: else: return ver <= max_versions[lib][arch] -def check_imported_symbols(filename) -> bool: - elf = pixie.load(filename) - cppfilt = CPPFilt() +def check_imported_symbols(binary) -> bool: ok = True - for symbol in elf.dyn_symbols: - if not symbol.is_import: + for symbol in binary.imported_symbols: + if not symbol.imported: continue - sym = symbol.name.decode() - version = symbol.version.decode() if symbol.version is not None else None - if version and not check_version(MAX_VERSIONS, version, elf.hdr.e_machine): - print('{}: symbol {} from unsupported version {}'.format(filename, cppfilt(sym), version)) - ok = False + + version = symbol.symbol_version if symbol.has_version else None + + if version: + aux_version = version.symbol_version_auxiliary.name if version.has_auxiliary_version else None + if aux_version and not check_version(MAX_VERSIONS, aux_version, binary.header.machine_type): + print(f'{filename}: symbol {symbol.name} from unsupported version {version}') + ok = False return ok -def check_exported_symbols(filename) -> bool: - elf = pixie.load(filename) - cppfilt = CPPFilt() +def check_exported_symbols(binary) -> bool: ok = True - for symbol in elf.dyn_symbols: - if not symbol.is_export: + + for symbol in binary.dynamic_symbols: + if not symbol.exported: continue - sym = symbol.name.decode() - if elf.hdr.e_machine == pixie.EM_RISCV or sym in IGNORE_EXPORTS: + name = symbol.name + if binary.header.machine_type == lief.ELF.ARCH.RISCV or name in IGNORE_EXPORTS: continue - print('{}: export of symbol {} not allowed'.format(filename, cppfilt(sym))) + print(f'{binary.name}: export of symbol {name} not allowed!') ok = False return ok -def check_ELF_libraries(filename) -> bool: +def check_ELF_libraries(binary) -> bool: ok = True - elf = pixie.load(filename) - for library_name in elf.query_dyn_tags(pixie.DT_NEEDED): - assert(isinstance(library_name, bytes)) - if library_name.decode() not in ELF_ALLOWED_LIBRARIES: - print('{}: NEEDED library {} is not allowed'.format(filename, library_name.decode())) + for library in binary.libraries: + if library not in ELF_ALLOWED_LIBRARIES: + print(f'{filename}: {library} is not in ALLOWED_LIBRARIES!') ok = False return ok -def macho_read_libraries(filename) -> List[str]: - p = subprocess.Popen([OTOOL_CMD, '-L', filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) - (stdout, stderr) = p.communicate() - if p.returncode: - raise IOError('Error opening file') - libraries = [] - for line in stdout.splitlines(): - tokens = line.split() - if len(tokens) == 1: # skip executable name - continue - libraries.append(tokens[0].split('/')[-1]) - return libraries - -def check_MACHO_libraries(filename) -> bool: +def check_MACHO_libraries(binary) -> bool: ok = True - for dylib in macho_read_libraries(filename): - if dylib not in MACHO_ALLOWED_LIBRARIES: - print('{} is not in ALLOWED_LIBRARIES!'.format(dylib)) + for dylib in binary.libraries: + split = dylib.name.split('/') + if split[-1] not in MACHO_ALLOWED_LIBRARIES: + print(f'{split[-1]} is not in ALLOWED_LIBRARIES!') ok = False return ok -def pe_read_libraries(filename) -> List[str]: - p = subprocess.Popen([OBJDUMP_CMD, '-x', filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) - (stdout, stderr) = p.communicate() - if p.returncode: - raise IOError('Error opening file') - libraries = [] - for line in stdout.splitlines(): - if 'DLL Name:' in line: - tokens = line.split(': ') - libraries.append(tokens[1]) - return libraries +def check_MACHO_min_os(binary) -> bool: + if binary.build_version.minos == [10,15,0]: + return True + return False + +def check_MACHO_sdk(binary) -> bool: + if binary.build_version.sdk == [10, 15, 6]: + return True + return False -def check_PE_libraries(filename) -> bool: +def check_PE_libraries(binary) -> bool: ok = True - for dylib in pe_read_libraries(filename): + for dylib in binary.libraries: if dylib not in PE_ALLOWED_LIBRARIES: - print('{} is not in ALLOWED_LIBRARIES!'.format(dylib)) + print(f'{dylib} is not in ALLOWED_LIBRARIES!') ok = False return ok +def check_PE_subsystem_version(binary) -> bool: + major: int = binary.optional_header.major_subsystem_version + minor: int = binary.optional_header.minor_subsystem_version + if major == 6 and minor == 1: + return True + return False + +def check_ELF_interpreter(binary) -> bool: + expected_interpreter = ELF_INTERPRETER_NAMES[binary.header.machine_type][binary.abstract.header.endianness] + + return binary.concrete.interpreter == expected_interpreter + CHECKS = { -'ELF': [ +lief.EXE_FORMATS.ELF: [ ('IMPORTED_SYMBOLS', check_imported_symbols), ('EXPORTED_SYMBOLS', check_exported_symbols), - ('LIBRARY_DEPENDENCIES', check_ELF_libraries) + ('LIBRARY_DEPENDENCIES', check_ELF_libraries), + ('INTERPRETER_NAME', check_ELF_interpreter), ], -'MACHO': [ - ('DYNAMIC_LIBRARIES', check_MACHO_libraries) +lief.EXE_FORMATS.MACHO: [ + ('DYNAMIC_LIBRARIES', check_MACHO_libraries), + ('MIN_OS', check_MACHO_min_os), + ('SDK', check_MACHO_sdk), ], -'PE' : [ - ('DYNAMIC_LIBRARIES', check_PE_libraries) +lief.EXE_FORMATS.PE: [ + ('DYNAMIC_LIBRARIES', check_PE_libraries), + ('SUBSYSTEM_VERSION', check_PE_subsystem_version), ] } -def identify_executable(executable) -> Optional[str]: - with open(filename, 'rb') as f: - magic = f.read(4) - if magic.startswith(b'MZ'): - return 'PE' - elif magic.startswith(b'\x7fELF'): - return 'ELF' - elif magic.startswith(b'\xcf\xfa'): - return 'MACHO' - return None - if __name__ == '__main__': retval = 0 for filename in sys.argv[1:]: try: - etype = identify_executable(filename) - if etype is None: - print('{}: unknown format'.format(filename)) + binary = lief.parse(filename) + etype = binary.format + if etype == lief.EXE_FORMATS.UNKNOWN: + print(f'{filename}: unknown executable format') retval = 1 continue failed = [] for (name, func) in CHECKS[etype]: - if not func(filename): + if not func(binary): failed.append(name) if failed: - print('{}: failed {}'.format(filename, ' '.join(failed))) + print(f'{filename}: failed {" ".join(failed)}') retval = 1 except IOError: - print('{}: cannot open'.format(filename)) + print(f'{filename}: cannot open') retval = 1 sys.exit(retval) diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py index 19b7b0572bc7f..50fb382fef38d 100755 --- a/contrib/devtools/test-security-check.py +++ b/contrib/devtools/test-security-check.py @@ -5,9 +5,14 @@ ''' Test script for security-check.py ''' +import lief #type:ignore +import os import subprocess +from typing import List import unittest +from utils import determine_wellknown_cmd + def write_testcode(filename): with open(filename, 'w', encoding="utf8") as f: f.write(''' @@ -19,68 +24,115 @@ def write_testcode(filename): } ''') +def clean_files(source, executable): + os.remove(source) + os.remove(executable) + def call_security_check(cc, source, executable, options): - subprocess.check_call([cc,source,'-o',executable] + options) - p = subprocess.Popen(['./security-check.py',executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) - (stdout, stderr) = p.communicate() - return (p.returncode, stdout.rstrip()) + # This should behave the same as AC_TRY_LINK, so arrange well-known flags + # in the same order as autoconf would. + # + # See the definitions for ac_link in autoconf's lib/autoconf/c.m4 file for + # reference. + env_flags: List[str] = [] + for var in ['CFLAGS', 'CPPFLAGS', 'LDFLAGS']: + env_flags += filter(None, os.environ.get(var, '').split(' ')) + + subprocess.run([*cc,source,'-o',executable] + env_flags + options, check=True) + p = subprocess.run(['./contrib/devtools/security-check.py',executable], stdout=subprocess.PIPE, universal_newlines=True) + return (p.returncode, p.stdout.rstrip()) + +def get_arch(cc, source, executable): + subprocess.run([*cc, source, '-o', executable], check=True) + binary = lief.parse(executable) + arch = binary.abstract.header.architecture + os.remove(executable) + return arch class TestSecurityChecks(unittest.TestCase): def test_ELF(self): source = 'test1.c' executable = 'test1' - cc = 'gcc' + cc = determine_wellknown_cmd('CC', 'gcc') write_testcode(source) + arch = get_arch(cc, source, executable) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-zexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), - (1, executable+': failed PIE NX RELRO Canary')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), - (1, executable+': failed PIE RELRO Canary')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), - (1, executable+': failed PIE RELRO')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-pie','-fPIE', '-Wl,-z,separate-code']), - (1, executable+': failed RELRO')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,noseparate-code']), - (1, executable+': failed separate_code')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,separate-code']), - (0, '')) + if arch == lief.ARCHITECTURES.X86: + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-zexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), + (1, executable+': failed PIE NX RELRO Canary CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), + (1, executable+': failed PIE RELRO Canary CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), + (1, executable+': failed PIE RELRO CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-pie','-fPIE', '-Wl,-z,separate-code']), + (1, executable+': failed RELRO CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,noseparate-code']), + (1, executable+': failed separate_code CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,separate-code']), + (1, executable+': failed CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,separate-code', '-fcf-protection=full']), + (0, '')) + else: + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-zexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), + (1, executable+': failed PIE NX RELRO Canary')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), + (1, executable+': failed PIE RELRO Canary')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), + (1, executable+': failed PIE RELRO')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-pie','-fPIE', '-Wl,-z,separate-code']), + (1, executable+': failed RELRO')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,noseparate-code']), + (1, executable+': failed separate_code')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,separate-code']), + (0, '')) + + clean_files(source, executable) def test_PE(self): source = 'test1.c' executable = 'test1.exe' - cc = 'x86_64-w64-mingw32-gcc' + cc = determine_wellknown_cmd('CC', 'x86_64-w64-mingw32-gcc') write_testcode(source) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--no-nxcompat','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), - (1, executable+': failed DYNAMIC_BASE HIGH_ENTROPY_VA NX RELOC_SECTION')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), - (1, executable+': failed DYNAMIC_BASE HIGH_ENTROPY_VA RELOC_SECTION')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), - (1, executable+': failed HIGH_ENTROPY_VA RELOC_SECTION')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--dynamicbase','-Wl,--high-entropy-va','-no-pie','-fno-PIE']), - (1, executable+': failed RELOC_SECTION')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--no-nxcompat','-Wl,--disable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA NX RELOC_SECTION CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--disable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA RELOC_SECTION CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-pie','-fPIE']), + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA CONTROL_FLOW')) # -pie -fPIE does nothing unless --dynamicbase is also supplied + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--no-high-entropy-va','-pie','-fPIE']), + (1, executable+': failed HIGH_ENTROPY_VA CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE']), + (1, executable+': failed CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE', '-fcf-protection=full']), (0, '')) + clean_files(source, executable) + def test_MACHO(self): source = 'test1.c' executable = 'test1' - cc = 'clang' + cc = determine_wellknown_cmd('CC', 'clang') write_testcode(source) self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fno-stack-protector']), - (1, executable+': failed PIE NOUNDEFS NX LAZY_BINDINGS Canary')) + (1, executable+': failed PIE NOUNDEFS NX LAZY_BINDINGS Canary CONTROL_FLOW')) self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fstack-protector-all']), - (1, executable+': failed PIE NOUNDEFS NX LAZY_BINDINGS')) + (1, executable+': failed PIE NOUNDEFS NX LAZY_BINDINGS CONTROL_FLOW')) self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-fstack-protector-all']), - (1, executable+': failed PIE NOUNDEFS LAZY_BINDINGS')) + (1, executable+': failed PIE NOUNDEFS LAZY_BINDINGS CONTROL_FLOW')) self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-fstack-protector-all']), - (1, executable+': failed PIE LAZY_BINDINGS')) + (1, executable+': failed PIE LAZY_BINDINGS CONTROL_FLOW')) self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all']), + (1, executable+': failed PIE CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full']), (1, executable+': failed PIE')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-pie','-Wl,-bind_at_load','-fstack-protector-all']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full']), (0, '')) + clean_files(source, executable) + if __name__ == '__main__': unittest.main() - diff --git a/contrib/devtools/test-symbol-check.py b/contrib/devtools/test-symbol-check.py index 11ac0297508db..48729ccdb66e6 100755 --- a/contrib/devtools/test-symbol-check.py +++ b/contrib/devtools/test-symbol-check.py @@ -5,47 +5,67 @@ ''' Test script for symbol-check.py ''' +import os import subprocess +from typing import List import unittest -def call_symbol_check(cc, source, executable, options): - subprocess.run([cc,source,'-o',executable] + options, check=True) +from utils import determine_wellknown_cmd + +def call_symbol_check(cc: List[str], source, executable, options): + # This should behave the same as AC_TRY_LINK, so arrange well-known flags + # in the same order as autoconf would. + # + # See the definitions for ac_link in autoconf's lib/autoconf/c.m4 file for + # reference. + env_flags: List[str] = [] + for var in ['CFLAGS', 'CPPFLAGS', 'LDFLAGS']: + env_flags += filter(None, os.environ.get(var, '').split(' ')) + + subprocess.run([*cc,source,'-o',executable] + env_flags + options, check=True) p = subprocess.run(['./contrib/devtools/symbol-check.py',executable], stdout=subprocess.PIPE, universal_newlines=True) + os.remove(source) + os.remove(executable) return (p.returncode, p.stdout.rstrip()) -def get_machine(cc): - p = subprocess.run([cc,'-dumpmachine'], stdout=subprocess.PIPE, universal_newlines=True) +def get_machine(cc: List[str]): + p = subprocess.run([*cc,'-dumpmachine'], stdout=subprocess.PIPE, universal_newlines=True) return p.stdout.rstrip() class TestSymbolChecks(unittest.TestCase): def test_ELF(self): source = 'test1.c' executable = 'test1' - cc = 'gcc' + cc = determine_wellknown_cmd('CC', 'gcc') - # there's no way to do this test for RISC-V at the moment; bionic's libc is 2.27 - # and we allow all symbols from 2.27. + # there's no way to do this test for RISC-V at the moment; we build for + # RISC-V in a glibc 2.31 envinonment and we allow all symbols from 2.27. if 'riscv' in get_machine(cc): self.skipTest("test not available for RISC-V") - # memfd_create was introduced in GLIBC 2.27, so is newer than the upper limit of - # all but RISC-V but still available on bionic + # there's no way to do this test for ARM at the moment; we build for + # ARM in a glibc 2.31 envinonment and we allow all symbols from 2.28. + if 'arm' in get_machine(cc): + self.skipTest("test not available for 32-bit ARM") + + # nextup was introduced in GLIBC 2.24, so is newer than our supported + # glibc (2.18), and available in our release build environment (2.31). with open(source, 'w', encoding="utf8") as f: f.write(''' #define _GNU_SOURCE - #include + #include - int memfd_create(const char *name, unsigned int flags); + double nextup(double x); int main() { - memfd_create("test", 0); + nextup(3.14); return 0; } ''') - self.assertEqual(call_symbol_check(cc, source, executable, []), - (1, executable + ': symbol memfd_create from unsupported version GLIBC_2.27\n' + + self.assertEqual(call_symbol_check(cc, source, executable, ['-lm']), + (1, executable + ': symbol nextup from unsupported version GLIBC_2.24(3)\n' + executable + ': failed IMPORTED_SYMBOLS')) # -lutil is part of the libc6 package so a safe bet that it's installed @@ -64,7 +84,7 @@ def test_ELF(self): ''') self.assertEqual(call_symbol_check(cc, source, executable, ['-lutil']), - (1, executable + ': NEEDED library libutil.so.1 is not allowed\n' + + (1, executable + ': libutil.so.1 is not in ALLOWED_LIBRARIES!\n' + executable + ': failed LIBRARY_DEPENDENCIES')) # finally, check a simple conforming binary @@ -87,7 +107,7 @@ def test_ELF(self): def test_MACHO(self): source = 'test1.c' executable = 'test1' - cc = 'clang' + cc = determine_wellknown_cmd('CC', 'clang') with open(source, 'w', encoding="utf8") as f: f.write(''' @@ -101,9 +121,9 @@ def test_MACHO(self): ''') - self.assertEqual(call_symbol_check(cc, source, executable, ['-lexpat']), + self.assertEqual(call_symbol_check(cc, source, executable, ['-lexpat', '-Wl,-platform_version','-Wl,macos', '-Wl,11.4', '-Wl,11.4']), (1, 'libexpat.1.dylib is not in ALLOWED_LIBRARIES!\n' + - executable + ': failed DYNAMIC_LIBRARIES')) + f'{executable}: failed DYNAMIC_LIBRARIES MIN_OS SDK')) source = 'test2.c' executable = 'test2' @@ -118,9 +138,72 @@ def test_MACHO(self): } ''') - self.assertEqual(call_symbol_check(cc, source, executable, ['-framework', 'CoreGraphics']), + self.assertEqual(call_symbol_check(cc, source, executable, ['-framework', 'CoreGraphics', '-Wl,-platform_version','-Wl,macos', '-Wl,11.4', '-Wl,11.4']), + (1, f'{executable}: failed MIN_OS SDK')) + + source = 'test3.c' + executable = 'test3' + with open(source, 'w', encoding="utf8") as f: + f.write(''' + int main() + { + return 0; + } + ''') + + self.assertEqual(call_symbol_check(cc, source, executable, ['-Wl,-platform_version','-Wl,macos', '-Wl,10.15', '-Wl,11.4']), + (1, f'{executable}: failed SDK')) + + def test_PE(self): + source = 'test1.c' + executable = 'test1.exe' + cc = determine_wellknown_cmd('CC', 'x86_64-w64-mingw32-gcc') + + with open(source, 'w', encoding="utf8") as f: + f.write(''' + #include + + int main() + { + PdhConnectMachineA(NULL); + return 0; + } + ''') + + self.assertEqual(call_symbol_check(cc, source, executable, ['-lpdh', '-Wl,--major-subsystem-version', '-Wl,6', '-Wl,--minor-subsystem-version', '-Wl,1']), + (1, 'pdh.dll is not in ALLOWED_LIBRARIES!\n' + + executable + ': failed DYNAMIC_LIBRARIES')) + + source = 'test2.c' + executable = 'test2.exe' + + with open(source, 'w', encoding="utf8") as f: + f.write(''' + int main() + { + return 0; + } + ''') + + self.assertEqual(call_symbol_check(cc, source, executable, ['-Wl,--major-subsystem-version', '-Wl,9', '-Wl,--minor-subsystem-version', '-Wl,9']), + (1, executable + ': failed SUBSYSTEM_VERSION')) + + source = 'test3.c' + executable = 'test3.exe' + with open(source, 'w', encoding="utf8") as f: + f.write(''' + #include + + int main() + { + CoFreeUnusedLibrariesEx(0,0); + return 0; + } + ''') + + self.assertEqual(call_symbol_check(cc, source, executable, ['-lole32', '-Wl,--major-subsystem-version', '-Wl,6', '-Wl,--minor-subsystem-version', '-Wl,1']), (0, '')) + if __name__ == '__main__': unittest.main() - diff --git a/contrib/devtools/utils.py b/contrib/devtools/utils.py new file mode 100755 index 0000000000000..68ad1c3aba191 --- /dev/null +++ b/contrib/devtools/utils.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +''' +Common utility functions +''' +import shutil +import sys +import os +from typing import List + + +def determine_wellknown_cmd(envvar, progname) -> List[str]: + maybe_env = os.getenv(envvar) + maybe_which = shutil.which(progname) + if maybe_env: + return maybe_env.split(' ') # Well-known vars are often meant to be word-split + elif maybe_which: + return [ maybe_which ] + else: + sys.exit(f"{progname} not found") diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index 4268a71bd73b7..4172c8265bcf1 100755 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -23,6 +23,7 @@ packages: - "patch" - "pkg-config" - "python3" +- "python3-pip" - "libxkbcommon0" - "ccache" # Cross compilation HOSTS: @@ -109,6 +110,8 @@ script: | done } + pip3 install lief==0.12.0 + # Faketime for depends so intermediate results are comparable export PATH_orig=${PATH} create_global_faketime_wrappers "2000-01-01 12:00:00" diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index d130d8e550bcd..472a5e4ce44dd 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -26,6 +26,7 @@ packages: - "python3" - "python3-dev" - "python3-setuptools" +- "python3-pip" - "fonts-tuffy" - "ccache" - "cmake" @@ -98,6 +99,8 @@ script: | done } + pip3 install lief==0.12.0 + # Faketime for depends so intermediate results are comparable export PATH_orig=${PATH} create_global_faketime_wrappers "2000-01-01 12:00:00" diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index 99edd80771164..ab41e5cdef1d2 100755 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -22,6 +22,7 @@ packages: - "zip" - "ca-certificates" - "python3" +- "python3-pip" - "ccache" remotes: - "url": "https://github.com/dashpay/dash.git" @@ -114,6 +115,8 @@ script: | done } + pip3 install lief==0.12.0 + # Faketime for depends so intermediate results are comparable export PATH_orig=${PATH} create_global_faketime_wrappers "2000-01-01 12:00:00" diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index ce80a71a01b0e..bd80fd8cbf687 100644 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -168,8 +168,8 @@ case "$HOST" in arm-linux-gnueabihf) echo /lib/ld-linux-armhf.so.3 ;; aarch64-linux-gnu) echo /lib/ld-linux-aarch64.so.1 ;; riscv64-linux-gnu) echo /lib/ld-linux-riscv64-lp64d.so.1 ;; - powerpc64-linux-gnu) echo /lib/ld64.so.1;; - powerpc64le-linux-gnu) echo /lib/ld64.so.2;; + powerpc64-linux-gnu) echo /lib64/ld64.so.1;; + powerpc64le-linux-gnu) echo /lib64/ld64.so.2;; *) exit 1 ;; esac ) @@ -295,10 +295,11 @@ mkdir -p "$DISTSRC" # Build Dash Core make --jobs="$JOBS" ${V:+V=1} - # Perform basic ELF security checks on a series of executables. + # Check that symbol/security checks tools are sane. + make test-security-check ${V:+V=1} + # Perform basic security checks on a series of executables. make -C src --jobs=1 check-security ${V:+V=1} - # Check that executables only contain allowed gcc, glibc and libstdc++ - # version symbols for Linux distro back-compatibility. + # Check that executables only contain allowed version symbols. make -C src --jobs=1 check-symbols ${V:+V=1} mkdir -p "$OUTDIR" diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index aac5d508003a3..5845348067700 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -85,6 +85,10 @@ http://www.linuxfromscratch.org/hlfs/view/development/chapter05/gcc-pass1.html" (define (explicit-cross-configure package) (package-with-extra-configure-variable package "--build" building-on)) +(define (make-binutils-with-mingw-w64-disable-flags xbinutils) + (package-with-extra-patches xbinutils + (search-our-patches "binutils-mingw-w64-disable-flags.patch"))) + (define (make-cross-toolchain target base-gcc-for-libc base-kernel-headers @@ -173,7 +177,7 @@ desirable for building Dash Core release binaries." (define (make-mingw-pthreads-cross-toolchain target) "Create a cross-compilation toolchain package for TARGET" - (let* ((xbinutils (cross-binutils target)) + (let* ((xbinutils (make-binutils-with-mingw-w64-disable-flags (cross-binutils target))) (pthreads-xlibc mingw-w64-x86_64-winpthreads) (pthreads-xgcc (make-gcc-with-pthreads (cross-gcc target @@ -580,6 +584,36 @@ inspecting signatures in Mach-O binaries.") (package-with-extra-patches glibc-2.27 (search-our-patches "glibc-2.27-riscv64-Use-__has_include__-to-include-asm-syscalls.h.patch"))) +(define-public lief + (package + (name "python-lief") + (version "0.12.0") + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/lief-project/LIEF.git") + (commit version))) + (file-name (git-file-name name version)) + (sha256 + (base32 + "026jchj56q25v6gc0754dj9cj5hz5zaza8ij93y5ga94w20kzm9q")))) + (build-system python-build-system) + (arguments + `(#:phases + (modify-phases %standard-phases + (add-after 'unpack 'parallel-jobs + ;; build with multiple cores + (lambda _ + (substitute* "setup.py" (("self.parallel if self.parallel else 1") (number->string (parallel-job-count))))))))) + (native-inputs + `(("cmake" ,cmake))) + (home-page "https://github.com/lief-project/LIEF") + (synopsis "Library to Instrument Executable Formats") + (description "Python library to to provide a cross platform library which can +parse, modify and abstract ELF, PE and MachO formats.") + (license license:asl2.0))) + (packages->manifest (append (list ;; The Basics @@ -616,6 +650,8 @@ inspecting signatures in Mach-O binaries.") python-3 ;; Git git + ;; Tests + lief ;; Native gcc 7 toolchain gcc-toolchain-7 (list gcc-toolchain-7 "static")) diff --git a/contrib/guix/patches/binutils-mingw-w64-disable-flags.patch b/contrib/guix/patches/binutils-mingw-w64-disable-flags.patch new file mode 100644 index 0000000000000..8f88eb9dfd592 --- /dev/null +++ b/contrib/guix/patches/binutils-mingw-w64-disable-flags.patch @@ -0,0 +1,171 @@ +Description: Add disable opposites to the security-related flags +Author: Stephen Kitt + +This patch adds "no-" variants to disable the various security flags: +"no-dynamicbase", "no-nxcompat", "no-high-entropy-va", "disable-reloc-section". + +--- a/ld/emultempl/pe.em ++++ b/ld/emultempl/pe.em +@@ -259,9 +261,11 @@ + (OPTION_ENABLE_LONG_SECTION_NAMES + 1) + /* DLLCharacteristics flags. */ + #define OPTION_DYNAMIC_BASE (OPTION_DISABLE_LONG_SECTION_NAMES + 1) +-#define OPTION_FORCE_INTEGRITY (OPTION_DYNAMIC_BASE + 1) ++#define OPTION_NO_DYNAMIC_BASE (OPTION_DYNAMIC_BASE + 1) ++#define OPTION_FORCE_INTEGRITY (OPTION_NO_DYNAMIC_BASE + 1) + #define OPTION_NX_COMPAT (OPTION_FORCE_INTEGRITY + 1) +-#define OPTION_NO_ISOLATION (OPTION_NX_COMPAT + 1) ++#define OPTION_NO_NX_COMPAT (OPTION_NX_COMPAT + 1) ++#define OPTION_NO_ISOLATION (OPTION_NO_NX_COMPAT + 1) + #define OPTION_NO_SEH (OPTION_NO_ISOLATION + 1) + #define OPTION_NO_BIND (OPTION_NO_SEH + 1) + #define OPTION_WDM_DRIVER (OPTION_NO_BIND + 1) +@@ -271,6 +275,7 @@ + #define OPTION_NO_INSERT_TIMESTAMP (OPTION_INSERT_TIMESTAMP + 1) + #define OPTION_BUILD_ID (OPTION_NO_INSERT_TIMESTAMP + 1) + #define OPTION_ENABLE_RELOC_SECTION (OPTION_BUILD_ID + 1) ++#define OPTION_DISABLE_RELOC_SECTION (OPTION_ENABLE_RELOC_SECTION + 1) + + static void + gld${EMULATION_NAME}_add_options +@@ -342,8 +347,10 @@ + {"enable-long-section-names", no_argument, NULL, OPTION_ENABLE_LONG_SECTION_NAMES}, + {"disable-long-section-names", no_argument, NULL, OPTION_DISABLE_LONG_SECTION_NAMES}, + {"dynamicbase",no_argument, NULL, OPTION_DYNAMIC_BASE}, ++ {"no-dynamicbase", no_argument, NULL, OPTION_NO_DYNAMIC_BASE}, + {"forceinteg", no_argument, NULL, OPTION_FORCE_INTEGRITY}, + {"nxcompat", no_argument, NULL, OPTION_NX_COMPAT}, ++ {"no-nxcompat", no_argument, NULL, OPTION_NO_NX_COMPAT}, + {"no-isolation", no_argument, NULL, OPTION_NO_ISOLATION}, + {"no-seh", no_argument, NULL, OPTION_NO_SEH}, + {"no-bind", no_argument, NULL, OPTION_NO_BIND}, +@@ -351,6 +358,7 @@ + {"tsaware", no_argument, NULL, OPTION_TERMINAL_SERVER_AWARE}, + {"build-id", optional_argument, NULL, OPTION_BUILD_ID}, + {"enable-reloc-section", no_argument, NULL, OPTION_ENABLE_RELOC_SECTION}, ++ {"disable-reloc-section", no_argument, NULL, OPTION_DISABLE_RELOC_SECTION}, + {NULL, no_argument, NULL, 0} + }; + +@@ -485,9 +494,12 @@ + in object files\n")); + fprintf (file, _(" --dynamicbase Image base address may be relocated using\n\ + address space layout randomization (ASLR)\n")); ++ fprintf (file, _(" --no-dynamicbase Image base address may not be relocated\n")); + fprintf (file, _(" --enable-reloc-section Create the base relocation table\n")); ++ fprintf (file, _(" --disable-reloc-section Disable the base relocation table\n")); + fprintf (file, _(" --forceinteg Code integrity checks are enforced\n")); + fprintf (file, _(" --nxcompat Image is compatible with data execution prevention\n")); ++ fprintf (file, _(" --no-nxcompat Image is not compatible with data execution prevention\n")); + fprintf (file, _(" --no-isolation Image understands isolation but do not isolate the image\n")); + fprintf (file, _(" --no-seh Image does not use SEH. No SE handler may\n\ + be called in this image\n")); +@@ -862,12 +874,21 @@ + case OPTION_ENABLE_RELOC_SECTION: + pe_dll_enable_reloc_section = 1; + break; ++ case OPTION_DISABLE_RELOC_SECTION: ++ pe_dll_enable_reloc_section = 0; ++ /* fall through */ ++ case OPTION_NO_DYNAMIC_BASE: ++ pe_dll_characteristics &= ~IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE; ++ break; + case OPTION_FORCE_INTEGRITY: + pe_dll_characteristics |= IMAGE_DLL_CHARACTERISTICS_FORCE_INTEGRITY; + break; + case OPTION_NX_COMPAT: + pe_dll_characteristics |= IMAGE_DLL_CHARACTERISTICS_NX_COMPAT; + break; ++ case OPTION_NO_NX_COMPAT: ++ pe_dll_characteristics &= ~IMAGE_DLL_CHARACTERISTICS_NX_COMPAT; ++ break; + case OPTION_NO_ISOLATION: + pe_dll_characteristics |= IMAGE_DLLCHARACTERISTICS_NO_ISOLATION; + break; +--- a/ld/emultempl/pep.em ++++ b/ld/emultempl/pep.em +@@ -237,9 +240,12 @@ + OPTION_ENABLE_LONG_SECTION_NAMES, + OPTION_DISABLE_LONG_SECTION_NAMES, + OPTION_HIGH_ENTROPY_VA, ++ OPTION_NO_HIGH_ENTROPY_VA, + OPTION_DYNAMIC_BASE, ++ OPTION_NO_DYNAMIC_BASE, + OPTION_FORCE_INTEGRITY, + OPTION_NX_COMPAT, ++ OPTION_NO_NX_COMPAT, + OPTION_NO_ISOLATION, + OPTION_NO_SEH, + OPTION_NO_BIND, +@@ -248,7 +254,8 @@ + OPTION_NO_INSERT_TIMESTAMP, + OPTION_TERMINAL_SERVER_AWARE, + OPTION_BUILD_ID, +- OPTION_ENABLE_RELOC_SECTION ++ OPTION_ENABLE_RELOC_SECTION, ++ OPTION_DISABLE_RELOC_SECTION + }; + + static void +@@ -315,9 +322,12 @@ + {"enable-long-section-names", no_argument, NULL, OPTION_ENABLE_LONG_SECTION_NAMES}, + {"disable-long-section-names", no_argument, NULL, OPTION_DISABLE_LONG_SECTION_NAMES}, + {"high-entropy-va", no_argument, NULL, OPTION_HIGH_ENTROPY_VA}, ++ {"no-high-entropy-va", no_argument, NULL, OPTION_NO_HIGH_ENTROPY_VA}, + {"dynamicbase",no_argument, NULL, OPTION_DYNAMIC_BASE}, ++ {"no-dynamicbase", no_argument, NULL, OPTION_NO_DYNAMIC_BASE}, + {"forceinteg", no_argument, NULL, OPTION_FORCE_INTEGRITY}, + {"nxcompat", no_argument, NULL, OPTION_NX_COMPAT}, ++ {"no-nxcompat", no_argument, NULL, OPTION_NO_NX_COMPAT}, + {"no-isolation", no_argument, NULL, OPTION_NO_ISOLATION}, + {"no-seh", no_argument, NULL, OPTION_NO_SEH}, + {"no-bind", no_argument, NULL, OPTION_NO_BIND}, +@@ -327,6 +337,7 @@ + {"no-insert-timestamp", no_argument, NULL, OPTION_NO_INSERT_TIMESTAMP}, + {"build-id", optional_argument, NULL, OPTION_BUILD_ID}, + {"enable-reloc-section", no_argument, NULL, OPTION_ENABLE_RELOC_SECTION}, ++ {"disable-reloc-section", no_argument, NULL, OPTION_DISABLE_RELOC_SECTION}, + {NULL, no_argument, NULL, 0} + }; + +@@ -448,11 +461,15 @@ + in object files\n")); + fprintf (file, _(" --high-entropy-va Image is compatible with 64-bit address space\n\ + layout randomization (ASLR)\n")); ++ fprintf (file, _(" --no-high-entropy-va Image is not compatible with 64-bit ASLR\n")); + fprintf (file, _(" --dynamicbase Image base address may be relocated using\n\ + address space layout randomization (ASLR)\n")); ++ fprintf (file, _(" --no-dynamicbase Image base address may not be relocated\n")); + fprintf (file, _(" --enable-reloc-section Create the base relocation table\n")); ++ fprintf (file, _(" --disable-reloc-section Disable the base relocation table\n")); + fprintf (file, _(" --forceinteg Code integrity checks are enforced\n")); + fprintf (file, _(" --nxcompat Image is compatible with data execution prevention\n")); ++ fprintf (file, _(" --no-nxcompat Image is not compatible with data execution prevention\n")); + fprintf (file, _(" --no-isolation Image understands isolation but do not isolate the image\n")); + fprintf (file, _(" --no-seh Image does not use SEH; no SE handler may\n\ + be called in this image\n")); +@@ -809,12 +826,24 @@ + case OPTION_ENABLE_RELOC_SECTION: + pep_dll_enable_reloc_section = 1; + break; ++ case OPTION_DISABLE_RELOC_SECTION: ++ pep_dll_enable_reloc_section = 0; ++ /* fall through */ ++ case OPTION_NO_DYNAMIC_BASE: ++ pe_dll_characteristics &= ~IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE; ++ /* fall through */ ++ case OPTION_NO_HIGH_ENTROPY_VA: ++ pe_dll_characteristics &= ~IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA; ++ break; + case OPTION_FORCE_INTEGRITY: + pe_dll_characteristics |= IMAGE_DLL_CHARACTERISTICS_FORCE_INTEGRITY; + break; + case OPTION_NX_COMPAT: + pe_dll_characteristics |= IMAGE_DLL_CHARACTERISTICS_NX_COMPAT; + break; ++ case OPTION_NO_NX_COMPAT: ++ pe_dll_characteristics &= ~IMAGE_DLL_CHARACTERISTICS_NX_COMPAT; ++ break; + case OPTION_NO_ISOLATION: + pe_dll_characteristics |= IMAGE_DLLCHARACTERISTICS_NO_ISOLATION; + break; diff --git a/contrib/macdeploy/README.md b/contrib/macdeploy/README.md index 73cbd71e8c5d4..802a8f9467386 100644 --- a/contrib/macdeploy/README.md +++ b/contrib/macdeploy/README.md @@ -6,10 +6,6 @@ The `macdeployqtplus` script should not be run manually. Instead, after building make deploy ``` -During the deployment process, the disk image window will pop up briefly -when the fancy settings are applied. This is normal, please do not interfere, -the process will unmount the DMG and cleanup before finishing. - When complete, it will have produced `Dash-Qt.dmg`. ## SDK Extraction @@ -110,7 +106,7 @@ broken. Only the compression feature is currently used. Ideally, the creation co and `xorrisofs` would no longer be necessary. Background images and other features can be added to DMG files by inserting a -`.DS_Store` before creation. This is generated by the script `contrib/macdeploy/custom_dsstore.py`. +`.DS_Store` during creation. As of OS X 10.9 Mavericks, using an Apple-blessed key to sign binaries is a requirement in order to satisfy the new Gatekeeper requirements. Because this private key cannot be diff --git a/contrib/macdeploy/custom_dsstore.py b/contrib/macdeploy/custom_dsstore.py deleted file mode 100755 index d7af07e9ed698..0000000000000 --- a/contrib/macdeploy/custom_dsstore.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2013-2015 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -from ds_store import DSStore -from mac_alias import Alias -import sys - -output_file = sys.argv[1] -package_name_ns = sys.argv[2] - -ds = DSStore.open(output_file, 'w+') -ds['.']['bwsp'] = { - 'ShowStatusBar': False, - 'WindowBounds': '{{300, 280}, {500, 343}}', - 'ContainerShowSidebar': False, - 'SidebarWidth': 0, - 'ShowTabView': False, - 'PreviewPaneVisibility': False, - 'ShowToolbar': False, - 'ShowSidebar': False, - 'ShowPathbar': True -} - -icvp = { - 'gridOffsetX': 0.0, - 'textSize': 12.0, - 'viewOptionsVersion': 1, - 'backgroundImageAlias': b'\x00\x00\x00\x00\x02\x1e\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xd1\x94\\\xb0H+\x00\x05\x00\x00\x00\x98\x0fbackground.tiff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x99\xd19\xb0\xf8\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\r\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b.background\x00\x00\x10\x00\x08\x00\x00\xd1\x94\\\xb0\x00\x00\x00\x11\x00\x08\x00\x00\xd19\xb0\xf8\x00\x00\x00\x01\x00\x04\x00\x00\x00\x98\x00\x0e\x00 \x00\x0f\x00b\x00a\x00c\x00k\x00g\x00r\x00o\x00u\x00n\x00d\x00.\x00t\x00i\x00f\x00f\x00\x0f\x00\x02\x00\x00\x00\x12\x00\x1c/.background/background.tiff\x00\x14\x01\x06\x00\x00\x00\x00\x01\x06\x00\x02\x00\x00\x0cMacintosh HD\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xce\x97\xab\xc3H+\x00\x00\x01\x88[\x88\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02u\xab\x8d\xd1\x94\\\xb0devrddsk\xff\xff\xff\xff\x00\x00\t \x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x07bitcoin\x00\x00\x10\x00\x08\x00\x00\xce\x97\xab\xc3\x00\x00\x00\x11\x00\x08\x00\x00\xd1\x94\\\xb0\x00\x00\x00\x01\x00\x14\x01\x88[\x88\x00\x16\xa9\t\x00\x08\xfaR\x00\x08\xfaQ\x00\x02d\x8e\x00\x0e\x00\x02\x00\x00\x00\x0f\x00\x1a\x00\x0c\x00M\x00a\x00c\x00i\x00n\x00t\x00o\x00s\x00h\x00 \x00H\x00D\x00\x13\x00\x01/\x00\x00\x15\x00\x02\x00\x14\xff\xff\x00\x00\xff\xff\x00\x00', - 'backgroundColorBlue': 1.0, - 'iconSize': 96.0, - 'backgroundColorGreen': 1.0, - 'arrangeBy': 'none', - 'showIconPreview': True, - 'gridSpacing': 100.0, - 'gridOffsetY': 0.0, - 'showItemInfo': False, - 'labelOnBottom': True, - 'backgroundType': 2, - 'backgroundColorRed': 1.0 -} -alias = Alias.from_bytes(icvp['backgroundImageAlias']) -alias.volume.name = package_name_ns -alias.volume.posix_path = '/Volumes/' + package_name_ns -alias.volume.disk_image_alias.target.filename = package_name_ns + '.temp.dmg' -alias.volume.disk_image_alias.target.carbon_path = 'Macintosh HD:Users:\x00dashcoreuser:\x00Documents:\x00dashcore:\x00dashcore:\x00' + package_name_ns + '.temp.dmg' -alias.volume.disk_image_alias.target.posix_path = 'Users/dashcoreuser/Documents/dashcore/dashcore/' + package_name_ns + '.temp.dmg' -alias.target.carbon_path = package_name_ns + ':.background:\x00background.tiff' -icvp['backgroundImageAlias'] = alias.to_bytes() -ds['.']['icvp'] = icvp - -ds['.']['vSrn'] = ('long', 1) - -ds['Applications']['Iloc'] = (370, 156) -ds['Dash-Qt.app']['Iloc'] = (128, 156) - -ds.flush() -ds.close() diff --git a/contrib/macdeploy/fancy.plist b/contrib/macdeploy/fancy.plist deleted file mode 100644 index 90071abeaba4d..0000000000000 --- a/contrib/macdeploy/fancy.plist +++ /dev/null @@ -1,32 +0,0 @@ - - - - - window_bounds - - 300 - 300 - 800 - 620 - - background_picture - background.tiff - icon_size - 96 - applications_symlink - - items_position - - Applications - - 370 - 156 - - Dash-Qt.app - - 128 - 156 - - - - diff --git a/contrib/macdeploy/macdeployqtplus b/contrib/macdeploy/macdeployqtplus index 1b3adb9c156c7..48b6d2eb58d3b 100755 --- a/contrib/macdeploy/macdeployqtplus +++ b/contrib/macdeploy/macdeployqtplus @@ -16,9 +16,13 @@ # along with this program. If not, see . # -import subprocess, sys, re, os, shutil, stat, os.path, time -from string import Template +import plistlib +import sys, re, os, shutil, stat, os.path from argparse import ArgumentParser +from ds_store import DSStore +from mac_alias import Alias +from pathlib import Path +from subprocess import PIPE, run from typing import List, Optional # This is ported from the original macdeployqt with modifications @@ -49,28 +53,18 @@ class FrameworkInfo(object): return False def __str__(self): - return """ Framework name: {} - Framework directory: {} - Framework path: {} - Binary name: {} - Binary directory: {} - Binary path: {} - Version: {} - Install name: {} - Deployed install name: {} - Source file Path: {} - Deployed Directory (relative to bundle): {} -""".format(self.frameworkName, - self.frameworkDirectory, - self.frameworkPath, - self.binaryName, - self.binaryDirectory, - self.binaryPath, - self.version, - self.installName, - self.deployedInstallName, - self.sourceFilePath, - self.destinationDirectory) + return f""" Framework name: {frameworkName} + Framework directory: {self.frameworkDirectory} + Framework path: {self.frameworkPath} + Binary name: {self.binaryName} + Binary directory: {self.binaryDirectory} + Binary path: {self.binaryPath} + Version: {self.version} + Install name: {self.installName} + Deployed install name: {self.deployedInstallName} + Source file Path: {self.sourceFilePath} + Deployed Directory (relative to bundle): {self.destinationDirectory} +""" def isDylib(self): return self.frameworkName.endswith(".dylib") @@ -97,7 +91,7 @@ class FrameworkInfo(object): m = cls.reOLine.match(line) if m is None: - raise RuntimeError("otool line could not be parsed: " + line) + raise RuntimeError(f"otool line could not be parsed: {line}") path = m.group(1) @@ -117,7 +111,7 @@ class FrameworkInfo(object): info.version = "-" info.installName = path - info.deployedInstallName = "@executable_path/../Frameworks/" + info.binaryName + info.deployedInstallName = f"@executable_path/../Frameworks/{info.binaryName}" info.sourceFilePath = path info.destinationDirectory = cls.bundleFrameworkDirectory else: @@ -129,7 +123,7 @@ class FrameworkInfo(object): break i += 1 if i == len(parts): - raise RuntimeError("Could not find .framework or .dylib in otool line: " + line) + raise RuntimeError(f"Could not find .framework or .dylib in otool line: {line}") info.frameworkName = parts[i] info.frameworkDirectory = "/".join(parts[:i]) @@ -140,7 +134,7 @@ class FrameworkInfo(object): info.binaryPath = os.path.join(info.binaryDirectory, info.binaryName) info.version = parts[i+2] - info.deployedInstallName = "@executable_path/../Frameworks/" + os.path.join(info.frameworkName, info.binaryPath) + info.deployedInstallName = f"@executable_path/../Frameworks/{os.path.join(info.frameworkName, info.binaryPath)}" info.destinationDirectory = os.path.join(cls.bundleFrameworkDirectory, info.frameworkName, info.binaryDirectory) info.sourceResourcesDirectory = os.path.join(info.frameworkPath, "Resources") @@ -154,10 +148,10 @@ class FrameworkInfo(object): class ApplicationBundleInfo(object): def __init__(self, path: str): self.path = path - appName = "Dash-Qt" - self.binaryPath = os.path.join(path, "Contents", "MacOS", appName) + # for backwards compatibility reasons, this must remain as Dash-Qt + self.binaryPath = os.path.join(path, "Contents", "MacOS", "Dash-Qt") if not os.path.exists(self.binaryPath): - raise RuntimeError("Could not find bundle binary for " + path) + raise RuntimeError(f"Could not find bundle binary for {path}") self.resourcesPath = os.path.join(path, "Contents", "Resources") self.pluginPath = os.path.join(path, "Contents", "PlugIns") @@ -181,30 +175,26 @@ class DeploymentInfo(object): self.pluginPath = pluginPath def usesFramework(self, name: str) -> bool: - nameDot = "{}.".format(name) - libNameDot = "lib{}.".format(name) for framework in self.deployedFrameworks: if framework.endswith(".framework"): - if framework.startswith(nameDot): + if framework.startswith(f"{name}."): return True elif framework.endswith(".dylib"): - if framework.startswith(libNameDot): + if framework.startswith(f"lib{name}."): return True return False def getFrameworks(binaryPath: str, verbose: int) -> List[FrameworkInfo]: - if verbose >= 3: - print("Inspecting with otool: " + binaryPath) + if verbose: + print(f"Inspecting with otool: {binaryPath}") otoolbin=os.getenv("OTOOL", "otool") - otool = subprocess.Popen([otoolbin, "-L", binaryPath], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) - o_stdout, o_stderr = otool.communicate() + otool = run([otoolbin, "-L", binaryPath], stdout=PIPE, stderr=PIPE, universal_newlines=True) if otool.returncode != 0: - if verbose >= 1: - sys.stderr.write(o_stderr) - sys.stderr.flush() - raise RuntimeError("otool failed with return code {}".format(otool.returncode)) + sys.stderr.write(otool.stderr) + sys.stderr.flush() + raise RuntimeError(f"otool failed with return code {otool.returncode}") - otoolLines = o_stdout.split("\n") + otoolLines = otool.stdout.split("\n") otoolLines.pop(0) # First line is the inspected binary if ".framework" in binaryPath or binaryPath.endswith(".dylib"): otoolLines.pop(0) # Frameworks and dylibs list themselves as a dependency. @@ -214,7 +204,7 @@ def getFrameworks(binaryPath: str, verbose: int) -> List[FrameworkInfo]: line = line.replace("@loader_path", os.path.dirname(binaryPath)) info = FrameworkInfo.fromOtoolLibraryLine(line.strip()) if info is not None: - if verbose >= 3: + if verbose: print("Found framework:") print(info) libraries.append(info) @@ -223,10 +213,10 @@ def getFrameworks(binaryPath: str, verbose: int) -> List[FrameworkInfo]: def runInstallNameTool(action: str, *args): installnametoolbin=os.getenv("INSTALLNAMETOOL", "install_name_tool") - subprocess.check_call([installnametoolbin, "-"+action] + list(args)) + run([installnametoolbin, "-"+action] + list(args), check=True) def changeInstallName(oldName: str, newName: str, binaryPath: str, verbose: int): - if verbose >= 3: + if verbose: print("Using install_name_tool:") print(" in", binaryPath) print(" change reference", oldName) @@ -234,7 +224,7 @@ def changeInstallName(oldName: str, newName: str, binaryPath: str, verbose: int) runInstallNameTool("change", oldName, newName, binaryPath) def changeIdentification(id: str, binaryPath: str, verbose: int): - if verbose >= 3: + if verbose: print("Using install_name_tool:") print(" change identification in", binaryPath) print(" to", id) @@ -242,22 +232,22 @@ def changeIdentification(id: str, binaryPath: str, verbose: int): def runStrip(binaryPath: str, verbose: int): stripbin=os.getenv("STRIP", "strip") - if verbose >= 3: + if verbose: print("Using strip:") print(" stripped", binaryPath) - subprocess.check_call([stripbin, "-x", binaryPath]) + run([stripbin, "-x", binaryPath], check=True) def copyFramework(framework: FrameworkInfo, path: str, verbose: int) -> Optional[str]: if framework.sourceFilePath.startswith("Qt"): #standard place for Nokia Qt installer's frameworks - fromPath = "/Library/Frameworks/" + framework.sourceFilePath + fromPath = f"/Library/Frameworks/{framework.sourceFilePath}" else: fromPath = framework.sourceFilePath toDir = os.path.join(path, framework.destinationDirectory) toPath = os.path.join(toDir, framework.binaryName) if not os.path.exists(fromPath): - raise RuntimeError("No file at " + fromPath) + raise RuntimeError(f"No file at {fromPath}") if os.path.exists(toPath): return None # Already there @@ -266,7 +256,7 @@ def copyFramework(framework: FrameworkInfo, path: str, verbose: int) -> Optional os.makedirs(toDir) shutil.copy2(fromPath, toPath) - if verbose >= 3: + if verbose: print("Copied:", fromPath) print(" to:", toPath) @@ -280,13 +270,12 @@ def copyFramework(framework: FrameworkInfo, path: str, verbose: int) -> Optional linkto = framework.version if not os.path.exists(linkfrom): os.symlink(linkto, linkfrom) - if verbose >= 2: - print("Linked:", linkfrom, "->", linkto) + print("Linked:", linkfrom, "->", linkto) fromResourcesDir = framework.sourceResourcesDirectory if os.path.exists(fromResourcesDir): toResourcesDir = os.path.join(path, framework.destinationResourcesDirectory) shutil.copytree(fromResourcesDir, toResourcesDir, symlinks=True) - if verbose >= 3: + if verbose: print("Copied resources:", fromResourcesDir) print(" to:", toResourcesDir) fromContentsDir = framework.sourceVersionContentsDirectory @@ -295,7 +284,7 @@ def copyFramework(framework: FrameworkInfo, path: str, verbose: int) -> Optional if os.path.exists(fromContentsDir): toContentsDir = os.path.join(path, framework.destinationVersionContentsDirectory) shutil.copytree(fromContentsDir, toContentsDir, symlinks=True) - if verbose >= 3: + if verbose: print("Copied Contents:", fromContentsDir) print(" to:", toContentsDir) elif framework.frameworkName.startswith("libQtGui"): # Copy qt_menu.nib (applies to non-framework layout) @@ -303,7 +292,7 @@ def copyFramework(framework: FrameworkInfo, path: str, verbose: int) -> Optional qtMenuNibDestinationPath = os.path.join(path, "Contents", "Resources", "qt_menu.nib") if os.path.exists(qtMenuNibSourcePath) and not os.path.exists(qtMenuNibDestinationPath): shutil.copytree(qtMenuNibSourcePath, qtMenuNibDestinationPath, symlinks=True) - if verbose >= 3: + if verbose: print("Copied for libQtGui:", qtMenuNibSourcePath) print(" to:", qtMenuNibDestinationPath) @@ -317,16 +306,14 @@ def deployFrameworks(frameworks: List[FrameworkInfo], bundlePath: str, binaryPat framework = frameworks.pop(0) deploymentInfo.deployedFrameworks.append(framework.frameworkName) - if verbose >= 2: - print("Processing", framework.frameworkName, "...") + print("Processing", framework.frameworkName, "...") # Get the Qt path from one of the Qt frameworks if deploymentInfo.qtPath is None and framework.isQtFramework(): deploymentInfo.detectQtPath(framework.frameworkDirectory) if framework.installName.startswith("@executable_path") or framework.installName.startswith(bundlePath): - if verbose >= 2: - print(framework.frameworkName, "already deployed, skipping.") + print(framework.frameworkName, "already deployed, skipping.") continue # install_name_tool the new id into the binary @@ -357,8 +344,8 @@ def deployFrameworks(frameworks: List[FrameworkInfo], bundlePath: str, binaryPat def deployFrameworksForAppBundle(applicationBundle: ApplicationBundleInfo, strip: bool, verbose: int) -> DeploymentInfo: frameworks = getFrameworks(applicationBundle.binaryPath, verbose) - if len(frameworks) == 0 and verbose >= 1: - print("Warning: Could not find any external frameworks to deploy in {}.".format(applicationBundle.path)) + if len(frameworks) == 0: + print(f"Warning: Could not find any external frameworks to deploy in {applicationBundle.path}.") return DeploymentInfo() else: return deployFrameworks(frameworks, applicationBundle.path, applicationBundle.binaryPath, strip, verbose) @@ -477,8 +464,7 @@ def deployPlugins(appBundleInfo: ApplicationBundleInfo, deploymentInfo: Deployme plugins.append((pluginDirectory, pluginName)) for pluginDirectory, pluginName in plugins: - if verbose >= 2: - print("Processing plugin", os.path.join(pluginDirectory, pluginName), "...") + print("Processing plugin", os.path.join(pluginDirectory, pluginName), "...") sourcePath = os.path.join(deploymentInfo.pluginPath, pluginDirectory, pluginName) destinationDirectory = os.path.join(appBundleInfo.pluginPath, pluginDirectory) @@ -487,7 +473,7 @@ def deployPlugins(appBundleInfo: ApplicationBundleInfo, deploymentInfo: Deployme destinationPath = os.path.join(destinationDirectory, pluginName) shutil.copy2(sourcePath, destinationPath) - if verbose >= 3: + if verbose: print("Copied:", sourcePath) print(" to:", destinationPath) @@ -503,147 +489,50 @@ def deployPlugins(appBundleInfo: ApplicationBundleInfo, deploymentInfo: Deployme if dependency.frameworkName not in deploymentInfo.deployedFrameworks: deployFrameworks([dependency], appBundleInfo.path, destinationPath, strip, verbose, deploymentInfo) -qt_conf="""[Paths] -Translations=Resources -Plugins=PlugIns -""" - ap = ArgumentParser(description="""Improved version of macdeployqt. Outputs a ready-to-deploy app in a folder "dist" and optionally wraps it in a .dmg file. Note, that the "dist" folder will be deleted before deploying on each run. -Optionally, Qt translation files (.qm) and additional resources can be added to the bundle. - -Also optionally signs the .app bundle; set the CODESIGNARGS environment variable to pass arguments -to the codesign tool. -E.g. CODESIGNARGS='--sign "Developer ID Application: ..." --keychain /encrypted/foo.keychain'""") +Optionally, Qt translation files (.qm) can be added to the bundle.""") ap.add_argument("app_bundle", nargs=1, metavar="app-bundle", help="application bundle to be deployed") -ap.add_argument("-verbose", type=int, nargs=1, default=[1], metavar="<0-3>", help="0 = no output, 1 = error/warning (default), 2 = normal, 3 = debug") +ap.add_argument("appname", nargs=1, metavar="appname", help="name of the app being deployed") +ap.add_argument("-verbose", nargs="?", const=True, help="Output additional debugging information") ap.add_argument("-no-plugins", dest="plugins", action="store_false", default=True, help="skip plugin deployment") ap.add_argument("-no-strip", dest="strip", action="store_false", default=True, help="don't run 'strip' on the binaries") -ap.add_argument("-sign", dest="sign", action="store_true", default=False, help="sign .app bundle with codesign tool") -ap.add_argument("-dmg", nargs="?", const="", metavar="basename", help="create a .dmg disk image; if basename is not specified, a camel-cased version of the app name is used") -ap.add_argument("-fancy", nargs=1, metavar="plist", default=[], help="make a fancy looking disk image using the given plist file with instructions; requires -dmg to work") -ap.add_argument("-add-qt-tr", nargs=1, metavar="languages", default=[], help="add Qt translation files to the bundle's resources; the language list must be separated with commas, not with whitespace") -ap.add_argument("-translations-dir", nargs=1, metavar="path", default=None, help="Path to Qt's translation files") -ap.add_argument("-add-resources", nargs="+", metavar="path", default=[], help="list of additional files or folders to be copied into the bundle's resources; must be the last argument") -ap.add_argument("-volname", nargs=1, metavar="volname", default=[], help="custom volume name for dmg") +ap.add_argument("-dmg", nargs="?", const="", metavar="basename", help="create a .dmg disk image") +ap.add_argument("-translations-dir", nargs=1, metavar="path", default=None, help="Path to Qt's translations. Base translations will automatically be added to the bundle's resources.") config = ap.parse_args() -verbose = config.verbose[0] +verbose = config.verbose # ------------------------------------------------ app_bundle = config.app_bundle[0] +appname = config.appname[0] if not os.path.exists(app_bundle): - if verbose >= 1: - sys.stderr.write("Error: Could not find app bundle \"{}\"\n".format(app_bundle)) + sys.stderr.write(f"Error: Could not find app bundle \"{app_bundle}\"\n") sys.exit(1) -app_bundle_name = os.path.splitext(os.path.basename(app_bundle))[0] - -# ------------------------------------------------ -translations_dir = None -if config.translations_dir and config.translations_dir[0]: - if os.path.exists(config.translations_dir[0]): - translations_dir = config.translations_dir[0] - else: - if verbose >= 1: - sys.stderr.write("Error: Could not find translation dir \"{}\"\n".format(translations_dir)) - sys.exit(1) -# ------------------------------------------------ - -for p in config.add_resources: - if verbose >= 3: - print("Checking for \"%s\"..." % p) - if not os.path.exists(p): - if verbose >= 1: - sys.stderr.write("Error: Could not find additional resource file \"{}\"\n".format(p)) - sys.exit(1) - -# ------------------------------------------------ - -if len(config.fancy) == 1: - if verbose >= 3: - print("Fancy: Importing plistlib...") - try: - import plistlib - except ImportError: - if verbose >= 1: - sys.stderr.write("Error: Could not import plistlib which is required for fancy disk images.\n") - sys.exit(1) - - p = config.fancy[0] - if verbose >= 3: - print("Fancy: Loading \"{}\"...".format(p)) - if not os.path.exists(p): - if verbose >= 1: - sys.stderr.write("Error: Could not find fancy disk image plist at \"{}\"\n".format(p)) - sys.exit(1) - - try: - with open(p, 'rb') as fp: - fancy = plistlib.load(fp, fmt=plistlib.FMT_XML) - except: - if verbose >= 1: - sys.stderr.write("Error: Could not parse fancy disk image plist at \"{}\"\n".format(p)) - sys.exit(1) - - try: - assert "window_bounds" not in fancy or (isinstance(fancy["window_bounds"], list) and len(fancy["window_bounds"]) == 4) - assert "background_picture" not in fancy or isinstance(fancy["background_picture"], str) - assert "icon_size" not in fancy or isinstance(fancy["icon_size"], int) - assert "applications_symlink" not in fancy or isinstance(fancy["applications_symlink"], bool) - if "items_position" in fancy: - assert isinstance(fancy["items_position"], dict) - for key, value in fancy["items_position"].items(): - assert isinstance(value, list) and len(value) == 2 and isinstance(value[0], int) and isinstance(value[1], int) - except: - if verbose >= 1: - sys.stderr.write("Error: Bad format of fancy disk image plist at \"{}\"\n".format(p)) - sys.exit(1) - - if "background_picture" in fancy: - bp = fancy["background_picture"] - if verbose >= 3: - print("Fancy: Resolving background picture \"{}\"...".format(bp)) - if not os.path.exists(bp): - bp = os.path.join(os.path.dirname(p), bp) - if not os.path.exists(bp): - if verbose >= 1: - sys.stderr.write("Error: Could not find background picture at \"{}\" or \"{}\"\n".format(fancy["background_picture"], bp)) - sys.exit(1) - else: - fancy["background_picture"] = bp -else: - fancy = None - # ------------------------------------------------ if os.path.exists("dist"): - if verbose >= 2: - print("+ Removing old dist folder +") - + print("+ Removing existing dist folder +") shutil.rmtree("dist") -# ------------------------------------------------ - -if len(config.volname) == 1: - volname = config.volname[0] -else: - volname = app_bundle_name +if os.path.exists(appname + ".dmg"): + print("+ Removing existing DMG +") + os.unlink(appname + ".dmg") # ------------------------------------------------ target = os.path.join("dist", "Dash-Qt.app") -if verbose >= 2: - print("+ Copying source bundle +") -if verbose >= 3: +print("+ Copying source bundle +") +if verbose: print(app_bundle, "->", target) os.mkdir("dist") @@ -653,257 +542,154 @@ applicationBundle = ApplicationBundleInfo(target) # ------------------------------------------------ -if verbose >= 2: - print("+ Deploying frameworks +") +print("+ Deploying frameworks +") try: deploymentInfo = deployFrameworksForAppBundle(applicationBundle, config.strip, verbose) if deploymentInfo.qtPath is None: deploymentInfo.qtPath = os.getenv("QTDIR", None) if deploymentInfo.qtPath is None: - if verbose >= 1: - sys.stderr.write("Warning: Could not detect Qt's path, skipping plugin deployment!\n") + sys.stderr.write("Warning: Could not detect Qt's path, skipping plugin deployment!\n") config.plugins = False except RuntimeError as e: - if verbose >= 1: - sys.stderr.write("Error: {}\n".format(str(e))) + sys.stderr.write(f"Error: {str(e)}\n") sys.exit(1) # ------------------------------------------------ if config.plugins: - if verbose >= 2: - print("+ Deploying plugins +") + print("+ Deploying plugins +") try: deployPlugins(applicationBundle, deploymentInfo, config.strip, verbose) except RuntimeError as e: - if verbose >= 1: - sys.stderr.write("Error: {}\n".format(str(e))) + sys.stderr.write(f"Error: {str(e)}\n") sys.exit(1) # ------------------------------------------------ -if len(config.add_qt_tr) == 0: - add_qt_tr = [] -else: - if translations_dir is not None: - qt_tr_dir = translations_dir - else: - if deploymentInfo.qtPath is not None: - qt_tr_dir = os.path.join(deploymentInfo.qtPath, "translations") - else: - sys.stderr.write("Error: Could not find Qt translation path\n") - sys.exit(1) - add_qt_tr = ["qt_{}.qm".format(lng) for lng in config.add_qt_tr[0].split(",")] - for lng_file in add_qt_tr: - p = os.path.join(qt_tr_dir, lng_file) - if verbose >= 3: - print("Checking for \"{}\"...".format(p)) - if not os.path.exists(p): - if verbose >= 1: - sys.stderr.write("Error: Could not find Qt translation file \"{}\"\n".format(lng_file)) - sys.exit(1) +if config.translations_dir: + if not Path(config.translations_dir[0]).exists(): + sys.stderr.write(f"Error: Could not find translation dir \"{config.translations_dir[0]}\"\n") + sys.exit(1) + +print("+ Adding Qt translations +") + +translations = Path(config.translations_dir[0]) + +regex = re.compile('qt_[a-z]*(.qm|_[A-Z]*.qm)') + +lang_files = [x for x in translations.iterdir() if regex.match(x.name)] + +for file in lang_files: + if verbose: + print(file.as_posix(), "->", os.path.join(applicationBundle.resourcesPath, file.name)) + shutil.copy2(file.as_posix(), os.path.join(applicationBundle.resourcesPath, file.name)) # ------------------------------------------------ -if verbose >= 2: - print("+ Installing qt.conf +") +print("+ Installing qt.conf +") + +qt_conf="""[Paths] +Translations=Resources +Plugins=PlugIns +""" with open(os.path.join(applicationBundle.resourcesPath, "qt.conf"), "wb") as f: f.write(qt_conf.encode()) # ------------------------------------------------ -if len(add_qt_tr) > 0 and verbose >= 2: - print("+ Adding Qt translations +") - -for lng_file in add_qt_tr: - if verbose >= 3: - print(os.path.join(qt_tr_dir, lng_file), "->", os.path.join(applicationBundle.resourcesPath, lng_file)) - shutil.copy2(os.path.join(qt_tr_dir, lng_file), os.path.join(applicationBundle.resourcesPath, lng_file)) +print("+ Generating .DS_Store +") + +output_file = os.path.join("dist", ".DS_Store") + +ds = DSStore.open(output_file, 'w+') + +ds['.']['bwsp'] = { + 'WindowBounds': '{{300, 280}, {500, 343}}', + 'PreviewPaneVisibility': False, +} + +icvp = { + 'gridOffsetX': 0.0, + 'textSize': 12.0, + 'viewOptionsVersion': 1, + 'backgroundImageAlias': b'\x00\x00\x00\x00\x02\x1e\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xd1\x94\\\xb0H+\x00\x05\x00\x00\x00\x98\x0fbackground.tiff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x99\xd19\xb0\xf8\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\r\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b.background\x00\x00\x10\x00\x08\x00\x00\xd1\x94\\\xb0\x00\x00\x00\x11\x00\x08\x00\x00\xd19\xb0\xf8\x00\x00\x00\x01\x00\x04\x00\x00\x00\x98\x00\x0e\x00 \x00\x0f\x00b\x00a\x00c\x00k\x00g\x00r\x00o\x00u\x00n\x00d\x00.\x00t\x00i\x00f\x00f\x00\x0f\x00\x02\x00\x00\x00\x12\x00\x1c/.background/background.tiff\x00\x14\x01\x06\x00\x00\x00\x00\x01\x06\x00\x02\x00\x00\x0cMacintosh HD\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xce\x97\xab\xc3H+\x00\x00\x01\x88[\x88\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02u\xab\x8d\xd1\x94\\\xb0devrddsk\xff\xff\xff\xff\x00\x00\t \x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x07bitcoin\x00\x00\x10\x00\x08\x00\x00\xce\x97\xab\xc3\x00\x00\x00\x11\x00\x08\x00\x00\xd1\x94\\\xb0\x00\x00\x00\x01\x00\x14\x01\x88[\x88\x00\x16\xa9\t\x00\x08\xfaR\x00\x08\xfaQ\x00\x02d\x8e\x00\x0e\x00\x02\x00\x00\x00\x0f\x00\x1a\x00\x0c\x00M\x00a\x00c\x00i\x00n\x00t\x00o\x00s\x00h\x00 \x00H\x00D\x00\x13\x00\x01/\x00\x00\x15\x00\x02\x00\x14\xff\xff\x00\x00\xff\xff\x00\x00', + 'backgroundColorBlue': 1.0, + 'iconSize': 96.0, + 'backgroundColorGreen': 1.0, + 'arrangeBy': 'none', + 'showIconPreview': True, + 'gridSpacing': 100.0, + 'gridOffsetY': 0.0, + 'showItemInfo': False, + 'labelOnBottom': True, + 'backgroundType': 2, + 'backgroundColorRed': 1.0 +} +alias = Alias().from_bytes(icvp['backgroundImageAlias']) +alias.volume.name = appname +alias.volume.posix_path = '/Volumes/' + appname +icvp['backgroundImageAlias'] = alias.to_bytes() +ds['.']['icvp'] = icvp + +ds['.']['vSrn'] = ('long', 1) + +ds['Applications']['Iloc'] = (370, 156) +ds['Dash-Qt.app']['Iloc'] = (128, 156) + +ds.flush() +ds.close() # ------------------------------------------------ -if len(config.add_resources) > 0 and verbose >= 2: - print("+ Adding additional resources +") +if config.dmg is not None: -for p in config.add_resources: - t = os.path.join(applicationBundle.resourcesPath, os.path.basename(p)) - if verbose >= 3: - print(p, "->", t) - if os.path.isdir(p): - shutil.copytree(p, t, symlinks=True) - else: - shutil.copy2(p, t) + print("+ Preparing .dmg disk image +") -# ------------------------------------------------ + if verbose: + print("Determining size of \"dist\"...") + size = 0 + for path, dirs, files in os.walk("dist"): + for file in files: + size += os.path.getsize(os.path.join(path, file)) + size += int(size * 0.15) -if config.sign and 'CODESIGNARGS' not in os.environ: - print("You must set the CODESIGNARGS environment variable. Skipping signing.") -elif config.sign: - if verbose >= 1: - print("Code-signing app bundle {}".format(target)) - subprocess.check_call("codesign --force {} {}".format(os.environ['CODESIGNARGS'], target), shell=True) + if verbose: + print("Creating temp image for modification...") -# ------------------------------------------------ + tempname: str = appname + ".temp.dmg" -if config.dmg is not None: + run(["hdiutil", "create", tempname, "-srcfolder", "dist", "-format", "UDRW", "-size", str(size), "-volname", appname], check=True, universal_newlines=True) - def runHDIUtil(verb: str, image_basename: str, **kwargs) -> int: - hdiutil_args = ["hdiutil", verb, image_basename + ".dmg"] - if "capture_stdout" in kwargs: - del kwargs["capture_stdout"] - run = subprocess.check_output - else: - if verbose < 2: - hdiutil_args.append("-quiet") - elif verbose >= 3: - hdiutil_args.append("-verbose") - run = subprocess.check_call - - for key, value in kwargs.items(): - hdiutil_args.append("-" + key) - if value is not True: - hdiutil_args.append(str(value)) - - return run(hdiutil_args, universal_newlines=True) - if verbose >= 2: - if fancy is None: - print("+ Creating .dmg disk image +") - else: - print("+ Preparing .dmg disk image +") + if verbose: + print("Attaching temp image...") + output = run(["hdiutil", "attach", tempname, "-readwrite"], check=True, universal_newlines=True, stdout=PIPE).stdout - if config.dmg != "": - dmg_name = config.dmg - else: - spl = app_bundle_name.split(" ") - dmg_name = spl[0] + "".join(p.capitalize() for p in spl[1:]) - - if fancy is None: - try: - runHDIUtil("create", dmg_name, srcfolder="dist", format="UDBZ", volname=volname, ov=True) - except subprocess.CalledProcessError as e: - sys.exit(e.returncode) - else: - if verbose >= 3: - print("Determining size of \"dist\"...") - size = 0 - for path, dirs, files in os.walk("dist"): - for file in files: - size += os.path.getsize(os.path.join(path, file)) - size += int(size * 0.15) - - if verbose >= 3: - print("Creating temp image for modification...") - try: - runHDIUtil("create", dmg_name + ".temp", srcfolder="dist", format="UDRW", size=size, volname=volname, ov=True) - except subprocess.CalledProcessError as e: - sys.exit(e.returncode) - - if verbose >= 3: - print("Attaching temp image...") - try: - output = runHDIUtil("attach", dmg_name + ".temp", readwrite=True, noverify=True, noautoopen=True, capture_stdout=True) - except subprocess.CalledProcessError as e: - sys.exit(e.returncode) - - m = re.search(r"/Volumes/(.+$)", output) - disk_root = m.group(0) - disk_name = m.group(1) - - if verbose >= 2: - print("+ Applying fancy settings +") - - if "background_picture" in fancy: - bg_path = os.path.join(disk_root, ".background", os.path.basename(fancy["background_picture"])) - os.mkdir(os.path.dirname(bg_path)) - if verbose >= 3: - print(fancy["background_picture"], "->", bg_path) - shutil.copy2(fancy["background_picture"], bg_path) - else: - bg_path = None - - if fancy.get("applications_symlink", False): - os.symlink("/Applications", os.path.join(disk_root, "Applications")) - - # The Python appscript package broke with OSX 10.8 and isn't being fixed. - # So we now build up an AppleScript string and use the osascript command - # to make the .dmg file pretty: - appscript = Template( """ - on run argv - tell application "Finder" - tell disk "$disk" - open - set current view of container window to icon view - set toolbar visible of container window to false - set statusbar visible of container window to false - set the bounds of container window to {$window_bounds} - set theViewOptions to the icon view options of container window - set arrangement of theViewOptions to not arranged - set icon size of theViewOptions to $icon_size - $background_commands - $items_positions - close -- close/reopen works around a bug... - open - update without registering applications - delay 5 - eject - end tell - end tell - end run - """) - - itemscript = Template('set position of item "${item}" of container window to {${position}}') - items_positions = [] - if "items_position" in fancy: - for name, position in fancy["items_position"].items(): - params = { "item" : name, "position" : ",".join([str(p) for p in position]) } - items_positions.append(itemscript.substitute(params)) - - params = { - "disk" : volname, - "window_bounds" : "300,300,800,620", - "icon_size" : "96", - "background_commands" : "", - "items_positions" : "\n ".join(items_positions) - } - if "window_bounds" in fancy: - params["window_bounds"] = ",".join([str(p) for p in fancy["window_bounds"]]) - if "icon_size" in fancy: - params["icon_size"] = str(fancy["icon_size"]) - if bg_path is not None: - # Set background file, then call SetFile to make it invisible. - # (note: making it invisible first makes set background picture fail) - bgscript = Template("""set background picture of theViewOptions to file ".background:$bgpic" - do shell script "SetFile -a V /Volumes/$disk/.background/$bgpic" """) - params["background_commands"] = bgscript.substitute({"bgpic" : os.path.basename(bg_path), "disk" : params["disk"]}) - - s = appscript.substitute(params) - if verbose >= 2: - print("Running AppleScript:") - print(s) - - time.sleep(2) # fixes '112:116: execution error: Finder got an error: Can’t get disk "Dash-Core". (-1728)' - p = subprocess.Popen(['osascript', '-'], stdin=subprocess.PIPE) - p.communicate(input=s.encode('utf-8')) - if p.returncode: - print("Error running osascript.") - - if verbose >= 2: - print("+ Finalizing .dmg disk image +") - time.sleep(5) - - try: - runHDIUtil("convert", dmg_name + ".temp", format="UDBZ", o=dmg_name + ".dmg", ov=True) - except subprocess.CalledProcessError as e: - sys.exit(e.returncode) - - os.unlink(dmg_name + ".temp.dmg") + m = re.search(r"/Volumes/(.+$)", output) + disk_root = m.group(0) + + print("+ Applying fancy settings +") + + bg_path = os.path.join(disk_root, ".background", os.path.basename('background.tiff')) + os.mkdir(os.path.dirname(bg_path)) + if verbose: + print('background.tiff', "->", bg_path) + shutil.copy2('background.tiff', bg_path) + + os.symlink("/Applications", os.path.join(disk_root, "Applications")) + + print("+ Finalizing .dmg disk image +") + + run(["hdiutil", "detach", f"/Volumes/{appname}"], universal_newlines=True) + + run(["hdiutil", "convert", tempname, "-format", "UDZO", "-o", appname, "-imagekey", "zlib-level=9"], check=True, universal_newlines=True) + + os.unlink(tempname) # ------------------------------------------------ -if verbose >= 2: - print("+ Done +") +print("+ Done +") sys.exit(0) diff --git a/depends/packages/boost.mk b/depends/packages/boost.mk index 8b499367be242..8f307beb8f52a 100644 --- a/depends/packages/boost.mk +++ b/depends/packages/boost.mk @@ -30,6 +30,7 @@ $(package)_config_libraries=filesystem,thread,test $(package)_cxxflags=-std=c++17 -fvisibility=hidden $(package)_cxxflags_linux=-fPIC $(package)_cxxflags_android=-fPIC +$(package)_cxxflags_x86_64=-fcf-protection=full endef # Fix unused variable in boost_process, can be removed after upgrading to 1.72 diff --git a/depends/packages/native_mac_alias.mk b/depends/packages/native_mac_alias.mk index e60b99dccc98c..5fe027fb8a51b 100644 --- a/depends/packages/native_mac_alias.mk +++ b/depends/packages/native_mac_alias.mk @@ -1,8 +1,8 @@ package=native_mac_alias -$(package)_version=2.0.7 +$(package)_version=2.1.1 $(package)_download_path=https://github.com/al45tair/mac_alias/archive/ $(package)_file_name=v$($(package)_version).tar.gz -$(package)_sha256_hash=6f606d3b6bccd2112aeabf1a063f5b5ece87005a5d7e97c8faca23b916e88838 +$(package)_sha256_hash=c0ffceee14f7d04a6eb323fb7b8217dc3f373b346198d2ca42300a8362db7efa $(package)_install_libdir=$(build_prefix)/lib/python3/dist-packages define $(package)_build_cmds diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index e29b8645b7dfe..0b7464fc7cd89 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -17,7 +17,6 @@ $(package)_patches+= fix_montery_include.patch $(package)_patches += glibc_compatibility.patch $(package)_patches+= qtbase-moc-ignore-gcc-macro.patch -# Update OSX_QT_TRANSLATIONS when this is updated $(package)_qttranslations_file_name=qttranslations-$($(package)_suffix) $(package)_qttranslations_sha256_hash=577b0668a777eb2b451c61e8d026d79285371597ce9df06b6dee6c814164b7c3 diff --git a/src/Makefile.am b/src/Makefile.am index 1678beeb55fe2..edbf2936b0f38 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -933,25 +933,13 @@ clean-local: $(AM_V_GEN) $(WINDRES) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(CPPFLAGS) -DWINDRES_PREPROC -i $< -o $@ check-symbols: $(bin_PROGRAMS) -if TARGET_DARWIN - @echo "Checking macOS dynamic libraries..." - $(AM_V_at) OTOOL=$(OTOOL) $(PYTHON) $(top_srcdir)/contrib/devtools/symbol-check.py $(bin_PROGRAMS) -endif - -if TARGET_WINDOWS - @echo "Checking Windows dynamic libraries..." - $(AM_V_at) OBJDUMP=$(OBJDUMP) $(PYTHON) $(top_srcdir)/contrib/devtools/symbol-check.py $(bin_PROGRAMS) -endif - -if GLIBC_BACK_COMPAT - @echo "Checking glibc back compat..." - $(AM_V_at) CPPFILT=$(CPPFILT) $(PYTHON) $(top_srcdir)/contrib/devtools/symbol-check.py $(bin_PROGRAMS) -endif + @echo "Running symbol and dynamic library checks..." + $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/symbol-check.py $(bin_PROGRAMS) check-security: $(bin_PROGRAMS) if HARDEN @echo "Checking binary security..." - $(AM_V_at) OBJDUMP=$(OBJDUMP) OTOOL=$(OTOOL) $(PYTHON) $(top_srcdir)/contrib/devtools/security-check.py $(bin_PROGRAMS) + $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/security-check.py $(bin_PROGRAMS) endif