Skip to content

Tidying on external_deps #1992

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
illwieckz wants to merge 9 commits into
base: master
Choose a base branch
from
+65 −55
Open

Tidying on external_deps #1992

Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
27fb03b
external_deps: do not copy WASI into WASI
illwieckz Jun 25, 2026
522fe4d
external_deps: better wasmtime copy
illwieckz Jun 25, 2026
3b4a451
external_deps: better sdl3 copy
illwieckz Jun 25, 2026
711fdf9
external_deps: better copy in extract()
illwieckz Jun 25, 2026
4852585
external_deps: make EXE_EXT reusable
illwieckz Jun 25, 2026
e93c320
external_deps: tell xz to use extreme compression
illwieckz Jun 27, 2026
de32ca6
external_deps: rewrite some comments
illwieckz Jul 4, 2026
6019f66
external_deps: unify the style
illwieckz Jul 4, 2026
6d3b992
external_deps: add smart_copy
illwieckz Jun 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
120 changes: 65 additions & 55 deletions external_deps/build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,13 @@ log() {
[ "${level}" != 'ERROR' ]
}

smart_copy() {
if ! cp --reflink=auto -P "${@}" 2>/dev/null

@slipher slipher Jul 13, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As of GNU cp 9.x which is a few years old --reflink=auto is the default. So this is unnecessary.

@illwieckz illwieckz Jul 14, 2026 •
edited
Loading

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We better be picky, and we still build the game and its external deps on Debian Bullseye which has coreutils 8.32-4+b1 anyway.

then
cp -P "${@}"
fi
}

# Extract an archive into the given subdirectory of the build dir and cd to it
# Usage: extract <filename> <directory>
extract() {
Expand Down Expand Up @@ -122,7 +129,7 @@ extract() {
*.dmg)
local dmg_temp_dir="$(mktemp -d)"
hdiutil attach -mountpoint "${dmg_temp_dir}" "${archive_file}"
cp -R "${dmg_temp_dir}/"* "${extract_dir}/"
smart_copy -R "${dmg_temp_dir}/." "${extract_dir}/"
hdiutil detach "${dmg_temp_dir}"
rmdir "${dmg_temp_dir}"
;;
Expand Down Expand Up @@ -284,7 +291,7 @@ build_nasm() {

"${download_only}" && return

cp "${dir_name}/nasm" "${PREFIX}/bin"
smart_copy "${dir_name}/nasm" "${PREFIX}/bin"
;;
*)
log ERROR 'Unsupported platform for NASM'
Expand Down Expand Up @@ -455,16 +462,18 @@ build_sdl3() {
case "${PLATFORM}" in
windows-*-mingw)
cd "${dir_name}"
cp -rv "${HOST}"/* "${PREFIX}/"
smart_copy -R "${HOST}/." "${PREFIX}/"
rm "${PREFIX}/lib/libSDL3_test.a"
rm "${PREFIX}/lib/cmake/SDL3/SDL3testTargets"*.cmake
;;
windows-*-msvc)
cd "${dir_name}"
mkdir -p "${PREFIX}/SDL3/cmake"
cp "cmake/"* "${PREFIX}/SDL3/cmake"
mkdir -p "${PREFIX}/SDL3/include/SDL3"
cp "include/SDL3/"* "${PREFIX}/SDL3/include/SDL3"
mkdir -p "${PREFIX}/SDL3"
rm -rf "${PREFIX}/SDL3/cmake"
smart_copy -R "cmake/." "${PREFIX}/SDL3/cmake/"
mkdir -p "${PREFIX}/SDL3/include"
rm -rf "${PREFIX}/SDL3/include/SDL3"
smart_copy -R "include/SDL3/." "${PREFIX}/SDL3/include/SDL3/"

case "${PLATFORM}" in
*-i686-*)
Expand All @@ -479,11 +488,12 @@ build_sdl3() {
esac

mkdir -p "${PREFIX}/SDL3/${sdl3_lib_dir}"
cp "${sdl3_lib_dir}/SDL3.lib" "${PREFIX}/SDL3/${sdl3_lib_dir}"
cp "${sdl3_lib_dir}/"*.dll "${PREFIX}/SDL3/${sdl3_lib_dir}"
smart_copy "${sdl3_lib_dir}/SDL3.lib" "${PREFIX}/SDL3/${sdl3_lib_dir}/"
smart_copy "${sdl3_lib_dir}/"*.dll "${PREFIX}/SDL3/${sdl3_lib_dir}/"
;;
macos-*-*)
cp -R "SDL3.xcframework/macos-arm64_x86_64/SDL3.framework" "${PREFIX}/lib"
rm -rf "${PREFIX}/lib/SDL3.framework"
smart_copy -R "SDL3.xcframework/macos-arm64_x86_64/SDL3.framework/." "${PREFIX}/lib/SDL3.framework/"
;;
*)
cd "${dir_name}"
Expand Down Expand Up @@ -550,7 +560,7 @@ build_glew() {
windows-*-*)
mv "${PREFIX}/lib/glew32.dll" "${PREFIX}/bin/"
rm "${PREFIX}/lib/libglew32.a"
cp lib/libglew32.dll.a "${PREFIX}/lib/"
smart_copy lib/libglew32.dll.a "${PREFIX}/lib/"
;;
macos-*-*)
install_name_tool -id "@rpath/libGLEW.${GLEW_VERSION}.dylib" "${PREFIX}/lib/libGLEW.${GLEW_VERSION}.dylib"
Expand Down Expand Up @@ -762,9 +772,9 @@ build_openal() {
case "${PLATFORM}" in
windows-*-*)
cd "${dir_name}"
cp -r "include/AL" "${PREFIX}/include"
cp "libs/${openal_win_dir}/libOpenAL32.dll.a" "${PREFIX}/lib"
cp "bin/${openal_win_dir}/soft_oal.dll" "${PREFIX}/bin/OpenAL32.dll"
smart_copy -R "include/AL" "${PREFIX}/include"
smart_copy "libs/${openal_win_dir}/libOpenAL32.dll.a" "${PREFIX}/lib"
smart_copy "bin/${openal_win_dir}/soft_oal.dll" "${PREFIX}/bin/OpenAL32.dll"
;;
*)
cd "${dir_name}"
Expand Down Expand Up @@ -897,7 +907,7 @@ build_ncurses() {
cd "${dir_name}"

# Brutally disable writing to database
cp /dev/null misc/run_tic.in
smart_copy /dev/null misc/run_tic.in
# Configure terminfo search dirs based on the ones used in Debian. By default it will only look in (only) the install directory.
configure_build \
--with-strip-program="${STRIP}" \
Expand Down Expand Up @@ -937,7 +947,8 @@ build_wasisdk() {

"${download_only}" && return

cp -r "${dir_name}" "${PREFIX}/wasi-sdk"
rm -rf "${PREFIX}/wasi-sdk"
smart_copy -R "${dir_name}" "${PREFIX}/wasi-sdk"
}

# "Builds" (downloads) wasmtime
Expand Down Expand Up @@ -977,26 +988,23 @@ build_wasmtime() {
"${download_only}" && return

cd "${dir_name}"
cp -r include/* "${PREFIX}/include"
cp -r lib/* "${PREFIX}/lib"
smart_copy -R 'include/.' "${PREFIX}/include/"

@slipher slipher Jul 13, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What's this weird /. syntax?

@illwieckz illwieckz Jul 14, 2026 •
edited
Loading

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It copies all the content of the folder without globbing (and, even if it's not needed there, it likely copies hidden files too, that globbing may misses).

If you copy dir to dir you may end up with dir/dir, while by coyping dir/. to dir/., you make sure only the content get copied.

That's because . is “self”, and doesn't have any name, and one cannot create . in a folder, it's non-ambiguous.

Edit: And dir/./. is dir/. wichi is dir/ so by copying . into dir/. even if you may be copying . into dir/. and get dir/./., then dir/./. is still dir/…

@slipher slipher Jul 14, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK I looked it up and it's a trick to get the same behavior from GNU and BSD cp.

Seems doubtful that we actually want dot files (e.g. Mac .DS_Store), but ones more than one level down would be copied anyway so at least it's consistent?

@illwieckz illwieckz Jul 14, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Seems doubtful that we actually want dot files (e.g. Mac .DS_Store), but ones more than one level down would be copied anyway so at least it's consistent?

Yes, it makes things consistent. If we need to filter-out .DS_Store, we better hunt for them than to rely on implicit behaviors.

smart_copy -R 'lib/.' "${PREFIX}/lib/"
}

# Build the NaCl SDK
build_naclsdk() {
case "${PLATFORM}" in
windows-*-*)
local NACLSDK_PLATFORM=win
local EXE=.exe
local TAR_EXT=cygtar
;;
macos-*-*)
local NACLSDK_PLATFORM=mac
local EXE=
local TAR_EXT=tar
;;
linux-*-*)
local NACLSDK_PLATFORM=linux
local EXE=
local TAR_EXT=tar
;;
esac
Expand All @@ -1022,24 +1030,24 @@ build_naclsdk() {

"${download_only}" && return

cp pepper_*"/tools/irt_core_${NACLSDK_ARCH}.nexe" "${PREFIX}/irt_core-${DAEMON_ARCH}.nexe"
smart_copy pepper_*"/tools/irt_core_${NACLSDK_ARCH}.nexe" "${PREFIX}/irt_core-${DAEMON_ARCH}.nexe"
case "${PLATFORM}" in
linux-amd64-*)
;; # Get sel_ldr from naclruntime package
*)
cp pepper_*"/tools/sel_ldr_${NACLSDK_ARCH}${EXE}" "${PREFIX}/nacl_loader${EXE}"
smart_copy pepper_*"/tools/sel_ldr_${NACLSDK_ARCH}${EXE_EXT}" "${PREFIX}/nacl_loader${EXE_EXT}"
;;
esac
case "${PLATFORM}" in
windows-i686-*|*-amd64-*)
cp pepper_*"/toolchain/${NACLSDK_PLATFORM}_x86_newlib/bin/x86_64-nacl-gdb${EXE}" "${PREFIX}/nacl-gdb${EXE}"
smart_copy pepper_*"/toolchain/${NACLSDK_PLATFORM}_x86_newlib/bin/x86_64-nacl-gdb${EXE_EXT}" "${PREFIX}/nacl-gdb${EXE_EXT}"

rm -rf "${PREFIX}/pnacl"

patch -d pepper_*"/toolchain/${NACLSDK_PLATFORM}_pnacl/bin/pydir" \
-p1 < "${SCRIPT_DIR}/naclsdk-pydir-python3.patch" >/dev/null

cp -a pepper_*"/toolchain/${NACLSDK_PLATFORM}_pnacl" "${PREFIX}/pnacl"
smart_copy -R pepper_*"/toolchain/${NACLSDK_PLATFORM}_pnacl" "${PREFIX}/pnacl"
rm -rf "${PREFIX}/pnacl/bin/"{i686,x86_64}-nacl-*
rm -rf "${PREFIX}/pnacl/arm-nacl"
rm -rf "${PREFIX}/pnacl/arm_bc-nacl"
Expand All @@ -1051,21 +1059,21 @@ build_naclsdk() {
esac
case "${PLATFORM}" in
windows-i686-*)
cp pepper_*"/tools/sel_ldr_x86_64.exe" "${PREFIX}/nacl_loader-amd64.exe"
cp pepper_*"/tools/irt_core_x86_64.nexe" "${PREFIX}/irt_core-amd64.nexe"
smart_copy pepper_*"/tools/sel_ldr_x86_64.exe" "${PREFIX}/nacl_loader-amd64.exe"
smart_copy pepper_*"/tools/irt_core_x86_64.nexe" "${PREFIX}/irt_core-amd64.nexe"
;;
linux-amd64-*)
# Fix permissions on a few files which deny access to non-owner
chmod 644 "${PREFIX}/irt_core-${DAEMON_ARCH}.nexe"
;;
linux-i686-*)
cp pepper_*"/tools/nacl_helper_bootstrap_${NACLSDK_ARCH}" "${PREFIX}/nacl_helper_bootstrap"
smart_copy pepper_*"/tools/nacl_helper_bootstrap_${NACLSDK_ARCH}" "${PREFIX}/nacl_helper_bootstrap"
# Fix permissions on a few files which deny access to non-owner
chmod 644 "${PREFIX}/irt_core-${DAEMON_ARCH}.nexe"
chmod 755 "${PREFIX}/nacl_helper_bootstrap" "${PREFIX}/nacl_loader"
;;
linux-armhf-*|linux-arm64-*)
cp pepper_*"/tools/nacl_helper_bootstrap_arm" "${PREFIX}/nacl_helper_bootstrap"
smart_copy pepper_*"/tools/nacl_helper_bootstrap_arm" "${PREFIX}/nacl_helper_bootstrap"
# Fix permissions on a few files which deny access to non-owner
chmod 644 "${PREFIX}/irt_core-${DAEMON_ARCH}.nexe"
chmod 755 "${PREFIX}/nacl_helper_bootstrap" "${PREFIX}/nacl_loader"
Expand All @@ -1074,10 +1082,10 @@ build_naclsdk() {
case "${PLATFORM}" in
linux-arm64-*)
mkdir -p "${PREFIX}/lib-armhf"
cp -a pepper_*"/tools/lib/arm_trusted/lib/." "${PREFIX}/lib-armhf/."
smart_copy -R pepper_*"/tools/lib/arm_trusted/lib/." "${PREFIX}/lib-armhf/."
# Copy the library loader instead of renaming it because there may still be some
# references to ld-linux-armhf.so.3 in binaries.
cp "${PREFIX}/lib-armhf/ld-linux-armhf.so.3" "${PREFIX}/lib-armhf/ld-linux-armhf"
smart_copy "${PREFIX}/lib-armhf/ld-linux-armhf.so.3" "${PREFIX}/lib-armhf/ld-linux-armhf"
# We can't use patchelf or 'nacl_helper_bootstrap nacl_loader' will complain:
# bootstrap_helper: nacl_loader: ELF file has unreasonable e_phnum=13
sed -e 's|/lib/ld-linux-armhf.so.3|lib-armhf/ld-linux-armhf|' -i "${PREFIX}/nacl_loader"
Expand Down Expand Up @@ -1106,8 +1114,8 @@ build_naclruntime() {

cd "${dir_name}"
env -i /usr/bin/env bash -l -c "python3 /usr/bin/scons --mode=opt-linux 'platform=${NACL_ARCH}' werror=0 sysinfo=0 sel_ldr"
cp "scons-out/opt-linux-${NACL_ARCH}/staging/nacl_helper_bootstrap" "${PREFIX}/nacl_helper_bootstrap"
cp "scons-out/opt-linux-${NACL_ARCH}/staging/sel_ldr" "${PREFIX}/nacl_loader"
smart_copy "scons-out/opt-linux-${NACL_ARCH}/staging/nacl_helper_bootstrap" "${PREFIX}/nacl_helper_bootstrap"
smart_copy "scons-out/opt-linux-${NACL_ARCH}/staging/sel_ldr" "${PREFIX}/nacl_loader"
}

# Check for DLL dependencies on MinGW stuff. For MSVC platforms this is bad because it should work
Expand Down Expand Up @@ -1276,7 +1284,7 @@ build_install() {
build_package() {
cd "${WORK_DIR}"
rm -f "${PKG_BASEDIR}.tar.xz"
local XZ_OPT='-9'
local XZ_OPT='-9e'
case "${PLATFORM}" in
windows-*-*)
tar --dereference -cvJf "${PKG_BASEDIR}.tar.xz" "${PKG_BASEDIR}"
Expand All @@ -1298,6 +1306,7 @@ common_setup() {
"common_setup_${1}"
common_setup_arch

EXE_EXT="${EXE_EXT:-}"
DOWNLOAD_DIR="${WORK_DIR}/download_cache"
PKG_BASEDIR="${PLATFORM}_${DEPS_VERSION}"
BUILD_BASEDIR="build-${PKG_BASEDIR}"
Expand Down Expand Up @@ -1355,6 +1364,7 @@ common_setup_windows() {
RANLIB="${HOST}-ranlib"
CFLAGS+=' -D__USE_MINGW_ANSI_STDIO=0'
CMAKE_TOOLCHAIN="${SCRIPT_DIR}/../cmake/cross-toolchain-mingw${BITNESS}.cmake"
EXE_EXT='.exe'
}

common_setup_msvc() {
Expand Down Expand Up @@ -1435,12 +1445,12 @@ setup_linux-amd64-default() {
common_setup linux x86_64-unknown-linux-gnu
}

# Set up environment for 32-bit armhf Linux
# Set up environment for 32-bit little-endian hard-float arm Linux
setup_linux-armhf-default() {
common_setup linux arm-unknown-linux-gnueabihf
}

# Set up environment for 64-bit arm Linux
# Set up environment for 64-bit little-endian arm Linux
setup_linux-arm64-default() {
common_setup linux aarch64-unknown-linux-gnu
}
Expand Down Expand Up @@ -1553,24 +1563,24 @@ require_theirs='false'
while [ -n "${1:-}" ]
do
case "${1-}" in
'--download-only')
--download-only)
download_only='true'
shift
;;
'--prefer-ours')
;;
--prefer-ours)
prefer_ours='true'
shift
;;
'--require-theirs')
;;
--require-theirs)
require_theirs='true'
shift
;;
'-h'|'--help')
;;
-h|--help)
printHelp
;;
'-'*)
;;
-*)
syntaxError 'Unknown option'
;;
;;
*)
break
esac
Expand Down Expand Up @@ -1600,18 +1610,18 @@ platform="${1}"; shift

platform_list=''
case "${platform}" in
'all')
all)
platform_list="${all_platforms}"
;;
'linux')
;;
linux)
platform_list="${all_linux_platforms}"
;;
'windows')
;;
windows)
platform_list="${all_windows_platforms}"
;;
'macos')
;;
macos)
platform_list="${all_macos_platforms}"
;;
;;
*)
for known_platform in ${all_platforms}
do
Expand All @@ -1625,7 +1635,7 @@ case "${platform}" in
then
syntaxError 'Unknown platform'
fi
;;
;;
esac

for PLATFORM in ${platform_list}
Expand Down
Loading