Re: Support for out-of-tree build

[Petr Hosek <phosek@chromium.org>]

[-- Attachment #1.1: Type: text/plain, Size: 1452 bytes --]

On Tue, Nov 17, 2015 at 4:20 PM Rich Felker <dalias@libc.org> wrote:

> BTW could you refrain from top posting on the list? Even if you just
> have a short reply that's in regards to the whole quoted message
> rather than specific points to address inline, things get awkward to
> follow in a hurry (just like now) when somebody else replies to you
> but also wants to keep the quoted content you were replying to as
> context (like below).
>

Yes, I'm sorry about that, that's the default configuration I use in my
mail client, but I'll avoid it on the list.


> Discussion on irc, along with some basic testing, showed that this
> does not work. The obvious solution with $(subst ...) also does not
> work because it's plausible that $(srcdir) also contains the substring
> /$(ARCH)/, in which case it would get messed up. However I did find a
> form that seems to work reliably and that's not hideous:
>
> $(patsubst %,$(srcdir)/%,$(subst /$(ARCH)/,/,$(patsubst
> $(srcdir)/%,%,$(ARCH_OBJS))))
>
> That yields the list of object files to filter out, which can then be
> passed into $(filter-out ...).
>

I've reworked the patch following our discussion on IRC by placing all
object files into a directory which enables the use of implicit rules. The
rest of the logic is similar to what you proposed as well as to my previous
patches: constructing the list of objects and then filtering out files for
which (sub)architecture versions are available.

[-- Attachment #1.2: Type: text/html, Size: 1929 bytes --]

[-- Attachment #2: support-out-of-tree-build.patch --]
[-- Type: text/x-patch, Size: 13623 bytes --]

From 06192153361a68ccb232fdbb0b491825f70ab792 Mon Sep 17 00:00:00 2001
From: Petr Hosek <phosek@chromium.org>
Date: Wed, 18 Nov 2015 12:07:32 -0800
Subject: [PATCH] support out-of-tree build

this change add support for building musl outside of the source
tree. the implementation is similar to autotools where running
configure in a different folder creates config.mak in the current
working directory and symlinks the makefile, which contains the
logic for creating all necessary directories and resolving paths
relative to the source directory.

to support both in-tree and out-of-tree builds with implicit make
rules, all object files are now placed into a separate directory.
---
 .gitignore |   1 +
 Makefile   | 124 ++++++++++++++++++++++++++++++++++---------------------------
 configure  |  24 ++++++++++--
 3 files changed, 92 insertions(+), 57 deletions(-)

diff --git a/.gitignore b/.gitignore
index c5d5c46..36dc5f9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,6 +3,7 @@
 *.a
 *.so
 *.so.1
+obj
 arch/*/bits/alltypes.h
 config.mak
 include/bits
diff --git a/Makefile b/Makefile
index 2b21015..952cee9 100644
--- a/Makefile
+++ b/Makefile
@@ -8,6 +8,8 @@
 # Do not make changes here.
 #
 
+srcdir = .
+objdir = obj
 exec_prefix = /usr/local
 bindir = $(exec_prefix)/bin
 
@@ -16,12 +18,19 @@ includedir = $(prefix)/include
 libdir = $(prefix)/lib
 syslibdir = /lib
 
-SRCS = $(sort $(wildcard src/*/*.c arch/$(ARCH)/src/*.c))
-OBJS = $(SRCS:.c=.o)
+BASE_SRCS = $(sort $(wildcard $(srcdir)/src/*/*.c $(srcdir)/arch/$(ARCH)/src/*.c))
+BASE_OBJS = $(BASE_SRCS:$(srcdir)/%.c=%.o)
+ARCH_SRCS = $(wildcard $(srcdir)/src/*/$(ARCH)/*.s)
+ARCH_OBJS = $(ARCH_SRCS:$(srcdir)/%.s=%.o)
+SUB_SRCS = $(wildcard $(srcdir)/src/*/$(ARCH)$(ASMSUBARCH)/*.sub)
+SUB_OBJS = $(SUB_SRCS:$(srcdir)/%.sub=%.o)
+EXCLUDE_ARCH_OBJS = $(patsubst $(srcdir)/%,%,$(subst /$(ARCH)/,/,$(patsubst $(srcdir)/%,%,$(ARCH_OBJS))))
+EXCLUDE_SUB_OBJS = $(patsubst $(srcdir)/%,%,$(subst /$(ARCH)$(ASMSUBARCH)/,/,$(patsubst $(srcdir)/%,%,$(SUB_OBJS))))
+OBJS = $(addprefix $(objdir)/, $(filter-out $(EXCLUDE_SUB_OBJS) $(EXCLUDE_ARCH_OBJS), $(BASE_OBJS)) $(ARCH_OBJS) $(SUB_OBJS))
 LOBJS = $(OBJS:.o=.lo)
-GENH = include/bits/alltypes.h
-GENH_INT = src/internal/version.h
-IMPH = src/internal/stdio_impl.h src/internal/pthread_impl.h src/internal/libc.h
+GENH = $(objdir)/include/bits/alltypes.h
+GENH_INT = $(objdir)/src/internal/version.h
+IMPH = $(addprefix $(srcdir)/, src/internal/stdio_impl.h src/internal/pthread_impl.h src/internal/libc.h)
 
 LDFLAGS =
 LDFLAGS_AUTO =
@@ -32,7 +41,7 @@ CFLAGS_AUTO = -Os -pipe
 CFLAGS_C99FSE = -std=c99 -ffreestanding -nostdinc 
 
 CFLAGS_ALL = $(CFLAGS_C99FSE)
-CFLAGS_ALL += -D_XOPEN_SOURCE=700 -I./arch/$(ARCH) -I./src/internal -I./include
+CFLAGS_ALL += -D_XOPEN_SOURCE=700 -I$(srcdir)/arch/$(ARCH) $(sort -I$(objdir)/src/internal -I$(srcdir)/src/internal) $(sort -I$(objdir)/include -I$(srcdir)/include)
 CFLAGS_ALL += $(CPPFLAGS) $(CFLAGS_AUTO) $(CFLAGS)
 CFLAGS_ALL_STATIC = $(CFLAGS_ALL)
 CFLAGS_ALL_SHARED = $(CFLAGS_ALL) -fPIC -DSHARED
@@ -41,10 +50,11 @@ LDFLAGS_ALL = $(LDFLAGS_AUTO) $(LDFLAGS)
 
 AR      = $(CROSS_COMPILE)ar
 RANLIB  = $(CROSS_COMPILE)ranlib
-INSTALL = ./tools/install.sh
+INSTALL = $(srcdir)/tools/install.sh
 
-ARCH_INCLUDES = $(wildcard arch/$(ARCH)/bits/*.h)
-ALL_INCLUDES = $(sort $(wildcard include/*.h include/*/*.h) $(GENH) $(ARCH_INCLUDES:arch/$(ARCH)/%=include/%))
+ARCH_INCLUDES = $(wildcard $(srcdir)/arch/$(ARCH)/bits/*.h)
+INCLUDES = $(wildcard $(srcdir)/include/*.h $(srcdir)/include/*/*.h)
+ALL_INCLUDES = $(sort $(INCLUDES:$(srcdir)/%=%) $(GENH) $(ARCH_INCLUDES:$(srcdir)/arch/$(ARCH)/%=include/%))
 
 EMPTY_LIB_NAMES = m rt pthread crypt util xnet resolv dl
 EMPTY_LIBS = $(EMPTY_LIB_NAMES:%=lib/lib%.a)
@@ -64,10 +74,23 @@ LDSO_PATHNAME = $(syslibdir)/ld-musl-$(ARCH)$(SUBARCH).so.1
 
 all: $(ALL_LIBS) $(ALL_TOOLS)
 
+$(ALL_LIBS): | lib/
+$(ALL_TOOLS): | tools/
+$(CRT_LIBS:lib/%=$(objdir)/crt/%): | $(objdir)/crt/
+$(OBJS) $(LOBJS): | $(sort $(dir $(OBJS)))
+$(GENH): | $(objdir)/arch/$(ARCH)/bits/
+$(GENH_INT): | $(objdir)/src/internal/
+$(objdir)/include/bits: | $(objdir)/include
+
+SRC_DIRS = $(sort $(dir $(ALL_LIBS) $(ALL_TOOLS) $(OBJS) $(GENH_INT)) $(addprefix $(objdir)/, arch/$(ARCH)/bits/ crt/ include/))
+
+$(SRC_DIRS):
+	mkdir -p $@
+
 install: install-libs install-headers install-tools
 
 clean:
-	rm -f crt/*.o
+	rm -f $(objdir)/crt/*.o
 	rm -f $(OBJS)
 	rm -f $(LOBJS)
 	rm -f $(ALL_LIBS) lib/*.[ao] lib/*.so
@@ -78,75 +101,65 @@ clean:
 distclean: clean
 	rm -f config.mak
 
-include/bits:
+$(objdir)/include/bits:
 	@test "$(ARCH)" || { echo "Please set ARCH in config.mak before running make." ; exit 1 ; }
 	ln -sf ../arch/$(ARCH)/bits $@
 
-include/bits/alltypes.h.in: include/bits
-
-include/bits/alltypes.h: include/bits/alltypes.h.in include/alltypes.h.in tools/mkalltypes.sed
-	sed -f tools/mkalltypes.sed include/bits/alltypes.h.in include/alltypes.h.in > $@
+$(objdir)/include/bits/alltypes.h: $(srcdir)/arch/$(ARCH)/bits/alltypes.h.in $(srcdir)/include/alltypes.h.in $(srcdir)/tools/mkalltypes.sed $(objdir)/include/bits
+	sed -f $(srcdir)/tools/mkalltypes.sed $(srcdir)/arch/$(ARCH)/bits/alltypes.h.in $(srcdir)/include/alltypes.h.in > $@
 
-src/internal/version.h: $(wildcard VERSION .git)
-	printf '#define VERSION "%s"\n' "$$(sh tools/version.sh)" > $@
+$(objdir)/src/internal/version.h: $(wildcard $(srcdir)/VERSION $(srcdir)/.git)
+	printf '#define VERSION "%s"\n' "$$(cd $(srcdir); sh tools/version.sh)" > $@
 
-src/internal/version.lo: src/internal/version.h
+$(objdir)/src/internal/version.lo: $(objdir)/src/internal/version.h
 
-crt/rcrt1.o src/ldso/dlstart.lo src/ldso/dynlink.lo: src/internal/dynlink.h arch/$(ARCH)/reloc.h
+$(objdir)/crt/rcrt1.o $(objdir)/src/ldso/dlstart.lo $(objdir)/src/ldso/dynlink.lo: $(srcdir)/src/internal/dynlink.h $(srcdir)/arch/$(ARCH)/reloc.h
 
-crt/crt1.o crt/Scrt1.o crt/rcrt1.o src/ldso/dlstart.lo: $(wildcard arch/$(ARCH)/crt_arch.h)
+$(objdir)/crt/crt1.o $(objdir)/crt/scrt1.o $(objdir)/crt/rcrt1.o $(objdir)/src/ldso/dlstart.lo: $(srcdir)/arch/$(ARCH)/crt_arch.h
 
-crt/rcrt1.o: src/ldso/dlstart.c
+$(objdir)/crt/rcrt1.o: $(srcdir)/src/ldso/dlstart.c
 
-crt/Scrt1.o crt/rcrt1.o: CFLAGS_ALL += -fPIC
+$(objdir)/crt/Scrt1.o $(objdir)/crt/rcrt1.o: CFLAGS_ALL += -fPIC
 
-OPTIMIZE_SRCS = $(wildcard $(OPTIMIZE_GLOBS:%=src/%))
-$(OPTIMIZE_SRCS:%.c=%.o) $(OPTIMIZE_SRCS:%.c=%.lo): CFLAGS += -O3
+OPTIMIZE_SRCS = $(wildcard $(OPTIMIZE_GLOBS:%=$(srcdir)/src/%))
+$(OPTIMIZE_SRCS:$(srcdir)/%.c=$(objdir)/%.o) $(OPTIMIZE_SRCS:$(srcdir)/%.c=$(objdir)/%.lo): CFLAGS += -O3
 
 MEMOPS_SRCS = src/string/memcpy.c src/string/memmove.c src/string/memcmp.c src/string/memset.c
-$(MEMOPS_SRCS:%.c=%.o) $(MEMOPS_SRCS:%.c=%.lo): CFLAGS_ALL += $(CFLAGS_MEMOPS)
+$(MEMOPS_SRCS:%.c=$(objdir)/%.o) $(MEMOPS_SRCS:%.c=$(objdir)/%.lo): CFLAGS_ALL += $(CFLAGS_MEMOPS)
 
 NOSSP_SRCS = $(wildcard crt/*.c) \
-	src/env/__libc_start_main.c src/env/__init_tls.c \
-	src/thread/__set_thread_area.c src/env/__stack_chk_fail.c \
-	src/string/memset.c src/string/memcpy.c \
-	src/ldso/dlstart.c src/ldso/dynlink.c
-$(NOSSP_SRCS:%.c=%.o) $(NOSSP_SRCS:%.c=%.lo): CFLAGS_ALL += $(CFLAGS_NOSSP)
-
-$(CRT_LIBS:lib/%=crt/%): CFLAGS_ALL += -DCRT
-
-# This incantation ensures that changes to any subarch asm files will
-# force the corresponding object file to be rebuilt, even if the implicit
-# rule below goes indirectly through a .sub file.
-define mkasmdep
-$(dir $(patsubst %/,%,$(dir $(1))))$(notdir $(1:.s=.o)): $(1)
-endef
-$(foreach s,$(wildcard src/*/$(ARCH)*/*.s),$(eval $(call mkasmdep,$(s))))
+	$(srcdir)/src/env/__libc_start_main.c $(srcdir)/src/env/__init_tls.c \
+	$(srcdir)/src/thread/__set_thread_area.c $(srcdir)/src/env/__stack_chk_fail.c \
+	$(srcdir)/src/string/memset.c $(srcdir)/src/string/memcpy.c \
+	$(srcdir)/src/ldso/dlstart.c $(srcdir)/src/ldso/dynlink.c
+$(NOSSP_SRCS:$(srcdir)/%.c=$(objdir)/%.o) $(NOSSP_SRCS:$(srcdir)/%.c=$(objdir)/%.lo): CFLAGS_ALL += $(CFLAGS_NOSSP)
+
+$(CRT_LIBS:lib/%=$(objdir)/crt/%): CFLAGS_ALL += -DCRT
 
 # Choose invocation of assembler to be used
 # $(1) is input file, $(2) is output file, $(3) is assembler flags
 ifeq ($(ADD_CFI),yes)
-	AS_CMD = LC_ALL=C awk -f tools/add-cfi.common.awk -f tools/add-cfi.$(ARCH).awk $< | $(CC) -x assembler -c -o $@ -
+	AS_CMD = LC_ALL=C awk -f $(srcdir)/tools/add-cfi.common.awk -f $(srcdir)/tools/add-cfi.$(ARCH).awk $< | $(CC) -x assembler -c -o $@ -
 else
 	AS_CMD = $(CC) -c -o $@ $<
 endif
 
-%.o: $(ARCH)$(ASMSUBARCH)/%.sub
+$(objdir)/%.o: $(srcdir)/%.sub
 	$(CC) $(CFLAGS_ALL_STATIC) -c -o $@ $(dir $<)$(shell cat $<)
 
-%.o: $(ARCH)/%.s
+$(objdir)/%.o: $(srcdir)/%.s
 	$(AS_CMD) $(CFLAGS_ALL_STATIC)
 
-%.o: %.c $(GENH) $(IMPH)
+$(objdir)/%.o: $(srcdir)/%.c $(GENH) $(IMPH)
 	$(CC) $(CFLAGS_ALL_STATIC) -c -o $@ $<
 
-%.lo: $(ARCH)$(ASMSUBARCH)/%.sub
-	$(CC) $(CFLAGS_ALL_SHARED) -c -o $@ $(dir $<)$(shell cat $<)
+$(objdir)/%.lo: $(srcdir)/%.sub
+	 $(CC) $(CFLAGS_ALL_SHARED) -c -o $@ $(dir $<)$(shell cat $<)
 
-%.lo: $(ARCH)/%.s
+$(objdir)/%.lo: $(srcdir)/%.s
 	$(AS_CMD) $(CFLAGS_ALL_SHARED)
 
-%.lo: %.c $(GENH) $(IMPH)
+$(objdir)/%.lo: $(srcdir)/%.c $(GENH) $(IMPH)
 	$(CC) $(CFLAGS_ALL_SHARED) -c -o $@ $<
 
 lib/libc.so: $(LOBJS)
@@ -163,17 +176,17 @@ $(EMPTY_LIBS):
 	rm -f $@
 	$(AR) rc $@
 
-lib/%.o: crt/%.o
+lib/%.o: $(objdir)/crt/%.o
 	cp $< $@
 
-lib/musl-gcc.specs: tools/musl-gcc.specs.sh config.mak
+lib/musl-gcc.specs: $(srcdir)/tools/musl-gcc.specs.sh config.mak
 	sh $< "$(includedir)" "$(libdir)" "$(LDSO_PATHNAME)" > $@
 
 tools/musl-gcc: config.mak
 	printf '#!/bin/sh\nexec "$${REALGCC:-$(WRAPCC_GCC)}" "$$@" -specs "%s/musl-gcc.specs"\n' "$(libdir)" > $@
 	chmod +x $@
 
-tools/%-clang: tools/%-clang.in config.mak
+tools/%-clang: $(srcdir)/tools/%-clang.in config.mak
 	sed -e 's!@CC@!$(WRAPCC_CLANG)!g' -e 's!@PREFIX@!$(prefix)!g' -e 's!@INCDIR@!$(includedir)!g' -e 's!@LIBDIR@!$(libdir)!g' -e 's!@LDSO@!$(LDSO_PATHNAME)!g' $< > $@
 	chmod +x $@
 
@@ -186,10 +199,13 @@ $(DESTDIR)$(libdir)/%.so: lib/%.so
 $(DESTDIR)$(libdir)/%: lib/%
 	$(INSTALL) -D -m 644 $< $@
 
+$(DESTDIR)$(includedir)/bits/%: $(srcdir)/arch/$(ARCH)/bits/%
+	$(INSTALL) -D -m 644 $< $@
+
 $(DESTDIR)$(includedir)/bits/%: arch/$(ARCH)/bits/%
 	$(INSTALL) -D -m 644 $< $@
 
-$(DESTDIR)$(includedir)/%: include/%
+$(DESTDIR)$(includedir)/%: $(srcdir)/include/%
 	$(INSTALL) -D -m 644 $< $@
 
 $(DESTDIR)$(LDSO_PATHNAME): $(DESTDIR)$(libdir)/libc.so
@@ -202,9 +218,9 @@ install-headers: $(ALL_INCLUDES:include/%=$(DESTDIR)$(includedir)/%)
 install-tools: $(ALL_TOOLS:tools/%=$(DESTDIR)$(bindir)/%)
 
 musl-git-%.tar.gz: .git
-	 git archive --format=tar.gz --prefix=$(patsubst %.tar.gz,%,$@)/ -o $@ $(patsubst musl-git-%.tar.gz,%,$@)
+	 git --git-dir=$(srcdir)/.git archive --format=tar.gz --prefix=$(patsubst %.tar.gz,%,$@)/ -o $@ $(patsubst musl-git-%.tar.gz,%,$@)
 
 musl-%.tar.gz: .git
-	 git archive --format=tar.gz --prefix=$(patsubst %.tar.gz,%,$@)/ -o $@ v$(patsubst musl-%.tar.gz,%,$@)
+	 git --git-dir=$(srcdir)/.git archive --format=tar.gz --prefix=$(patsubst %.tar.gz,%,$@)/ -o $@ v$(patsubst musl-%.tar.gz,%,$@)
 
 .PHONY: all clean install install-libs install-headers install-tools
diff --git a/configure b/configure
index dece1d0..60eb94f 100755
--- a/configure
+++ b/configure
@@ -9,6 +9,9 @@ VAR=VALUE.  See below for descriptions of some of the useful variables.
 
 Defaults for the options are specified in brackets.
 
+Configuration:
+  --srcdir=DIR            source directory [detected]
+
 Installation directories:
   --prefix=PREFIX         main installation prefix [/usr/local/musl]
   --exec-prefix=EPREFIX   installation prefix for executable files [PREFIX]
@@ -117,6 +120,7 @@ CFLAGS_TRY=
 LDFLAGS_AUTO=
 LDFLAGS_TRY=
 OPTIMIZE_GLOBS=
+srcdir=
 prefix=/usr/local/musl
 exec_prefix='$(prefix)'
 bindir='$(exec_prefix)/bin'
@@ -139,6 +143,7 @@ clang_wrapper=no
 for arg ; do
 case "$arg" in
 --help) usage ;;
+--srcdir=*) srcdir=${arg#*=} ;;
 --prefix=*) prefix=${arg#*=} ;;
 --exec-prefix=*) exec_prefix=${arg#*=} ;;
 --bindir=*) bindir=${arg#*=} ;;
@@ -179,11 +184,23 @@ LIBCC=*) LIBCC=${arg#*=} ;;
 esac
 done
 
-for i in prefix exec_prefix bindir libdir includedir syslibdir ; do
+for i in srcdir prefix exec_prefix bindir libdir includedir syslibdir ; do
 stripdir $i
 done
 
 #
+# Get the musl source dir for out-of-tree builds
+#
+if test -z "$srcdir" ; then
+srcdir="${0%/configure}"
+stripdir srcdir
+fi
+abs_builddir="$(pwd)" || fail "$0: cannot determine working directory"
+abs_srcdir="$(cd $srcdir && pwd)" || fail "$0: invalid source directory $srcdir"
+test "$abs_srcdir" = "$abs_builddir" && srcdir=.
+ln -sf $srcdir/Makefile .
+
+#
 # Get a temp filename we can use
 #
 i=0
@@ -321,7 +338,7 @@ __attribute__((__may_alias__))
 #endif
 x;
 EOF
-if $CC $CFLAGS_C99FSE -I./arch/$ARCH -I./include $CPPFLAGS $CFLAGS \
+if $CC $CFLAGS_C99FSE -I$srcdir/arch/$ARCH -I$srcdir/include $CPPFLAGS $CFLAGS \
   -c -o /dev/null "$tmpc" >/dev/null 2>&1 ; then
 printf "no\n"
 else
@@ -620,7 +637,7 @@ echo '#include <float.h>' > "$tmpc"
 echo '#if LDBL_MANT_DIG == 53' >> "$tmpc"
 echo 'typedef char ldcheck[9-(int)sizeof(long double)];' >> "$tmpc"
 echo '#endif' >> "$tmpc"
-if $CC $CFLAGS_C99FSE -I./arch/$ARCH -I./include $CPPFLAGS $CFLAGS \
+if $CC $CFLAGS_C99FSE -I$srcdir/arch/$ARCH -I$srcdir/include $CPPFLAGS $CFLAGS \
   -c -o /dev/null "$tmpc" >/dev/null 2>&1 ; then
 printf "yes\n"
 else
@@ -643,6 +660,7 @@ cat << EOF
 ARCH = $ARCH
 SUBARCH = $SUBARCH
 ASMSUBARCH = $ASMSUBARCH
+srcdir = $srcdir
 prefix = $prefix
 exec_prefix = $exec_prefix
 bindir = $bindir
-- 
2.6.0.rc2.230.g3dd15c0