Re: Patch to allow configure out-with source directory

[Szabolcs Nagy <nsz@port70.net>]

[-- Attachment #1: Type: text/plain, Size: 2351 bytes --]

* Rich Felker <dalias@libc.org> [2015-03-04 12:45:55 -0500]:
> On Wed, Mar 04, 2015 at 05:52:30PM +0100, Laurent Bercot wrote:
> > to study a small and clean case - and was considering implementing
> > out-of-tree builds at some point in the (distant) future. If there
> > were some notes describing the obstacles you faced, it would help
> > me when I get to it - or more likely convince me, like it
> > apparently convinced you, that it's just not worth it.
> 
> The first problem you face is how the implicit rules are written,
> especially the ones for asm files that replace C files. They depend on
> make's implicit application of leading directories. But I'm pretty
> sure nsz had this worked out clearly and it was more subtle corner
> cases that kept biting. He could probably explain better.
> 

i couldnt fix the implicit rules issue
http://www.openwall.com/lists/musl/2014/02/06/6

i worked it around with vpath:
http://www.openwall.com/lists/musl/2014/02/08/1

> Another issue I recall is the presence of generated include files and
> the bits symlink in the include tree. This is part of the build
> process I want to revisit/change anyway when refactoring the bits
> headers to reduce duplication, so understanding the obstacles to
> out-of-tree builds before getting started with that would be a big
> help.

the vpath solution cannot work in all cases:

it magically adds the srcdir prefix to the right side
of make rules (if otherwise the file is not found)

but if a right side is generated and you have an in-tree
build then the out-of-tree build will find the generated
file in-tree and use that instead of rebuilding it

(of course this only matters if one mixes in-tree and
out-of-tree builds, which is not a good idea in general
so we could warn instead of fixing it)

if you could apply vpath selectively only to non-generated
source files that would fix the issue, but i havent found
any way to do that

normally explicit $(srcdir) prefixing would be the clean
solution but the current implicit rules in musl don't work
with that:

%.o: $(ARCH)/%.s

can expand to "src/foo/bar.o: src/foo/i386/bar.s", but

%.o: $(srcdir)/???/$(ARCH)/%.s

does not work

other approaches are possible: eg. generate the .o rules
with foreach, generate the entire Makefile, etc
i havent tried these

i attach the vpath version that mostly works

[-- Attachment #2: out-of-tree.diff --]
[-- Type: text/x-diff, Size: 6469 bytes --]

diff --git a/Makefile b/Makefile
index 8cc3de8..5adbe1a 100644
--- a/Makefile
+++ b/Makefile
@@ -10,14 +10,16 @@
 
 exec_prefix = /usr/local
 bindir = $(exec_prefix)/bin
+srcdir = .
 
 prefix = /usr/local/musl
 includedir = $(prefix)/include
 libdir = $(prefix)/lib
 syslibdir = /lib
 
-SRCS = $(sort $(wildcard src/*/*.c arch/$(ARCH)/src/*.c))
-OBJS = $(SRCS:.c=.o)
+M = $(srcdir)
+SRCS = $(sort $(wildcard $(M)/src/*/*.c $(M)/arch/$(ARCH)/src/*.c))
+OBJS = $(SRCS:$(M)/%.c=%.o)
 LOBJS = $(OBJS:.o=.lo)
 GENH = include/bits/alltypes.h
 GENH_INT = src/internal/version.h
@@ -30,17 +32,18 @@ CFLAGS = -Os -pipe
 CFLAGS_C99FSE = -std=c99 -ffreestanding -nostdinc 
 
 CFLAGS_ALL = $(CFLAGS_C99FSE)
-CFLAGS_ALL += -D_XOPEN_SOURCE=700 -I./arch/$(ARCH) -I./src/internal -I./include
+CFLAGS_ALL += -D_XOPEN_SOURCE=700 -I$(M)/arch/$(ARCH) -I./src/internal -I$(M)/src/internal -I./include -I$(M)/include
 CFLAGS_ALL += $(CPPFLAGS) $(CFLAGS)
 CFLAGS_ALL_STATIC = $(CFLAGS_ALL)
 CFLAGS_ALL_SHARED = $(CFLAGS_ALL) -fPIC -DSHARED
 
 AR      = $(CROSS_COMPILE)ar
 RANLIB  = $(CROSS_COMPILE)ranlib
-INSTALL = ./tools/install.sh
+INSTALL = $(M)/tools/install.sh
 
-ARCH_INCLUDES = $(wildcard arch/$(ARCH)/bits/*.h)
-ALL_INCLUDES = $(sort $(wildcard include/*.h include/*/*.h) $(GENH) $(ARCH_INCLUDES:arch/$(ARCH)/%=include/%))
+ARCH_INCLUDES = $(wildcard $(M)/arch/$(ARCH)/bits/*.h)
+INCLUDES = $(wildcard $(M)/include/*.h $(M)/include/*/*.h)
+ALL_INCLUDES = $(sort $(INCLUDES:$(M)/%=%) $(GENH) $(ARCH_INCLUDES:$(M)/arch/$(ARCH)/%=include/%))
 
 EMPTY_LIB_NAMES = m rt pthread crypt util xnet resolv dl
 EMPTY_LIBS = $(EMPTY_LIB_NAMES:%=lib/lib%.a)
@@ -57,6 +60,18 @@ LDSO_PATHNAME = $(syslibdir)/ld-musl-$(ARCH)$(SUBARCH).so.1
 
 all: $(ALL_LIBS) $(ALL_TOOLS)
 
+vpath %.s $(M)
+vpath %.c $(M)
+vpath %.h $(M)
+
+$(ALL_TOOLS): tools/.mark
+$(ALL_LIBS): lib/.mark
+$(CRT_LIBS:lib/%=crt/%): crt/.mark
+$(OBJS) $(LOBJS): $(patsubst %/,%/.mark,$(sort $(dir $(OBJS))))
+%/.mark:
+	mkdir -p $*
+	touch $@
+
 install: install-libs install-headers install-tools
 
 clean:
@@ -71,28 +86,27 @@ clean:
 distclean: clean
 	rm -f config.mak
 
-include/bits:
+include/bits: include/.mark arch/$(ARCH)/bits/.mark
 	@test "$(ARCH)" || { echo "Please set ARCH in config.mak before running make." ; exit 1 ; }
 	ln -sf ../arch/$(ARCH)/bits $@
 
-include/bits/alltypes.h.in: include/bits
-
-include/bits/alltypes.h: include/bits/alltypes.h.in include/alltypes.h.in tools/mkalltypes.sed
-	sed -f tools/mkalltypes.sed include/bits/alltypes.h.in include/alltypes.h.in > $@
+INS = $(M)/arch/$(ARCH)/bits/alltypes.h.in $(M)/include/alltypes.h.in
+include/bits/alltypes.h: $(M)/tools/mkalltypes.sed $(INS) include/bits
+	sed -f $(M)/tools/mkalltypes.sed $(INS) > $@
 
-src/internal/version.h: $(wildcard VERSION .git)
-	printf '#define VERSION "%s"\n' "$$(sh tools/version.sh)" > $@
+src/internal/version.h: $(wildcard $(M)/VERSION $(M)/.git) src/internal/.mark
+	printf '#define VERSION "%s"\n' "$$(cd $(M); sh tools/version.sh)" > $@
 
 src/internal/version.lo: src/internal/version.h
 
 src/ldso/dynlink.lo: arch/$(ARCH)/reloc.h
 
-crt/crt1.o crt/Scrt1.o: $(wildcard arch/$(ARCH)/crt_arch.h)
+crt/crt1.o crt/Scrt1.o: $(wildcard $(M)/arch/$(ARCH)/crt_arch.h)
 
 crt/Scrt1.o: CFLAGS += -fPIC
 
-OPTIMIZE_SRCS = $(wildcard $(OPTIMIZE_GLOBS:%=src/%))
-$(OPTIMIZE_SRCS:%.c=%.o) $(OPTIMIZE_SRCS:%.c=%.lo): CFLAGS += -O3
+OPTIMIZE_SRCS = $(wildcard $(OPTIMIZE_GLOBS:%=$(M)/src/%))
+$(OPTIMIZE_SRCS:$(M)/%.c=%.o) $(OPTIMIZE_SRCS:$(M)/%.c=%.lo): CFLAGS += -O3
 
 MEMOPS_SRCS = src/string/memcpy.c src/string/memmove.c src/string/memcmp.c src/string/memset.c
 $(MEMOPS_SRCS:%.c=%.o) $(MEMOPS_SRCS:%.c=%.lo): CFLAGS += $(CFLAGS_MEMOPS)
@@ -101,9 +115,9 @@ $(MEMOPS_SRCS:%.c=%.o) $(MEMOPS_SRCS:%.c=%.lo): CFLAGS += $(CFLAGS_MEMOPS)
 # force the corresponding object file to be rebuilt, even if the implicit
 # rule below goes indirectly through a .sub file.
 define mkasmdep
-$(dir $(patsubst %/,%,$(dir $(1))))$(notdir $(1:.s=.o)): $(1)
+$(dir $(patsubst $(M)/%/,%,$(dir $(1))))$(notdir $(1:.s=.o)): $(1)
 endef
-$(foreach s,$(wildcard src/*/$(ARCH)*/*.s),$(eval $(call mkasmdep,$(s))))
+$(foreach s,$(wildcard $(M)/src/*/$(ARCH)*/*.s),$(eval $(call mkasmdep,$(s))))
 
 %.o: $(ARCH)$(ASMSUBARCH)/%.sub
 	$(CC) $(CFLAGS_ALL_STATIC) -c -o $@ $(dir $<)$(shell cat $<)
@@ -140,7 +154,7 @@ $(EMPTY_LIBS):
 lib/%.o: crt/%.o
 	cp $< $@
 
-lib/musl-gcc.specs: tools/musl-gcc.specs.sh config.mak
+lib/musl-gcc.specs: $(M)/tools/musl-gcc.specs.sh config.mak
 	sh $< "$(includedir)" "$(libdir)" "$(LDSO_PATHNAME)" > $@
 
 tools/musl-gcc: config.mak
@@ -156,7 +170,7 @@ $(DESTDIR)$(libdir)/%.so: lib/%.so
 $(DESTDIR)$(libdir)/%: lib/%
 	$(INSTALL) -D -m 644 $< $@
 
-$(DESTDIR)$(includedir)/bits/%: arch/$(ARCH)/bits/%
+$(DESTDIR)$(includedir)/bits/%: $(M)/arch/$(ARCH)/bits/%
 	$(INSTALL) -D -m 644 $< $@
 
 $(DESTDIR)$(includedir)/%: include/%
diff --git a/configure b/configure
index e80fdfb..76fdf5f 100755
--- a/configure
+++ b/configure
@@ -168,6 +168,23 @@ stripdir $i
 done
 
 #
+# Get the musl source dir for out-of-tree builds
+#
+test -z "$srcdir" && srcdir="${0%/configure}"
+stripdir srcdir
+abs_builddir="$(pwd)" || fail "$0: cannot determine current directory"
+abs_srcdir="$(cd $srcdir && pwd)" || fail "$0: invalid source directory $srcdir"
+test "$abs_srcdir" = "$abs_builddir" && srcdir=.
+echo "source directory: $srcdir"
+if test -f Makefile ; then
+echo "using existing Makefile"
+else
+printf "creating Makefile... "
+echo "include $srcdir/Makefile" >Makefile
+echo "done"
+fi
+
+#
 # Get a temp filename we can use
 #
 i=0
@@ -281,7 +298,7 @@ __attribute__((__may_alias__))
 #endif
 x;
 EOF
-if $CC $CFLAGS_C99FSE -I./arch/$ARCH -I./include $CPPFLAGS $CFLAGS \
+if $CC $CFLAGS_C99FSE -I$srcdir/arch/$ARCH -I$srcdir/include $CPPFLAGS $CFLAGS \
   -c -o /dev/null "$tmpc" >/dev/null 2>&1 ; then
 printf "no\n"
 else
@@ -488,7 +505,7 @@ echo '#include <float.h>' > "$tmpc"
 echo '#if LDBL_MANT_DIG == 53' >> "$tmpc"
 echo 'typedef char ldcheck[9-(int)sizeof(long double)];' >> "$tmpc"
 echo '#endif' >> "$tmpc"
-if $CC $CFLAGS_C99FSE -I./arch/$ARCH -I./include $CPPFLAGS $CFLAGS \
+if $CC $CFLAGS_C99FSE -I$srcdir/arch/$ARCH -I$srcdir/include $CPPFLAGS $CFLAGS \
   -c -o /dev/null "$tmpc" >/dev/null 2>&1 ; then
 printf "yes\n"
 else
@@ -511,6 +528,7 @@ cat << EOF
 ARCH = $ARCH
 SUBARCH = $SUBARCH
 ASMSUBARCH = $ASMSUBARCH
+srcdir = $srcdir
 prefix = $prefix
 exec_prefix = $exec_prefix
 bindir = $bindir