RE: Queries with less than `ndots` dots never lead to resolution using the global namespace if the `search` domains don't work

[<dominic.chambers@glencore.com>]

[-- Attachment #1: Type: text/plain, Size: 4279 bytes --]

> Yes, this works. Here's the problem:
> 
> > execve("./gai3a", ["./gai3a", "google.com"], [/* 15 vars */]) = 0 
> > [...] socket(AF_INET, SOCK_DGRAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 
> > IPPROTO_IP) = 3 bind(3, {sa_family=AF_INET, sin_port=htons(0), 
> > sin_addr=inet_addr("0.0.0.0")}, 16) = 0 sendto(3, 
> > "\271\377\1\0\0\1\0\0\0\0\0\0\6google\3com\7default\3"..., 54, 
> > MSG_NOSIGNAL, {sa_family=AF_INET, sin_port=htons(53), 
> > sin_addr=inet_addr("10.43.0.10")}, 16) = 54 sendto(3, 
> > "\341\262\1\0\0\1\0\0\0\0\0\0\6google\3com\7default\3"..., 54, 
> > MSG_NOSIGNAL, {sa_family=AF_INET, sin_port=htons(53), 
> > sin_addr=inet_addr("10.43.0.10")}, 16) = 54 poll([{fd=3, 
> > events=POLLIN}], 1, 2500)  = 1 ([{fd=3, revents=POLLIN}]) recvfrom(3, 
> > "\271\377\201\203\0\1\0\0\0\1\0\0\6google\3com\7default\3"..., 512, 0, 
> > {sa_family=AF_INET, sin_port=htons(53), 
> > sin_addr=inet_addr("10.43.0.10")}, [16]) = 147 recvfrom(3, 
> > "\341\262\201\203\0\1\0\0\0\1\0\0\6google\3com\7default\3"..., 512, 0, 
> > {sa_family=AF_INET, sin_port=htons(53), 
> > sin_addr=inet_addr("10.43.0.10")}, [16]) = 147
>                         ^^^^^^^
> 
> This correctly returns NxDomain (the \203 byte). But:
> 
> > [...]
> > socket(AF_INET, SOCK_DGRAM|SOCK_CLOEXEC|SOCK_NONBLOCK, IPPROTO_IP) = 3 
> > bind(3, {sa_family=AF_INET, sin_port=htons(0), 
> > sin_addr=inet_addr("0.0.0.0")}, 16) = 0 sendto(3, 
> > "H\244\1\0\0\1\0\0\0\0\0\0\6google\3com\7kubelet\n"..., 64, 
> > MSG_NOSIGNAL, {sa_family=AF_INET, sin_port=htons(53), 
> > sin_addr=inet_addr("10.43.0.10")}, 16) = 64 sendto(3, 
> > "K\266\1\0\0\1\0\0\0\0\0\0\6google\3com\7kubelet\n"..., 64, 
> > MSG_NOSIGNAL, {sa_family=AF_INET, sin_port=htons(53), 
> > sin_addr=inet_addr("10.43.0.10")}, 16) = 64 poll([{fd=3, 
> > events=POLLIN}], 1, 2500)  = 1 ([{fd=3, revents=POLLIN}]) recvfrom(3, 
> > "H\244\205\200\0\1\0\0\0\1\0\0\6google\3com\7kubelet\n"..., 512, 0, 
> > {sa_family=AF_INET, sin_port=htons(53), 
> > sin_addr=inet_addr("10.43.0.10")}, [16]) = 100 recvfrom(3, 
> > 0x7fff9c3f83c0, 512, 0, 0x7fff9c3f7e70, [16]) = -1 EAGAIN (Resource 
> > temporarily unavailable) poll([{fd=3, events=POLLIN}], 1, 2498)  = 1 
> > ([{fd=3, revents=POLLIN}]) recvfrom(3, 
> > "K\266\205\200\0\1\0\0\0\1\0\0\6google\3com\7kubelet\n"..., 512, 0, 
> > {sa_family=AF_INET, sin_port=htons(53), 
> > sin_addr=inet_addr("10.43.0.10")}, [16]) = 100
>                     ^^^^^^^^
> 
> Here we see your nameserver returning a RCODE=0 (Success, the \200
> byte) reply for google.com.kubelet[...] rather than NxDomain. Sorry I don't have the full name; you need to pass a larger -s to strace to get it not to truncate strings. You need to figure out why the nameserver is claiming this exists; it might be some sort of wildcard record or just a buggy nameserver (probably some component of kubernetes).
> 
> Rich

By preventing the trace from being truncated (see attached) it appears that this only occurs when querying names for which Rancher's DNS is authoritative, and is not happening for names for which Kubernetes' DNS is authoritative.

FYI, this is how `search` was defined in `resolv.conf`:

```
search default.svc.cluster.local svc.cluster.local cluster.local kubelet.kubernetes.rancher.internal kubernetes.rancher.internal rancher.internal
```

Where `default.svc.cluster.local`, `svc.cluster.local` and `cluster.local` are for service discovery in Kubernetes and `kubelet.kubernetes.rancher.internal`, `kubernetes.rancher.internal` and `rancher.internal` are (I believe) something to do with rancher-dns.

This would explain why only some people have continued having problems since Alpine 3.4 was released with the `search` and `domain` support contained within musl libc.

Raising a bug with Rancher now. Thanks so much for your help here!
LEGAL DISCLAIMER. The contents of this electronic communication
and any attached documents are strictly confidential and they may not
be used or disclosed by someone who is not a named recipient.
If you have received this electronic communication in error please notify
the sender by replying to this electronic communication inserting the
word "misdirected" as the subject and delete this communication from
your system.

[-- Attachment #2: strace_of_google-com_non-truncated.txt --]
[-- Type: text/plain, Size: 7033 bytes --]

execve("./gai3a", ["./gai3a", "google.com"], [/* 15 vars */]) = 0
arch_prctl(ARCH_SET_FS, 0x7fbaf4db7b48) = 0
set_tid_address(0x7fbaf4db7b80)         = 49
mprotect(0x7fbaf4db4000, 4096, PROT_READ) = 0
mprotect(0x560558d0f000, 4096, PROT_READ) = 0
open("/etc/hosts", O_RDONLY|O_CLOEXEC)  = 3
fcntl(3, F_SETFD, FD_CLOEXEC)           = 0
readv(3, [{iov_base="", iov_len=0}, {iov_base="# Kubernetes-managed hosts file.\n127.0.0.1\tlocalhost\n::1\tlocalhost ip6-localhost ip6-loopback\nfe00::0\tip6-localnet\nfe00::0\tip6-m"..., iov_len=1024}], 2) = 202
readv(3, [{iov_base="", iov_len=0}, {iov_base="", iov_len=1024}], 2) = 0
close(3)                                = 0
open("/etc/resolv.conf", O_RDONLY|O_CLOEXEC) = 3
fcntl(3, F_SETFD, FD_CLOEXEC)           = 0
readv(3, [{iov_base="", iov_len=0}, {iov_base="search default.svc.cluster.local svc.cluster.local cluster.local kubelet.kubernetes.rancher.internal kubernetes.rancher.internal"..., iov_len=248}], 2) = 218
readv(3, [{iov_base="", iov_len=0}, {iov_base="", iov_len=248}], 2) = 0
close(3)                                = 0
socket(AF_INET, SOCK_DGRAM|SOCK_CLOEXEC|SOCK_NONBLOCK, IPPROTO_IP) = 3
bind(3, {sa_family=AF_INET, sin_port=htons(0), sin_addr=inet_addr("0.0.0.0")}, 16) = 0
sendto(3, "\307\240\1\0\0\1\0\0\0\0\0\0\6google\3com\7default\3svc\7cluster\5local\0\0\1\0\1", 54, MSG_NOSIGNAL, {sa_family=AF_INET, sin_port=htons(53), sin_addr=inet_addr("10.43.0.10")}, 16) = 54
sendto(3, "\342\253\1\0\0\1\0\0\0\0\0\0\6google\3com\7default\3svc\7cluster\5local\0\0\34\0\1", 54, MSG_NOSIGNAL, {sa_family=AF_INET, sin_port=htons(53), sin_addr=inet_addr("10.43.0.10")}, 16) = 54
poll([{fd=3, events=POLLIN}], 1, 2500)  = 1 ([{fd=3, revents=POLLIN}])
recvfrom(3, "\307\240\201\203\0\1\0\0\0\1\0\0\6google\3com\7default\3svc\7cluster\5local\0\0\1\0\1\7cluster\5local\0\0\6\0\1\0\0\0<\0D\2ns\3dns\7cluster\5local\0\nhostmaster\7cluster\5local\0X"..., 512, 0, {sa_family=AF_INET, sin_port=htons(53), sin_addr=inet_addr("10.43.0.10")}, [16]) = 147
recvfrom(3, 0x7ffcca669dd0, 512, 0, 0x7ffcca669880, [16]) = -1 EAGAIN (Resource temporarily unavailable)
poll([{fd=3, events=POLLIN}], 1, 2499)  = 1 ([{fd=3, revents=POLLIN}])
recvfrom(3, "\342\253\201\203\0\1\0\0\0\1\0\0\6google\3com\7default\3svc\7cluster\5local\0\0\34\0\1\7cluster\5local\0\0\6\0\1\0\0\0<\0D\2ns\3dns\7cluster\5local\0\nhostmaster\7cluster\5local\0X"..., 512, 0, {sa_family=AF_INET, sin_port=htons(53), sin_addr=inet_addr("10.43.0.10")}, [16]) = 147
close(3)                                = 0
socket(AF_INET, SOCK_DGRAM|SOCK_CLOEXEC|SOCK_NONBLOCK, IPPROTO_IP) = 3
bind(3, {sa_family=AF_INET, sin_port=htons(0), sin_addr=inet_addr("0.0.0.0")}, 16) = 0
sendto(3, "\34U\1\0\0\1\0\0\0\0\0\0\6google\3com\3svc\7cluster\5local\0\0\1\0\1", 46, MSG_NOSIGNAL, {sa_family=AF_INET, sin_port=htons(53), sin_addr=inet_addr("10.43.0.10")}, 16) = 46
sendto(3, "\37\211\1\0\0\1\0\0\0\0\0\0\6google\3com\3svc\7cluster\5local\0\0\34\0\1", 46, MSG_NOSIGNAL, {sa_family=AF_INET, sin_port=htons(53), sin_addr=inet_addr("10.43.0.10")}, 16) = 46
poll([{fd=3, events=POLLIN}], 1, 2500)  = 1 ([{fd=3, revents=POLLIN}])
recvfrom(3, "\34U\201\203\0\1\0\0\0\1\0\0\6google\3com\3svc\7cluster\5local\0\0\1\0\1\7cluster\5local\0\0\6\0\1\0\0\0<\0D\2ns\3dns\7cluster\5local\0\nhostmaster\7cluster\5local\0X\311\201 \0\0p\200\0"..., 512, 0, {sa_family=AF_INET, sin_port=htons(53), sin_addr=inet_addr("10.43.0.10")}, [16]) = 139
recvfrom(3, 0x7ffcca669dd0, 512, 0, 0x7ffcca669880, [16]) = -1 EAGAIN (Resource temporarily unavailable)
poll([{fd=3, events=POLLIN}], 1, 2499)  = 1 ([{fd=3, revents=POLLIN}])
recvfrom(3, "\37\211\201\203\0\1\0\0\0\1\0\0\6google\3com\3svc\7cluster\5local\0\0\34\0\1\7cluster\5local\0\0\6\0\1\0\0\0<\0D\2ns\3dns\7cluster\5local\0\nhostmaster\7cluster\5local\0X\311\201 \0\0p\200\0"..., 512, 0, {sa_family=AF_INET, sin_port=htons(53), sin_addr=inet_addr("10.43.0.10")}, [16]) = 139
close(3)                                = 0
socket(AF_INET, SOCK_DGRAM|SOCK_CLOEXEC|SOCK_NONBLOCK, IPPROTO_IP) = 3
bind(3, {sa_family=AF_INET, sin_port=htons(0), sin_addr=inet_addr("0.0.0.0")}, 16) = 0
sendto(3, "\21f\1\0\0\1\0\0\0\0\0\0\6google\3com\7cluster\5local\0\0\1\0\1", 42, MSG_NOSIGNAL, {sa_family=AF_INET, sin_port=htons(53), sin_addr=inet_addr("10.43.0.10")}, 16) = 42
sendto(3, "\24\204\1\0\0\1\0\0\0\0\0\0\6google\3com\7cluster\5local\0\0\34\0\1", 42, MSG_NOSIGNAL, {sa_family=AF_INET, sin_port=htons(53), sin_addr=inet_addr("10.43.0.10")}, 16) = 42
poll([{fd=3, events=POLLIN}], 1, 2500)  = 1 ([{fd=3, revents=POLLIN}])
recvfrom(3, "\21f\201\203\0\1\0\0\0\1\0\0\6google\3com\7cluster\5local\0\0\1\0\1\7cluster\5local\0\0\6\0\1\0\0\0<\0D\2ns\3dns\7cluster\5local\0\nhostmaster\7cluster\5local\0X\311\201 \0\0p\200\0\0\34 \0"..., 512, 0, {sa_family=AF_INET, sin_port=htons(53), sin_addr=inet_addr("10.43.0.10")}, [16]) = 135
recvfrom(3, 0x7ffcca669dd0, 512, 0, 0x7ffcca669880, [16]) = -1 EAGAIN (Resource temporarily unavailable)
poll([{fd=3, events=POLLIN}], 1, 2499)  = 1 ([{fd=3, revents=POLLIN}])
recvfrom(3, "\24\204\201\203\0\1\0\0\0\1\0\0\6google\3com\7cluster\5local\0\0\34\0\1\7cluster\5local\0\0\6\0\1\0\0\0<\0D\2ns\3dns\7cluster\5local\0\nhostmaster\7cluster\5local\0X\311\201 \0\0p\200\0\0\34 \0"..., 512, 0, {sa_family=AF_INET, sin_port=htons(53), sin_addr=inet_addr("10.43.0.10")}, [16]) = 135
close(3)                                = 0
socket(AF_INET, SOCK_DGRAM|SOCK_CLOEXEC|SOCK_NONBLOCK, IPPROTO_IP) = 3
bind(3, {sa_family=AF_INET, sin_port=htons(0), sin_addr=inet_addr("0.0.0.0")}, 16) = 0
sendto(3, "\211\260\1\0\0\1\0\0\0\0\0\0\6google\3com\7kubelet\nkubernetes\7rancher\10internal\0\0\1\0\1", 64, MSG_NOSIGNAL, {sa_family=AF_INET, sin_port=htons(53), sin_addr=inet_addr("10.43.0.10")}, 16) = 64
sendto(3, "\214\374\1\0\0\1\0\0\0\0\0\0\6google\3com\7kubelet\nkubernetes\7rancher\10internal\0\0\34\0\1", 64, MSG_NOSIGNAL, {sa_family=AF_INET, sin_port=htons(53), sin_addr=inet_addr("10.43.0.10")}, 16) = 64
poll([{fd=3, events=POLLIN}], 1, 2500)  = 1 ([{fd=3, revents=POLLIN}])
recvfrom(3, "\211\260\205\200\0\1\0\0\0\1\0\0\6google\3com\7kubelet\nkubernetes\7rancher\10internal\0\0\1\0\1\300*\0\6\0\1\0\0\2X\0\30\300*\300*\0-\21B\0\0\0<\0\0\0\n\0\1Q\200\0\0\0\1", 512, 0, {sa_family=AF_INET, sin_port=htons(53), sin_addr=inet_addr("10.43.0.10")}, [16]) = 100
recvfrom(3, 0x7ffcca669dd0, 512, 0, 0x7ffcca669880, [16]) = -1 EAGAIN (Resource temporarily unavailable)
poll([{fd=3, events=POLLIN}], 1, 2499)  = 1 ([{fd=3, revents=POLLIN}])
recvfrom(3, "\214\374\205\200\0\1\0\0\0\1\0\0\6google\3com\7kubelet\nkubernetes\7rancher\10internal\0\0\34\0\1\300*\0\6\0\1\0\0\2X\0\30\300*\300*\0-\21C\0\0\0<\0\0\0\n\0\1Q\200\0\0\0\1", 512, 0, {sa_family=AF_INET, sin_port=htons(53), sin_addr=inet_addr("10.43.0.10")}, [16]) = 100
close(3)                                = 0
ioctl(1, TIOCGWINSZ, {ws_row=0, ws_col=0, ws_xpixel=0, ws_ypixel=0}) = 0
writev(1, [{iov_base="google.com: Name does not resolve", iov_len=33}, {iov_base="\n", iov_len=1}], 2) = 34
exit_group(0)                           = ?
+++ exited with 0 +++