Switch IDNA implementation to libidn2 [BZ #19728] [BZ #19729] [BZ #22247]

This provides an implementation of the IDNA2008 standard and fixes
CVE-2016-6261, CVE-2016-6263, CVE-2017-14062.

7 files changed, 390 insertions, 37 deletions

diff --git a/inet/Makefile b/inet/Makefile
index 5d02c37626..09f5ba78fc 100644
--- a/inet/Makefile
+++ b/inet/Makefile
@@ -45,7 +45,7 @@ routines := htonl htons		\
 	    in6_addr getnameinfo if_index ifaddrs inet6_option \
 	    getipv4sourcefilter setipv4sourcefilter \
 	    getsourcefilter setsourcefilter inet6_opt inet6_rth \
-	    inet6_scopeid_pton deadline
+	    inet6_scopeid_pton deadline idna idna_name_classify
 
 aux := check_pf check_native ifreq
 
@@ -59,12 +59,20 @@ tests := htontest test_ifindex tst-ntoa tst-ether_aton tst-network \
 tests-static += tst-deadline
 tests-internal += tst-deadline
 
+# tst-idna_name_classify must be linked statically because it tests
+# internal functionality.
+tests-static += tst-idna_name_classify
+tests-internal += tst-idna_name_classify
+
 # tst-inet6_scopeid_pton also needs internal functions but does not
 # need to be linked statically.
 tests-internal += tst-inet6_scopeid_pton
 
 include ../Rules
 
+LOCALES := en_US.UTF-8 en_US.ISO-8859-1
+include ../gen-locales.mk
+
 ifeq ($(have-thread-library),yes)
 
 CFLAGS-gethstbyad_r.c += -fexceptions
@@ -103,3 +111,5 @@ endif
 ifeq ($(build-static-nss),yes)
 CFLAGS += -DSTATIC_NSS
 endif
+
+$(objpfx)tst-idna_name_classify.out: $(gen-locales)
diff --git a/inet/Versions b/inet/Versions
index 6f663f3648..9b3661e046 100644
--- a/inet/Versions
+++ b/inet/Versions
@@ -88,5 +88,7 @@ libc {
 
     # Used from nscd.
     __inet6_scopeid_pton;
+    __idna_to_dns_encoding;
+    __idna_from_dns_encoding;
   }
 }
diff --git a/inet/getnameinfo.c b/inet/getnameinfo.c
index a20d20b7cd..5d4978e383 100644
--- a/inet/getnameinfo.c
+++ b/inet/getnameinfo.c
@@ -71,10 +71,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <sys/utsname.h>
 #include <libc-lock.h>
 #include <scratch_buffer.h>
-
-#ifdef HAVE_LIBIDN
-# include <idna.h>
-#endif
+#include <net-internal.h>
 
 #ifndef min
 # define min(x,y) (((x) > (y)) ? (y) : (x))
@@ -82,6 +79,9 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 libc_freeres_ptr (static char *domain);
 
+/* Former NI_IDN_ALLOW_UNASSIGNED, NI_IDN_USE_STD3_ASCII_RULES flags,
+   now ignored.  */
+#define DEPRECATED_NI_IDN 192
 
 static char *
 nrl_domainname (void)
@@ -285,41 +285,28 @@ gni_host_inet_name (struct scratch_buffer *tmpbuf,
 	/* Terminate the string after the prefix.  */
 	*c = '\0';
 
-#ifdef HAVE_LIBIDN
       /* If requested, convert from the IDN format.  */
-      if (flags & NI_IDN)
+      bool do_idn = flags & NI_IDN;
+      char *h_name;
+      if (do_idn)
 	{
-	  int idn_flags = 0;
-	  if  (flags & NI_IDN_ALLOW_UNASSIGNED)
-	    idn_flags |= IDNA_ALLOW_UNASSIGNED;
-	  if (flags & NI_IDN_USE_STD3_ASCII_RULES)
-	    idn_flags |= IDNA_USE_STD3_ASCII_RULES;
-
-	  char *out;
-	  int rc = __idna_to_unicode_lzlz (h->h_name, &out,
-					   idn_flags);
-	  if (rc != IDNA_SUCCESS)
-	    {
-	      if (rc == IDNA_MALLOC_ERROR)
-		return EAI_MEMORY;
-	      if (rc == IDNA_DLOPEN_ERROR)
-		return EAI_SYSTEM;
-	      return EAI_IDN_ENCODE;
-	    }
-
-	  if (out != h->h_name)
-	    {
-	      h->h_name = strdupa (out);
-	      free (out);
-	    }
+	  int rc = __idna_from_dns_encoding (h->h_name, &h_name);
+	  if (rc == EAI_IDN_ENCODE)
+	    /* Use the punycode name as a fallback.  */
+	    do_idn = false;
+	  else if (rc != 0)
+	    return rc;
 	}
-#endif
+      if (!do_idn)
+	h_name = h->h_name;
 
-      size_t len = strlen (h->h_name) + 1;
+      size_t len = strlen (h_name) + 1;
       if (len > hostlen)
 	return EAI_OVERFLOW;
+      memcpy (host, h_name, len);
 
-      memcpy (host, h->h_name, len);
+      if (do_idn)
+	free (h_name);
 
       return 0;
     }
@@ -501,10 +488,7 @@ getnameinfo (const struct sockaddr *sa, socklen_t addrlen, char *host,
 	     int flags)
 {
   if (flags & ~(NI_NUMERICHOST|NI_NUMERICSERV|NI_NOFQDN|NI_NAMEREQD|NI_DGRAM
-#ifdef HAVE_LIBIDN
-		|NI_IDN|NI_IDN_ALLOW_UNASSIGNED|NI_IDN_USE_STD3_ASCII_RULES
-#endif
-		))
+		|NI_IDN|DEPRECATED_NI_IDN))
     return EAI_BADFLAGS;
 
   if (sa == NULL || addrlen < sizeof (sa_family_t))
diff --git a/inet/idna.c b/inet/idna.c
new file mode 100644
index 0000000000..c561bf2e9e
--- /dev/null
+++ b/inet/idna.c
@@ -0,0 +1,182 @@
+/* IDNA functions, forwarding to implementations in libidn2.
+   Copyright (C) 2018 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <allocate_once.h>
+#include <dlfcn.h>
+#include <inet/net-internal.h>
+#include <netdb.h>
+#include <stdbool.h>
+
+/* Use the soname and version to locate libidn2, to ensure a
+   compatible ABI.  */
+#define LIBIDN2_SONAME "libidn2.so.0"
+#define LIBIDN2_VERSION "IDN2_0.0.0"
+
+/* Return codes from libidn2.  */
+enum
+  {
+    IDN2_OK = 0,
+    IDN2_MALLOC = -100,
+  };
+
+/* Functions from libidn2.  */
+struct functions
+{
+  void *handle;
+  int (*lookup_ul) (const char *src, char **result, int flags);
+  int (*to_unicode_lzlz) (const char *name, char **result, int flags);
+};
+
+static void *
+functions_allocate (void *closure)
+{
+  struct functions *result = malloc (sizeof (*result));
+  if (result == NULL)
+    return NULL;
+
+  void *handle = __libc_dlopen (LIBIDN2_SONAME);
+  if (handle == NULL)
+    /* Do not cache open failures.  The library may appear
+       later.  */
+    {
+      free (result);
+      return NULL;
+    }
+
+  void *ptr_lookup_ul
+    = __libc_dlvsym (handle, "idn2_lookup_ul", LIBIDN2_VERSION);
+  void *ptr_to_unicode_lzlz
+    = __libc_dlvsym (handle, "idn2_to_unicode_lzlz", LIBIDN2_VERSION);
+  if (ptr_lookup_ul == NULL || ptr_to_unicode_lzlz == NULL)
+    {
+      __libc_dlclose (handle);
+      free (result);
+      return NULL;
+    }
+
+  result->handle = handle;
+  result->lookup_ul = ptr_lookup_ul;
+  result->to_unicode_lzlz = ptr_to_unicode_lzlz;
+#ifdef PTR_MANGLE
+  PTR_MANGLE (result->lookup_ul);
+  PTR_MANGLE (result->to_unicode_lzlz);
+#endif
+
+  return result;
+}
+
+static void
+functions_deallocate (void *closure, void *ptr)
+{
+  struct functions *functions = ptr;
+  __libc_dlclose (functions->handle);
+  free (functions);
+}
+
+/* Ensure that *functions is initialized and return the value of the
+   pointer.  If the library cannot be loaded, return NULL.  */
+static inline struct functions *
+get_functions (void)
+{
+  static void *functions;
+  return allocate_once (&functions, functions_allocate, functions_deallocate,
+                        NULL);
+}
+
+/* strdup with an EAI_* error code.  */
+static int
+gai_strdup (const char *name, char **result)
+{
+  char *ptr = __strdup (name);
+  if (ptr == NULL)
+    return EAI_MEMORY;
+  *result = ptr;
+  return 0;
+}
+
+int
+__idna_to_dns_encoding (const char *name, char **result)
+{
+  switch (__idna_name_classify (name))
+    {
+    case idna_name_ascii:
+      /* Nothing to convert.  */
+      return gai_strdup (name, result);
+    case idna_name_nonascii:
+      /* Encoding needed.  Handled below.  */
+      break;
+    case idna_name_nonascii_backslash:
+    case idna_name_encoding_error:
+      return EAI_IDN_ENCODE;
+    case idna_name_memory_error:
+      return EAI_MEMORY;
+    case idna_name_error:
+      return EAI_SYSTEM;
+    }
+
+  struct functions *functions = get_functions ();
+  if (functions == NULL)
+    /* We report this as an encoding error (assuming that libidn2 is
+       not installed), although the root cause may be a temporary
+       error condition due to resource shortage.  */
+    return EAI_IDN_ENCODE;
+  char *ptr = NULL;
+  __typeof__ (functions->lookup_ul) fptr = functions->lookup_ul;
+#ifdef PTR_DEMANGLE
+  PTR_DEMANGLE (fptr);
+#endif
+  int ret = fptr (name, &ptr, 0);
+  if (ret == 0)
+    {
+      /* Assume that idn2_free is equivalent to free.  */
+      *result = ptr;
+      return 0;
+    }
+  else if (ret == IDN2_MALLOC)
+    return EAI_MEMORY;
+  else
+    return EAI_IDN_ENCODE;
+}
+libc_hidden_def (__idna_to_dns_encoding)
+
+int
+__idna_from_dns_encoding (const char *name, char **result)
+{
+  struct functions *functions = get_functions ();
+  if (functions == NULL)
+    /* Simply use the encoded name, assuming that it is not punycode
+       (but even a punycode name would be syntactically valid).  */
+    return gai_strdup (name, result);
+  char *ptr = NULL;
+  __typeof__ (functions->to_unicode_lzlz) fptr = functions->to_unicode_lzlz;
+#ifdef PTR_DEMANGLE
+  PTR_DEMANGLE (fptr);
+#endif
+  int ret = fptr (name, &ptr, 0);
+  if (ret == 0)
+    {
+      /* Assume that idn2_free is equivalent to free.  */
+      *result = ptr;
+      return 0;
+    }
+  else if (ret == IDN2_MALLOC)
+    return EAI_MEMORY;
+  else
+    return EAI_IDN_ENCODE;
+}
+libc_hidden_def (__idna_from_dns_encoding)
diff --git a/inet/idna_name_classify.c b/inet/idna_name_classify.c
new file mode 100644
index 0000000000..3683e1133f
--- /dev/null
+++ b/inet/idna_name_classify.c
@@ -0,0 +1,75 @@
+/* Classify a domain name for IDNA purposes.
+   Copyright (C) 2018 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <errno.h>
+#include <inet/net-internal.h>
+#include <stdbool.h>
+#include <string.h>
+#include <wchar.h>
+
+enum idna_name_classification
+__idna_name_classify (const char *name)
+{
+  mbstate_t mbs;
+  memset (&mbs, 0, sizeof (mbs));
+  const char *p = name;
+  const char *end = p + strlen (p) + 1;
+  bool nonascii = false;
+  bool backslash = false;
+  while (true)
+    {
+      wchar_t wc;
+      size_t result = mbrtowc (&wc, p, end - p, &mbs);
+      if (result == 0)
+        /* NUL terminator was reached.  */
+        break;
+      else if (result == (size_t) -2)
+        /* Incomplete trailing multi-byte character.  This is an
+           encoding error becaue we received the full name.  */
+        return idna_name_encoding_error;
+      else if (result == (size_t) -1)
+        {
+          /* Other error, including EILSEQ.  */
+          if (errno == EILSEQ)
+            return idna_name_encoding_error;
+          else if (errno == ENOMEM)
+            return idna_name_memory_error;
+          else
+            return idna_name_error;
+        }
+      else
+        {
+          /* A wide character was decoded.  */
+          p += result;
+          if (wc == L'\\')
+            backslash = true;
+          else if (wc > 127)
+            nonascii = true;
+        }
+    }
+
+  if (nonascii)
+    {
+      if (backslash)
+        return idna_name_nonascii_backslash;
+      else
+        return idna_name_nonascii;
+    }
+  else
+    return idna_name_ascii;
+}
diff --git a/inet/net-internal.h b/inet/net-internal.h
index 8a9505cf99..0ba6736aef 100644
--- a/inet/net-internal.h
+++ b/inet/net-internal.h
@@ -29,6 +29,33 @@ int __inet6_scopeid_pton (const struct in6_addr *address,
 libc_hidden_proto (__inet6_scopeid_pton)
 
 
+/* IDNA conversion.  These functions convert domain names between the
+   current multi-byte character set and the IDNA encoding.  On
+   success, the result string is written to *RESULT (which the caller
+   has to free), and zero is returned.  On error, an EAI_* error code
+   is returned (see <netdb.h>), and *RESULT is not changed.  */
+int __idna_to_dns_encoding (const char *name, char **result);
+libc_hidden_proto (__idna_to_dns_encoding)
+int __idna_from_dns_encoding (const char *name, char **result);
+libc_hidden_proto (__idna_from_dns_encoding)
+
+
+/* Return value of __idna_name_classify below.  */
+enum idna_name_classification
+{
+  idna_name_ascii,          /* No non-ASCII characters.  */
+  idna_name_nonascii,       /* Non-ASCII characters, no backslash.  */
+  idna_name_nonascii_backslash, /* Non-ASCII characters with backslash.  */
+  idna_name_encoding_error, /* Decoding error.  */
+  idna_name_memory_error,   /* Memory allocation failure.  */
+  idna_name_error,          /* Other error during decoding.  Check errno.  */
+};
+
+/* Check the specified name for non-ASCII characters and backslashes
+   or encoding errors.  */
+enum idna_name_classification __idna_name_classify (const char *name)
+  attribute_hidden;
+
 /* Deadline handling for enforcing timeouts.
 
    Code should call __deadline_current_time to obtain the current time
diff --git a/inet/tst-idna_name_classify.c b/inet/tst-idna_name_classify.c
new file mode 100644
index 0000000000..c4a2c91329
--- /dev/null
+++ b/inet/tst-idna_name_classify.c
@@ -0,0 +1,73 @@
+/* Test IDNA name classification.
+   Copyright (C) 2018 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <inet/net-internal.h>
+#include <locale.h>
+#include <stdio.h>
+#include <support/check.h>
+
+static void
+locale_insensitive_tests (void)
+{
+  TEST_COMPARE (__idna_name_classify (""), idna_name_ascii);
+  TEST_COMPARE (__idna_name_classify ("abc"), idna_name_ascii);
+  TEST_COMPARE (__idna_name_classify (".."), idna_name_ascii);
+  TEST_COMPARE (__idna_name_classify ("\001abc\177"), idna_name_ascii);
+  TEST_COMPARE (__idna_name_classify ("\\065bc"), idna_name_ascii);
+}
+
+static int
+do_test (void)
+{
+  puts ("info: C locale tests");
+  locale_insensitive_tests ();
+  TEST_COMPARE (__idna_name_classify ("abc\200def"),
+                idna_name_encoding_error);
+  TEST_COMPARE (__idna_name_classify ("abc\200\\def"),
+                idna_name_encoding_error);
+  TEST_COMPARE (__idna_name_classify ("abc\377def"),
+                idna_name_encoding_error);
+
+  puts ("info: en_US.ISO-8859-1 locale tests");
+  if (setlocale (LC_CTYPE, "en_US.ISO-8859-1") == 0)
+    FAIL_EXIT1 ("setlocale for en_US.ISO-8859-1: %m\n");
+  locale_insensitive_tests ();
+  TEST_COMPARE (__idna_name_classify ("abc\200def"), idna_name_nonascii);
+  TEST_COMPARE (__idna_name_classify ("abc\377def"), idna_name_nonascii);
+  TEST_COMPARE (__idna_name_classify ("abc\\\200def"),
+                idna_name_nonascii_backslash);
+  TEST_COMPARE (__idna_name_classify ("abc\200\\def"),
+                idna_name_nonascii_backslash);
+
+  puts ("info: en_US.UTF-8 locale tests");
+  if (setlocale (LC_CTYPE, "en_US.UTF-8") == 0)
+    FAIL_EXIT1 ("setlocale for en_US.UTF-8: %m\n");
+  locale_insensitive_tests ();
+  TEST_COMPARE (__idna_name_classify ("abc\xc3\x9f""def"), idna_name_nonascii);
+  TEST_COMPARE (__idna_name_classify ("abc\\\xc3\x9f""def"),
+                idna_name_nonascii_backslash);
+  TEST_COMPARE (__idna_name_classify ("abc\xc3\x9f\\def"),
+                idna_name_nonascii_backslash);
+  TEST_COMPARE (__idna_name_classify ("abc\200def"), idna_name_encoding_error);
+  TEST_COMPARE (__idna_name_classify ("abc\xc3""def"), idna_name_encoding_error);
+  TEST_COMPARE (__idna_name_classify ("abc\xc3"), idna_name_encoding_error);
+
+  return 0;
+}
+
+#include <support/test-driver.c>