1 files changed, 82 insertions, 45 deletions

diff --git a/gl/open.c b/gl/open.c
index 7ec8fdc3..d76372fd 100644
--- a/gl/open.c
+++ b/gl/open.c
@@ -1,5 +1,5 @@
 /* Open a descriptor to a file.
-   Copyright (C) 2007-2023 Free Software Foundation, Inc.
+   Copyright (C) 2007-2025 Free Software Foundation, Inc.
 
    This file is free software: you can redistribute it and/or modify
    it under the terms of the GNU Lesser General Public License as
@@ -38,9 +38,13 @@ orig_open (const char *filename, int flags, mode_t mode)
 }
 
 /* Specification.  */
+#ifdef __osf__
 /* Write "fcntl.h" here, not <fcntl.h>, otherwise OSF/1 5.1 DTK cc eliminates
    this include because of the preliminary #include <fcntl.h> above.  */
-#include "fcntl.h"
+# include "fcntl.h"
+#else
+# include <fcntl.h>
+#endif
 
 #include "cloexec.h"
 
@@ -51,24 +55,29 @@ orig_open (const char *filename, int flags, mode_t mode)
 #include <sys/stat.h>
 #include <unistd.h>
 
+#ifndef HAVE_WORKING_O_DIRECTORY
+# define HAVE_WORKING_O_DIRECTORY false
+#endif
+
+#ifndef OPEN_TRAILING_SLASH_BUG
+# define OPEN_TRAILING_SLASH_BUG false
+#endif
+
 #ifndef REPLACE_OPEN_DIRECTORY
-# define REPLACE_OPEN_DIRECTORY 0
+# define REPLACE_OPEN_DIRECTORY false
 #endif
 
+static int
+lstatif (char const *filename, struct stat *st, int flags)
+{
+  return flags & O_NOFOLLOW ? lstat (filename, st) : stat (filename, st);
+}
+
 int
 open (const char *filename, int flags, ...)
 {
-  /* 0 = unknown, 1 = yes, -1 = no.  */
-#if GNULIB_defined_O_CLOEXEC
-  int have_cloexec = -1;
-#else
-  static int have_cloexec;
-#endif
-
-  mode_t mode;
-  int fd;
+  mode_t mode = 0;
 
-  mode = 0;
   if (flags & O_CREAT)
     {
       va_list arg;
@@ -95,7 +104,6 @@ open (const char *filename, int flags, ...)
     filename = "NUL";
 #endif
 
-#if OPEN_TRAILING_SLASH_BUG
   /* Fail if one of O_CREAT, O_WRONLY, O_RDWR is specified and the filename
      ends in a slash, as POSIX says such a filename must name a directory
      <https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13>:
@@ -114,21 +122,55 @@ open (const char *filename, int flags, ...)
          directories,
        - if O_WRONLY or O_RDWR is specified, open() must fail because the
          file does not contain a '.' directory.  */
-  if ((flags & O_CREAT)
-      || (flags & O_ACCMODE) == O_RDWR
-      || (flags & O_ACCMODE) == O_WRONLY)
+  bool check_for_slash_bug;
+  if (OPEN_TRAILING_SLASH_BUG)
     {
       size_t len = strlen (filename);
-      if (len > 0 && filename[len - 1] == '/')
+      check_for_slash_bug = len && filename[len - 1] == '/';
+    }
+  else
+    check_for_slash_bug = false;
+
+  if (check_for_slash_bug
+      && (flags & O_CREAT
+          || (flags & O_ACCMODE) == O_RDWR
+          || (flags & O_ACCMODE) == O_WRONLY))
+    {
+      errno = EISDIR;
+      return -1;
+    }
+
+  /* With the trailing slash bug or without working O_DIRECTORY, check with
+     stat first lest we hang trying to open a fifo.  Although there is
+     a race between this and opening the file, we can do no better.
+     After opening the file we will check again with fstat.  */
+  bool check_directory =
+    (check_for_slash_bug
+     || (!HAVE_WORKING_O_DIRECTORY && flags & O_DIRECTORY));
+  if (check_directory)
+    {
+      struct stat statbuf;
+      if (lstatif (filename, &statbuf, flags) < 0)
+        {
+          if (! (flags & O_CREAT && errno == ENOENT))
+            return -1;
+        }
+      else if (!S_ISDIR (statbuf.st_mode))
         {
-          errno = EISDIR;
+          errno = ENOTDIR;
           return -1;
         }
     }
+
+  /* 0 = unknown, 1 = yes, -1 = no.  */
+#if GNULIB_defined_O_CLOEXEC
+  int have_cloexec = -1;
+#else
+  static int have_cloexec;
 #endif
 
-  fd = orig_open (filename,
-                  flags & ~(have_cloexec < 0 ? O_CLOEXEC : 0), mode);
+  int fd = orig_open (filename,
+                      flags & ~(have_cloexec < 0 ? O_CLOEXEC : 0), mode);
 
   if (flags & O_CLOEXEC)
     {
@@ -150,19 +192,21 @@ open (const char *filename, int flags, ...)
 #if REPLACE_FCHDIR
   /* Implementing fchdir and fdopendir requires the ability to open a
      directory file descriptor.  If open doesn't support that (as on
-     mingw), we use a dummy file that behaves the same as directories
+     mingw), use a dummy file that behaves the same as directories
      on Linux (ie. always reports EOF on attempts to read()), and
-     override fstat() in fchdir.c to hide the fact that we have a
-     dummy.  */
+     override fstat in fchdir.c to hide the dummy.  */
   if (REPLACE_OPEN_DIRECTORY && fd < 0 && errno == EACCES
-      && ((flags & O_ACCMODE) == O_RDONLY
-          || (O_SEARCH != O_RDONLY && (flags & O_ACCMODE) == O_SEARCH)))
+      && ((flags & (O_ACCMODE | O_CREAT)) == O_RDONLY
+          || (O_SEARCH != O_RDONLY
+              && (flags & (O_ACCMODE | O_CREAT)) == O_SEARCH)))
     {
       struct stat statbuf;
-      if (stat (filename, &statbuf) == 0 && S_ISDIR (statbuf.st_mode))
+      if (check_directory
+          || (lstatif (filename, &statbuf, flags) == 0
+              && S_ISDIR (statbuf.st_mode)))
         {
           /* Maximum recursion depth of 1.  */
-          fd = open ("/dev/null", flags, mode);
+          fd = open ("/dev/null", flags & ~O_DIRECTORY, mode);
           if (0 <= fd)
             fd = _gl_register_fd (fd, filename);
         }
@@ -171,10 +215,8 @@ open (const char *filename, int flags, ...)
     }
 #endif
 
-#if OPEN_TRAILING_SLASH_BUG
-  /* If the filename ends in a slash and fd does not refer to a directory,
-     then fail.
-     Rationale: POSIX says such a filename must name a directory
+  /* If checking for directories, fail if fd does not refer to a directory.
+     Rationale: A filename ending in slash cannot name a non-directory
      <https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13>:
        "A pathname that contains at least one non-<slash> character and that
         ends with one or more trailing <slash> characters shall not be resolved
@@ -182,23 +224,18 @@ open (const char *filename, int flags, ...)
         <slash> characters names an existing directory"
      If the named file without the slash is not a directory, open() must fail
      with ENOTDIR.  */
-  if (fd >= 0)
+  if (check_directory && 0 <= fd)
     {
-      /* We know len is positive, since open did not fail with ENOENT.  */
-      size_t len = strlen (filename);
-      if (filename[len - 1] == '/')
+      struct stat statbuf;
+      int r = fstat (fd, &statbuf);
+      if (r < 0 || !S_ISDIR (statbuf.st_mode))
         {
-          struct stat statbuf;
-
-          if (fstat (fd, &statbuf) >= 0 && !S_ISDIR (statbuf.st_mode))
-            {
-              close (fd);
-              errno = ENOTDIR;
-              return -1;
-            }
+          int err = r < 0 ? errno : ENOTDIR;
+          close (fd);
+          errno = err;
+          return -1;
         }
     }
-#endif
 
 #if REPLACE_FCHDIR
   if (!REPLACE_OPEN_DIRECTORY && 0 <= fd)