Re: [musl] pthread_mutex_t shared between processes with different pid namespaces

[Daniele Personal <d.dario76@gmail.com>]

[-- Attachment #1: Type: text/plain, Size: 2600 bytes --]

On Mon, 2025-02-03 at 18:25 +0100, Florian Weimer wrote:
> * Daniele Personal:
> 
> > On Sat, 2025-02-01 at 17:03 +0100, Florian Weimer wrote:
> > > * Daniele Personal:
> > > 
> > > > > Is this required for implementing the unlock-if-not-owner
> > > > > error
> > > > > code
> > > > > on mutex unlock?
> > > > 
> > > > No, I don't see problems related to EOWNERDEAD.
> > > 
> > > Sorry, what I meant is that the TID is needed for efficient
> > > reporting
> > > of
> > > usage errors.  It's not imposed by the robust list protocol as
> > > such.
> > > There could be a PID-namespace-compatible robust mutex type that
> > > does
> > > not have this problem (but with less error checking).
> > > 
> > > Thanks,
> > > Florian
> > > 
> > 
> > Are you saying that there are pthread_mutexes which can be shared
> > across processes run on different pid namespaces? If yes I'm
> > definitely
> > interested on this. Can you tell me something more?
> 
> You would have to add a new mutex type that is a mix of
> PTHREAD_MUTEX_NORMAL amd PTHREAD_MUTEX_ROBUST.  Closer to the latter,
> but without the ownership checks.
> 
> Thanks,
> Florian
> 

I wrote a stupid test to exercise things. It creates (if needed) a
shared object which will hold the pthread_mutex_t instance, mmaps it
and if the shared object has been created, initializes the mutex
(changing mutex initialization requires to delete the shared object).

Then it locks the mutex for 5 seconds, unlocks it and locks it again
after another 2 seconds and exits with the mutex locked.

I compiled the code linking to musl 1.2.4 with Yocto or for my laptop
linking to glibc 2.34 with

gcc -D_GNU_SOURCE -Wall -pthread -g -O2 -c -o main.o main.c
gcc -pthread -g -O2 -pthread -o mutex-test main.o -lrt

I have a container which is started with its own pid, mount and uts
namespaces (it shares IPC and network with the host).

Running the test separately on host and container, both can recognize
the case where the previous instance left the mutex locked and can
recover from it.

Running them in parallel gives two distinct results if the mutex is
initialized with PTHREAD_PRIO_INHERIT protocol or PTHREAD_PRIO_NONE.

If PTHREAD_PRIO_INHERIT is set, in case of contention, the waiter gets
stuck.

If PTHREAD_PRIO_NONE is used, everything seems to work fine: the
application which starts later waits for the mutex to be released by
the other one and gets waked property.

I now need to understand if this behavior is expected and reliable or
not.

Thanks in advance,
Daniele.

[-- Attachment #2: main.c --]
[-- Type: text/x-csrc, Size: 11941 bytes --]

/*
 * main.c
 *
 *  Created on: Feb 4, 2025
 */

#include <time.h>
#include <errno.h>
#include <fcntl.h>
#include <sched.h>
#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <pthread.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/types.h>

/**
 * Definition of shared object to be
 * mmapped() by the different processes
 */
typedef struct _SharedMutex
{
  /* A magic number just to check if object has been initialized */
  uint32_t magic;

  /* The attributes used to initialize the shared mutex */
  pthread_mutexattr_t attr;

  /* The shared mutex */
  pthread_mutex_t lock;
} SharedMutex;

#define MAGIC 0xdeadbeef

/** Default name to use when printing debug stuff */
static const char *app = "A";

#define DBG_PRINT(f,a...) \
  do {\
      struct timespec now; \
      clock_gettime (CLOCK_MONOTONIC, &now); \
      printf ("(%s) [%li.%09li] "f"\n", app, now.tv_sec, now.tv_nsec, ##a); \
  } while (0);

/** Protocol type: by default PTHREAD_PRIO_NONE */
static int prio = PTHREAD_PRIO_NONE;

/**
 * @brief Initialize a mutex instance setting #PTHREAD_MUTEX_ROBUST and
 *        #PTHREAD_PROCESS_SHARED attributes. This allows to have a mutex
 *        usable within different processes and recoverable in case the
 *        process which owns it crashes.
 *
 * @param lock A #pthread_mutex_t instance to initialize
 *
 * @return 0 on success, -1 otherwise
 */
static int mutex_init (pthread_mutex_t *lock, pthread_mutexattr_t *attr)
{
  int res;

  pthread_mutexattr_init (attr);

  DBG_PRINT ("setting PTHREAD_PROCESS_SHARED attribute");
  res = pthread_mutexattr_setpshared (attr, PTHREAD_PROCESS_SHARED);
  if (res != 0)
    {
      /* Failed to set SHARED */
      printf ("failed to set PTHREAD_PROCESS_SHARED: %i %s\n", res,
          strerror (res));
      return -1;
    }

  if (prio != PTHREAD_PRIO_NONE)
    {
      DBG_PRINT ("setting PTHREAD_PRIO_INHERIT attribute");
      res = pthread_mutexattr_setprotocol (attr, PTHREAD_PRIO_INHERIT);
      if (res != 0)
        {
          /* Failed to set protocol */
          printf ("failed to set PTHREAD_PRIO_INHERIT: %i %s\n", res,
              strerror (res));
          return -1;
        }
    }

  DBG_PRINT ("setting PTHREAD_MUTEX_ROBUST attribute");
  res = pthread_mutexattr_setrobust (attr, PTHREAD_MUTEX_ROBUST);
  if (res != 0)
    {
      printf ("failed to set PTHREAD_MUTEX_ROBUST: %i %s\n", res,
          strerror (res));
      return -1;
    }

  /*
   * Initialize mutex instance.
   * These method always return 0 so no need to check for success.
   */
  pthread_mutex_init (lock, attr);

  return 0;
}

/**
 * @brief Initialize a mutex instance setting #PTHREAD_MUTEX_ROBUST and
 *        #PTHREAD_PROCESS_SHARED attributes. This allows to have a mutex
 *        usable within different processes and recoverable in case the
 *        process which owns it crashes.
 *
 * @param lock A #pthread_mutex_t instance to initialize
 *
 * @return 0 on success, -1 otherwise
 */
static int mutexattr_check (pthread_mutexattr_t *attr)
{
  int res;
  int val;

  DBG_PRINT ("checking PTHREAD_PROCESS attribute");
  res = pthread_mutexattr_getpshared (attr, &val);
  if (res != 0)
    {
      /* Failed to set SHARED */
      printf ("failed to get PTHREAD_PROCESS attribute: %i %s\n",
          res, strerror (res));
      return -1;
    }
  if (val != PTHREAD_PROCESS_SHARED)
    {
      printf ("mutex was not initialized with PTHREAD_PROCESS_SHARED"
          " attribute\n");
      return -1;
    }

  DBG_PRINT ("checking protocol attribute");
  res = pthread_mutexattr_getprotocol (attr, &val);
  if (res != 0)
    {
      /* Failed to set protocol */
      printf ("failed to get protocol attribute: %i %s\n", res,
          strerror (res));
      return -1;
    }
  if (val != prio)
    {
      printf ("mutex was initialized with protocol %s but we wanted %s\n",
          (val == PTHREAD_PRIO_NONE) ? "PTHREAD_PRIO_NONE" : "PTHREAD_PRIO_INHERIT",
          (prio == PTHREAD_PRIO_NONE) ? "PTHREAD_PRIO_NONE" : "PTHREAD_PRIO_INHERIT");
      return -1;
    }

  DBG_PRINT ("checking PTHREAD_MUTEX_ROBUST attribute");
  res = pthread_mutexattr_getrobust (attr, &val);
  if (res != 0)
    {
      printf ("failed to get robust attribute: %i %s\n", res,
          strerror (res));
      return -1;
    }
  if (val != PTHREAD_MUTEX_ROBUST)
    {
      printf ("mutex was not initialized with PTHREAD_MUTEX_ROBUST"
          " attribute\n");
      return -1;
    }

  return 0;
}

/**
 * @brief Lock the given mutex. If the mutex is currently unlocked, it
 *        becomes locked and owned by the calling thread, and method
 *        returns immediately. If the mutex is already locked by another
 *        thread, method suspends the calling thread until the mutex is
 *        unlocked. If the thread which was owning the mutex terminates,
 *        method restores consistency of the mutex and locks it returning 0.
 *
 * @param lock A #pthread_mutex_t instance to lock
 *
 * @return 0 on success, a positive error code otherwise
 */
static int mutex_lock (pthread_mutex_t *lock)
{
  /* Try to lock mutex */
  int res = pthread_mutex_lock (lock);
  switch (res) {
    case 0:
      /* Common use case */
      break;
    case EOWNERDEAD:
      /*
       * Process which was holding the mutex terminated: now we're
       * holding it but before to go on we have to make it consistent.
       */
      DBG_PRINT ("restoring mutex consistency");
      res = pthread_mutex_consistent (lock);
      if (res != 0)
        {
          /* Failed to restore consistency */
          printf ("failed to restore consistency of mutex: %i %s\n", res,
              strerror (res));
        }

      break;
    default:
      printf ("failed to lock mutex: %i %s\n", res, strerror (res));
      break;
  }

  return res;
}

/**
 * @brief Try to open @shmfile shared object and if this is the first instance
 *        that opens it, initialize shared memory.
 *
 * @param name Name of the shared memory object to be created or opened.
 *             For portable use, a shared memory object should be identified
 *             by a name of the form /some-name; that is, a null-terminated
 *             string of up to NAME_MAX (i.e., 255) characters consisting of
 *             an initial slash, followed by one or more characters, none of
 *             which are slashes.
 *
 * @return pointer to #SharedMutex instance on success, NULL otherwise
 */
static SharedMutex *shm_map (const char *name)
{
#define SHM_OPS_INIT  1
#define SHM_OPS_CHECK 2

  int fd;
  int res;
  int ops = SHM_OPS_INIT;
  size_t size = sizeof (SharedMutex);

  /* Sanity checks */
  if (name == NULL)
    {
      printf ("unable to open shared object: missing shared object name\n");
      return NULL;
    }

  /* Open handle to shared object */
  fd = shm_open (name, O_RDWR | O_CREAT | O_EXCL, 0600);
  if (fd < 0 && errno == EEXIST)
    {
      /*
       * If memory object @name already exists, another process has
       * initialized the memory area.
       */
      ops = SHM_OPS_CHECK;
      fd = shm_open (name, O_RDWR, 0600);
    }

  if (fd < 0)
    {
      /* Failed to open shared object */
      printf ("unable to open shared object %s: %i %s\n", name, errno,
          strerror (errno));
      return NULL;
    }

  if (ops == SHM_OPS_INIT)
    {
      /* Set desired size of shared object */
      res = ftruncate (fd, (off_t) size);
      if (res < 0)
        {
          printf ("unable to set size of shared object %s: %i %s\n", name,
              errno, strerror (errno));
          close (fd);
          shm_unlink (name);
          return NULL;
        }
    }

  /* Map shared memory region */
  void *p = mmap (NULL, size, (PROT_READ | PROT_WRITE), MAP_SHARED, fd, 0);
  if (p == MAP_FAILED)
    {
      printf ("unable to map shared object %s contents: %i %s\n", name,
          errno, strerror (errno));

      close (fd);
      if (ops == SHM_OPS_INIT)
        {
          /* Also unlink shared object */
          shm_unlink (name);
        }

      return NULL;
    }

  /* We can safely close file descriptor */
  close (fd);

  /* Helper to access shared info */
  SharedMutex *info = p;

  switch (ops) {
    case SHM_OPS_INIT:
      /*
       * Clear contents of shared memory area before to start working on it.
       * This should not be needed because #ftruncate() should do it but
       * it doesn't harm.
       */
      memset (p, 0, size);

      /* Initialize shared lock */
      if (mutex_init (&info->lock, &info->attr) != 0)
        {
          /* Cleanup stuff */
          munmap (p, size);

          /* Remove shared object */
          shm_unlink (name);

          return NULL;
        }

      /* Write magic number */
      info->magic = MAGIC;
      break;
    case SHM_OPS_CHECK:
      /*
       * Shared object has already been created. Last thing that is set is the
       * magic value: loop until it becomes valid before to do other things.
       */
      while (info->magic != MAGIC)
        {
          if (info->magic != 0 && info->magic != MAGIC)
            {
              printf ("shared object %s initialized with wrong magic:"
                  " aborting\n", name);
              return NULL;
            }

          sched_yield ();
        }

      /* Check if attributes are consistent with our choices */
      res = mutexattr_check (&info->attr);
      if (res != 0)
        {
          printf ("mutex attributes incompatible: aborting\n");

          /* Cleanup stuff */
          munmap (p, size);

          return NULL;
        }

      break;
    default:
      /* This should never happen */
      printf ("invalid initialization option\n");
      return NULL;
  }

  /* Return mapped shared object */
  return info;
}

/**
 * @brief Show application usage
 */
static void usage (const char *name)
{
  printf ("Usage: %s [options]\n", name);
  printf (" -h | --help : show this help\n");
  printf (" -n | --name : a string to distinguish between running instances"
      " (default \"A\"\n");
  printf (" -p | --prio_inherit : set PTHREAD_PRIO_INHERIT (default"
      " PTHREAD_PRIO_NONE)\n");
}

/**
 * Main method
 */
int main (int argc, char *argv [])
{
  int res;
  SharedMutex *obj;

  for (int c = 1; c < argc; c++)
    {
      if (strcmp (argv [c], "-h") == 0 ||
          strcmp (argv [c], "--help") == 0)
        {
          usage (argv [0]);
          return -1;
        }

      if (strcmp (argv [c], "-n") == 0 ||
          strcmp (argv [c], "--name") == 0)
        {
          if ((++c) >= argc)
            {
              /* Missing name argument */
              usage (argv [0]);
              return -1;
            }

          /* Replace application name */
          app = argv [c];
          continue;
        }

      if (strcmp (argv [c], "-p") == 0 ||
          strcmp (argv [c], "--prio-inherit") == 0)
        {
          /* Set priority inheritance */
          prio = PTHREAD_PRIO_INHERIT;
          continue;
        }
    }

  DBG_PRINT ("Mapping shared object");
  obj = shm_map ("/test");
  if (obj == NULL)
    {
      /* Failed to create/open shared object */
      return -1;
    }

  DBG_PRINT ("locking mutex");
  res = mutex_lock (&obj->lock);
  if (res != 0)
    {
      /* Failed to lock mutex */
      return -1;
    }

  DBG_PRINT ("waiting 5 [s] before to unlock mutex");
  sleep (5);

  DBG_PRINT ("releasing mutex");
  res = pthread_mutex_unlock (&obj->lock);
  if (res != 0)
    {
      /* Failed to unlock */
      DBG_PRINT ("failed to unlock mutex: %i %s", res, strerror (res));
      return -1;
    }

  DBG_PRINT ("waiting 2 [s] before to retry locking mutex");
  sleep (2);

  DBG_PRINT ("locking mutex");
  res = mutex_lock (&obj->lock);
  if (res != 0)
    {
      /* Failed to lock mutex */
      return -1;
    }

  DBG_PRINT ("terminating with mutex locked");

  return 0;
}