/* Copyright © Triad National Security, LLC, and others. */

#define _GNU_SOURCE
#include "config.h"

#include <grp.h>
#include <errno.h>
#include <fcntl.h>
#include <pwd.h>
#include <semaphore.h>
#include <string.h>
#include <sys/mman.h>
#include <sys/mount.h>
#include <sys/prctl.h>
#include <sys/syscall.h>
#include <time.h>
#include <unistd.h>

#include "all.h"


/** Macros **/

/* Timeout in seconds for waiting for join semaphore. */
#define JOIN_TIMEOUT 30

/* Maximum length of paths we’re willing to deal with. (Note that
   system-defined PATH_MAX isn't reliable.) */
#define PATH_CHARS 4096

/* Mount point for the tmpfs used by -W. We want this to be (a) always
   available [1], (b) short, (c) not used by anything else we care about
   during container setup, and (d) not wildly confusing if users see it in an
   error message. Must be a string literal because we use C’s literal
   concatenation feature. Options considered (all of these required by FHS):

      /boot       Not present if host is booted in some strange way?
      /etc        Likely very reliable but seems risky
      /mnt        Used for images on GitHub Actions and causes CI failures
      /opt        Seems very omittable
      /srv        I’ve never actually seen it used; reliable?
      /var        Too aggressive?
      /var/spool  Long; omittable for lightweight hosts?

   [1]: https://www.pathname.com/fhs/pub/fhs-2.3.pdf */
#define WF_MNT "/srv"


/** Constants **/

/* Default bind-mounts. */
struct bind BINDS_DEFAULT[] = {
   { "/dev",                     "/dev",                     0 },
   { "/proc",                    "/proc",                    0 },
   { "/sys",                     "/sys",                     0 },
   { "/etc/hosts",               "/etc/hosts",               BD_OPTIONAL },
   { "/etc/machine-id",          "/etc/machine-id",          BD_OPTIONAL },
   { "/etc/resolv.conf",         "/etc/resolv.conf",         BD_OPTIONAL },
   /* Cray bind-mounts. See #1473. */
   { "/var/lib/hugetlbfs",       "/var/lib/hugetlbfs",       BD_OPTIONAL },
   /* Cray Gemini/Aries interconnect bind-mounts. */
   { "/etc/opt/cray/wlm_detect", "/etc/opt/cray/wlm_detect", BD_OPTIONAL },
   { "/opt/cray/wlm_detect",     "/opt/cray/wlm_detect",     BD_OPTIONAL },
   { "/opt/cray/alps",           "/opt/cray/alps",           BD_OPTIONAL },
   { "/opt/cray/udreg",          "/opt/cray/udreg",          BD_OPTIONAL },
   { "/opt/cray/ugni",           "/opt/cray/ugni",           BD_OPTIONAL },
   { "/opt/cray/xpmem",          "/opt/cray/xpmem",          BD_OPTIONAL },
   { "/var/opt/cray/alps",       "/var/opt/cray/alps",       BD_OPTIONAL },
   /* Cray Shasta/Slingshot bind-mounts. */
   { "/var/spool/slurmd",        "/var/spool/slurmd",        BD_OPTIONAL },
   { 0 }
};


/** Global variables **/

/* Variables for coordinating --join. */
struct {
   bool winner_p;
   char *sem_name;
   sem_t *sem;
   char *shm_name;
   struct {
      pid_t winner_pid;
      int proc_left_ct;  // serial-only access
   } *shared;
} join;

/* Bind mounts done so far; canonical host paths. If null, there are none. */
char **bind_mount_paths = NULL;


/** Function prototypes (private) **/

void bind_merge(const char *src, const char *dst, const char *newroot,
                bool missing_ok, const char *scratch);
void bind_mounts(const struct bind *binds, const char *newroot,
                 unsigned long flags, const char * scratch);
void join_begin(const char *join_tag);
void join_end(int join_ct);
void mounts_setup(struct container *c);
void namespace_join(pid_t pid, const char *ns);
void namespaces_join(pid_t pid);
void namespaces_setup(const struct container *c, uid_t uid_out, uid_t uid_in,
                      gid_t gid_out, gid_t gid_in);
void passwd_setup(const struct container *c);
void pivot(struct container *c);
void sem_timedwait_relative(sem_t *sem, int timeout);
void tmpfs_mount(const char *dst, const char *newroot, const char *data);


/** Functions **/

/** “Bind-mount” file or directory, but with symlink trickery to reduce the
    number of actual bind mounts.

    @param src_h       Host path to original file or directory to appear at @p
                       dst. This is *not* bind-mounted directly, but rather
                       some parent of it will be bound somewhere under @p
                       scratch. An important implication of this is that the
                       contents of that parent will be visible to any process
                       that can see @p scratch.

    @param dst_g       Guest path to “mount” point. The parent of @p dst must
                       be writeable. This will be a symlink to somewhere under
                       @p scratch. Overmount is simulated by renaming any
                       existing @p dst within the same directory (because that
                       is a convenient rename location and deleting non-empty
                       directories is complex).

    @param newroot     Host path of container root directory.

    @param missing_ok  If true, create @p dst if it does not exist. If false,
                       @p dst must already exist.

    @param scratch_g   Guest path to writeable directory usable for arbitrary
                       bind-mounts. */
void bind_merge(const char *src_h, const char *dst_g, const char *newroot,
                bool missing_ok, const char *scratch_g)
{
   char *src_h_par, *src_name;
   char *dst_g_par, *dst_h, *dst_name;
   char *mp_h, *mp_g, *target_g;

   path_split(src_h, &src_h_par, &src_name);
   path_split(dst_g, &dst_g_par, &dst_name);
   dst_h = path_join(newroot, dst_g);
   mp_g = path_join(scratch_g, replace_char(src_h_par, '/', '_'));
   mp_h = path_join(newroot, mp_g);

   // mount actual mount point if not yet mounted
   if (!path_exists(mp_h, NULL, true)) {
      Z_e (mkdir(mp_h, 0777));
      Zfe (mount(src_h_par, mp_h, NULL, MS_BIND|MS_REC|MS_PRIVATE, NULL),
           "can't bind %s at %s", src_h_par, mp_h);
   }

   // move dst out of the way if appropriate
   if (path_exists(dst_h, NULL, true))
      Z_e (rename(dst_h, cat(dst_h, ".orig")));
   else
      T__ (missing_ok);

   // create symlink
   target_g = path_join(mp_g, src_name);
   DEBUG("symlink instead of bind: %s -> %s", dst_h, target_g);
   Z_e (symlink(target_g, dst_h));
}

/** Bind-mount file or directory.

    @param src      Host path to bind-mount source.

    @param dst      Guest path to mount point. Must be an absolute path.

    @param newroot  Root directory of @p dst; must be absolute. Typically this
                    is the root of the container image, or @c "/" to indicate
                    that @p dst has the same root directory as @p src.

    @param flags    Bits configuring the mount. May contain both @c mount(2)
                    flags as well as additional flags listed below. @c MS_BIND
                    and @c MS_REC are implicit and need not be set.

    @param scratch  Guest path to writeable directory used in certain bind
                    modes. See flags below.

    The following Charliecloud-specific flags are available:

      - @c BD_SRC_OPTIONAL — If @p src does not exist, silently do nothing.

      - @c BD_DST_OPTIONAL — If @p dst does not exist, silently do nothing.

      - @c BD_DST_CREATE —   If @p dst does not exist, create it with a file
                             type where @p src can be bound. If @p dst is a
                             writeable path, it is simply created. If @p dst
                             is not writeable, @p scratch is a path to a
                             directory used by @ref mkdirs to create a
                             writeable overlay atop @p dst’s parent directory.

      - @c BD_DST_MERGE —    Rather than bind-mounting @p src at @p dst, bind
                             some ancestor directory of @p src to some
                             descendant of @p scratch, and symlink @p dst into
                             that appropriately.

    @c BD_DST_OPTIONAL and @c BD_DST_CREATE cannot be specified together.

    @note
    @p dst is separate from @p newroot to facilitate lists of bind mounts with
    a common @p newroot.

    @note
    There is no way to specify a non-recursive bind mount (i.e., without @c
    MS_REC). If @p src has no sub-mounts, then recursive and non-recursive are
    equivalent; if it does, then a non-recursive bind-mount could reveal
    overmounted files, which fails with @c EINVAL [1].

    [1]: https://man7.org/linux/man-pages/man2/mount.2.html */
void bind_mount(const char *src, const char *dst, const char *newroot,
                bindflags flags, const char *scratch)
{
   // FIXME: I think scratch might be used in two subtly different and
   // conflicting (?) ways in this function, but I haven’t looked into it
   // carefully.

   char *dst_fullc, *newrootc;
   char *dst_full = path_join(newroot, dst);

   // basic error checking
   T__ (src != NULL && dst != NULL && newroot != NULL);    // non-NULL
   T__ (src[0] != 0 && dst[0] != 0 && newroot[0] != 0);    // non-empty strings
   T__ (dst[0] == '/' && newroot[0] == '/');               // absolute paths
   Z__ (flags & BD_DST_OPTIONAL && flags & BD_DST_CREATE); // incompatible

   // no source
   if (!path_exists(src, NULL, true)) {
      Tf_ (flags & BD_SRC_OPTIONAL, "can't bind: source not found: %s", src);
      return;
   }

   // no destination (mount point)
   if (!path_exists(dst_full, NULL, true)) {
      if (flags & BD_DST_OPTIONAL)  // it’s optional; do nothing
         return;
      Tf_ (flags & BD_DST_CREATE,   // ensure we can create it
           "can't bind: destination not found: %s", dst_full);
      if (flags & BD_DST_MERGE)     // bind_merge() makes mount point itself
         mkdirs(newroot, dirname_ch(dst), bind_mount_paths, scratch);
      else {                        // create mount point here
         struct stat st;
         Z_e (stat(src, &st));
         if (S_ISDIR(st.st_mode))   // mount point must be directory
            mkdirs(newroot, dst, bind_mount_paths, scratch);
         else {                     // mount point must be regular file
            int fd;
            mkdirs(newroot, dirname_ch(dst), bind_mount_paths, scratch);
            T_e (fd = open(dst_full, O_CREAT|O_EXCL|O_WRONLY, 0600));
            Z_e (close(fd)); // close(2) returns 0 on success
         }
      }
   }

   // canonicalize and check paths
   newrootc = realpath_ch(newroot);
   dst_fullc = realpath_ch(dst_full);
   Tf_ (path_subdir_p(newrootc, dst_fullc),
        "can't bind: %s not subdirectory of %s", dst_fullc, newrootc);

   // add to mkdirs() deny-list if appropriate
   if (!streq(newroot, "/"))  // don't record if newroot is "/"
      list_append((void **)&bind_mount_paths, &dst_fullc, sizeof(char *));

   // mount
   if (flags & BD_DST_MERGE)
      bind_merge(src, dst, newrootc, flags & BD_DST_CREATE, scratch);
   else
      Zfe (mount(src, dst_fullc, NULL, MS_BIND|MS_REC|(flags&~BD_MASK), NULL),
           "can't bind %s at %s", src, dst_fullc);
}

/* Bind-mount a null-terminated array of struct bind objects. */
void bind_mounts(const struct bind *binds, const char *newroot,
                 unsigned long moreflags, const char *scratch)
{
   for (int i = 0; binds[i].src != NULL; i++)
      bind_mount(binds[i].src, binds[i].dst, newroot,
                 binds[i].flags | moreflags, scratch);
}

/* Set up new namespaces or join existing namespaces. */
void containerize(struct container *c)
{
   if (c->join_pid) {
      namespaces_join(c->join_pid);
      return;
   }
   if (c->join)
      join_begin(c->join_tag);
   if (!c->join || join.winner_p) {
      // Set up two nested user+mount namespaces: the outer so we can run
      // fusermount3 non-setuid, and the inner so we get the desired UID
      // within the container. We do this even if the image is a directory, to
      // reduce the number of code paths.
      namespaces_setup(c, geteuid(), 0, getegid(), 0);
#ifdef HAVE_SQUASHFUSE
      if (c->type == IMG_SQUASH)
         sq_fork(c);
#endif
      namespaces_setup(c, 0, c->container_uid, 0, c->container_gid);
      mounts_setup(c);
      hooks_run(c, "host", c->hooks_host);
      pivot(c);
      hooks_run(c, "guest", c->hooks_guest);
   } else {
      namespaces_join(join.shared->winner_pid);
   }
   hooks_run(c, "guest", c->hooks_all);
   if (c->join)
      join_end(c->join_ct);

}

/* Append hook function f to hook_list. When called, the hook will be passed
   d; this lets hooks receive arbitrary arguments (i.e., it’s a poor person’s
   closure). hook_list must be a member of c.

   “dup” says what to do if a hook with the same name is already in the list:

      HOOK_DUP_OK    add the hook anyway
      HOOK_DUP_SKIP  silently do nothing (i.e., don’t add the hook)
      HOOK_DUP_FAIL  fatal error  */
void hook_add(struct hook **hook_list, enum hook_dup dup,
              const char *name, hookf_t *f, void *d)
{
   struct hook h;

   if (dup == HOOK_DUP_SKIP || dup == HOOK_DUP_FAIL) {
      bool dup_found = false;
      for (int i = 0; (*hook_list)[i].name != NULL; i++)
         if (streq((*hook_list)[i].name, name)) {
            dup_found = true;
            break;
         }
      if (dup_found) {
         Tf_ (dup == HOOK_DUP_SKIP, "invalid duplicate hook: %s", name);
         return;  // skip adding hook
      }
   }

   h.name = name;
   h.f = f;
   h.data = d;

   list_append((void **)hook_list, &h, sizeof(h));
}

/** Run hooks in @p hook_list (which can be @c NULL), passing them @p c, then
    set @p hook_list to @c NULL (to make it impossible to run the list more
    than once). Use @p tag for logging.

    @p hook_list must be a member of @p c. */
void hooks_run(struct container *c, const char *tag, struct hook *hook_list)
{
   size_t hook_ct = list_count(hook_list, sizeof(hook_list[0]));

   VERBOSE("hooks: %s: found %zu hooks", tag, hook_ct);
   for (int i = 0; i < hook_ct; i++) {
      struct hook h = hook_list[i];
      DEBUG("hooks: %s: calling %d/%zu: %s", tag, i+1, hook_ct, h.name);
      h.f(c, h.data);
   }
}

char *img_name2path(const char *name, const char *storage_dir)
{
   return path_join(storage_dir,
                    path_join("img", replace_char(replace_char(name, '/', '%'),
                                                 ':', '+')));
}

/* Return image type of path, or exit with error if not a valid type. */
enum img_type image_type(const char *ref, const char *storage_dir)
{
   struct stat st;
   FILE *fp;
   char magic[4];  // four bytes, not a string

   // If there’s a directory in storage where we would expect there to be if
   // ref were an image name, assume it really is an image name.
   if (path_exists(img_name2path(ref, storage_dir), NULL, false))
      return IMG_NAME;

   // Now we know ref is a path of some kind, so find it.
   Zfe (stat(ref, &st), "can't stat: %s", ref);

   // If ref is the path to a directory, then it’s a directory.
   if (S_ISDIR(st.st_mode))
      return IMG_DIRECTORY;

   // Now we know it’s file-like enough to read. See if it has the SquashFS
   // magic number.
   fp = fopen(ref, "rb");
   Tfe (fp != NULL, "can't open: %s", ref);
   Tfe (fread(magic, sizeof(char), 4, fp) == 4, "can't read: %s", ref);
   Zfe (fclose(fp), "can't close: %s", ref);
   VERBOSE("image file magic expected: 6873 7173; actual: %x%x %x%x",
           magic[0], magic[1], magic[2], magic[3]);

   // If magic number matches, it’s a squash. Note: Magic number is 6873 7173,
   // i.e. “hsqs”. I think “sqsh” was intended but the superblock designers
   // were confused about endianness.
   // See: https://dr-emann.github.io/squashfs/
   if (memcmp(magic, "hsqs", 4) == 0)
      return IMG_SQUASH;

   // Well now we’re stumped.
   FATAL(-1, "unknown image type: %s", ref);
}

/* Begin coordinated section of namespace joining. */
void join_begin(const char *join_tag)
{
   int fd;
   join.sem_name = cat("/ch-run_sem-", join_tag);
   join.shm_name = cat("/ch-run_shm-", join_tag);

   // Serialize.
   join.sem = sem_open(join.sem_name, O_CREAT, 0600, 1);
   T_e (join.sem != SEM_FAILED);
   sem_timedwait_relative(join.sem, JOIN_TIMEOUT);

   // Am I the winner?
   fd = shm_open(join.shm_name, O_CREAT|O_EXCL|O_RDWR, 0600);
   if (fd > 0) {
      VERBOSE("join: I won");
      join.winner_p = true;
      // WARNING: The segment is resized later in join_env_save().
      Z_e (ftruncate(fd, sizeof(*join.shared)));
   } else {
      T_e (errno == EEXIST);
      VERBOSE("join: I lost");
      join.winner_p = false;
      fd = shm_open(join.shm_name, O_RDWR, 0);
      T_e (fd > 0);
   }

   join.shared = mmap(NULL, sizeof(*join.shared), PROT_READ|PROT_WRITE,
                      MAP_SHARED, fd, 0);
   T__ (join.shared != NULL);
   Z_e (close(fd));


   // Winner keeps lock; losers parallelize (winner will be done by now).
   if (!join.winner_p)
      Z_e (sem_post(join.sem));
}

/* End coordinated section of namespace joining. */
void join_end(int join_ct)
{
   if (join.winner_p) {                                // winner still serial
      VERBOSE("join: winner initializing shared data");
      join.shared->winner_pid = getpid();
      join.shared->proc_left_ct = join_ct;
   } else                                              // losers serialize
      sem_timedwait_relative(join.sem, JOIN_TIMEOUT);

   join.shared->proc_left_ct--;
   VERBOSE("join: %d peers left excluding myself", join.shared->proc_left_ct);

   if (join.shared->proc_left_ct <= 0) {
      VERBOSE("join: cleaning up IPC resources");
      Tf_ (join.shared->proc_left_ct == 0,
           "expected 0 peers left but found %d", join.shared->proc_left_ct);
      Zfe (sem_unlink(join.sem_name), "can't unlink sem: %s", join.sem_name);
      Zfe (shm_unlink(join.shm_name), "can't unlink shm: %s", join.shm_name);
   }

   Z_e (sem_post(join.sem));  // parallelize (all)

   Z_e (munmap(join.shared, sizeof(*join.shared)));
   Z_e (sem_close(join.sem));

   VERBOSE("join: done");
}


/* Set up the container filesystem tree. Namespaces must already be done. */
void mounts_setup(struct container *c)
{
   char *nr_parent, *mkdir_scratch;

   VERBOSE("creating container filesystem tree");
   LOG_IDS;
   mkdir_scratch = NULL;
   path_split(c->newroot, &nr_parent, NULL);

   // Claim new root for this namespace. Despite MS_REC, we do need both calls
   // to avoid pivot_root(2) failing with EBUSY later.
   DEBUG("claiming new root for this namespace");
   bind_mount(c->newroot, c->newroot, "/", MS_PRIVATE, NULL);
   bind_mount(nr_parent, nr_parent, "/", MS_PRIVATE, NULL);
   // Re-mount new root read-only unless --write or already read-only.
   if (!c->writable && !(access(c->newroot, W_OK) == -1 && errno == EROFS)) {
      unsigned long flags =   path_mount_flags(c->newroot)
                            | MS_REMOUNT  // Re-mount ...
                            | MS_BIND     // only this mount point ...
                            | MS_RDONLY;  // read-only.
      Z_e (mount(NULL, c->newroot, NULL, flags, NULL));
   }
   // Overlay a tmpfs if --write-fake. See for useful details:
   // https://www.kernel.org/doc/html/v5.11/filesystems/tmpfs.html
   // https://www.kernel.org/doc/html/v5.11/filesystems/overlayfs.html
   if (c->overlay_size != NULL) {
      char *options;
      struct stat st;
      VERBOSE("overlaying tmpfs for --write-fake (%s)", c->overlay_size);
      options = cat("size=", c->overlay_size);
      Zfe (mount(NULL, WF_MNT, "tmpfs", 0, options),
          "cannot mount tmpfs for overlay");
      Z_e (mkdir(WF_MNT "/upper", 0700));
      Z_e (mkdir(WF_MNT "/work", 0700));
      Z_e (mkdir(WF_MNT "/merged", 0700));
      mkdir_scratch = WF_MNT "/mkdir_overmount";
      Z_e (mkdir(mkdir_scratch, 0700));
      options = asprintf_ch(("lowerdir=%s,upperdir=%s,workdir=%s,"
                             "index=on,userxattr,volatile"),
                            c->newroot, WF_MNT "/upper", WF_MNT "/work");
      // update newroot
      Zfe (stat(c->newroot, &st),
           "can't stat new root; overmounted by tmpfs for -W?: %s", c->newroot);
      c->newroot = WF_MNT "/merged";
      Zfe (mount("weirdal", c->newroot, "overlay", 0, options),
           "can't overlay: %s, %s", c->newroot, options);
      VERBOSE("newroot updated: %s", c->newroot);
   }
   DEBUG("starting bind-mounts");
   // Bind-mount default files and directories.
   bind_mounts(BINDS_DEFAULT, c->newroot, MS_RDONLY, NULL);
   // /etc/passwd and /etc/group.
   if (!c->private_passwd)
      passwd_setup(c);
   // Container /tmp.
   if (c->private_tmp) {
      tmpfs_mount("/tmp", c->newroot, NULL);
   } else {
      bind_mount(host_tmp, "/tmp", c->newroot, 0, NULL);
   }
   // Bind-mount user’s home directory at /home/$USER if requested.
   if (c->host_home) {
      T__ (c->overlay_size != NULL);
      bind_mount(c->host_home, cat("/home/", username), c->newroot,
                 BD_DST_CREATE, mkdir_scratch);
   }
   // Bind-mount user-specified directories.
   bind_mounts(c->binds, c->newroot, 0, mkdir_scratch);
}

/* Join a specific namespace. */
void namespace_join(pid_t pid, const char *ns)
{
   char *path;
   int fd;

   path = asprintf_ch("/proc/%d/ns/%s", pid, ns);
   fd = open(path, O_RDONLY);
   if (fd == -1) {
      if (errno == ENOENT)
         Tf_ (0, "join: no PID %d: %s not found", pid, path);
      else
         Tfe (0, "join: can't open %s", path);
   }
   /* setns(2) seems to be involved in some kind of race with syslog(3).
      Rarely, when configured with --enable-syslog, the call fails with
      EINVAL. We never figured out a proper fix, so just retry a few times in
      a loop. See issue #1270. */
   for (int i = 1; setns(fd, 0) != 0; i++)
      if (i >= 5) {
         FATAL(-1, "can’t join %s namespace of pid %d", ns, pid);
      } else {
         WARNING("can’t join %s namespace; trying again", ns);
         sleep(1);
      }
}

/* Join the existing namespaces containing process pid, which could be the
   join winner or another process. */
void namespaces_join(pid_t pid)
{
   VERBOSE("joining namespaces of pid %d", pid);
   namespace_join(pid, "user");
   namespace_join(pid, "mnt");
}

/* Activate the desired isolation namespaces. */
void namespaces_setup(const struct container *c, uid_t uid_out, uid_t uid_in,
                      gid_t gid_out, gid_t gid_in)
{
   int fd;

   VERBOSE("setting up namespaces: %d:%d -> %d:%d",
           uid_out, gid_out, uid_in, gid_in);
   LOG_IDS;
   Zfe (unshare(CLONE_NEWNS|CLONE_NEWUSER), "can't init user+mount namespaces");
   LOG_IDS;

   /* Write UID map. What we are allowed to put here is quite limited. Because
      we do not have CAP_SETUID in the *parent* user namespace, we can map
      exactly one UID: an arbitrary container UID to our EUID in the parent
      namespace.

      This is sufficient to change our UID within the container; no setuid(2)
      or similar required. This is because the EUID of the process in the
      parent namespace is unchanged, so the kernel uses our new 1-to-1 map to
      convert that EUID into the container UID for most (maybe all)
      purposes. */
   T_e (-1 != (fd = open("/proc/self/uid_map", O_WRONLY)));
   T_e (1 <= dprintf(fd, "%d %d 1\n", uid_in, uid_out));
   Z_e (close(fd));
   LOG_IDS;

   T_e (-1 != (fd = open("/proc/self/setgroups", O_WRONLY)));
   T_e (1 <= dprintf(fd, "deny\n"));
   Z_e (close(fd));
   T_e (-1 != (fd = open("/proc/self/gid_map", O_WRONLY)));
   T_e (1 <= dprintf(fd, "%d %d 1\n", gid_in, gid_out));
   Z_e (close(fd));
   LOG_IDS;
}

/* Build /etc/passwd and /etc/group files and bind-mount them into newroot.

   /etc/passwd contains root, nobody, and an entry for the container UID,
   i.e., three entries, or two if the container UID is 0 or 65534. We copy the
   host's user data for the container UID, if that exists, and use dummy data
   otherwise (see issue #649). /etc/group works similarly: root, nogroup, and
   an entry for the container GID.

   We build new files to capture the relevant host username and group name
   mappings regardless of where they come from. We used to simply bind-mount
   the host's /etc/passwd and /etc/group, but this fails for LDAP at least;
   see issue #212. After bind-mounting, we remove the files from the host;
   they persist inside the container and then disappear completely when the
   container exits. */
void passwd_setup(const struct container *c)
{
   int fd;
   char *path;
   struct group *g;
   struct passwd *p;

   // /etc/passwd
   path = cat(host_tmp, "/ch-run_passwd.XXXXXX");
   T_e (-1 != (fd = mkstemp(path)));  // mkstemp(3) writes path
   if (c->container_uid != 0)
      T_e (1 <= dprintf(fd, "root:x:0:0:root:/root:/bin/sh\n"));
   if (c->container_uid != 65534)
      T_e (1 <= dprintf(fd, "nobody:x:65534:65534:nobody:/:/bin/false\n"));
   errno = 0;
   p = getpwuid(c->container_uid);
   if (p) {
      T_e (1 <= dprintf(fd, "%s:x:%u:%u:%s:/:/bin/sh\n", p->pw_name,
                        c->container_uid, c->container_gid, p->pw_gecos));
   } else {
      Zfe (errno, "getpwuid(3) failed");
      VERBOSE("UID %d not found; using dummy info", c->container_uid);
      T_e (1 <= dprintf(fd, "%s:x:%u:%u:%s:/:/bin/sh\n", "charlie",
                        c->container_uid, c->container_gid, "Charlie"));
   }
   Z_e (close(fd));
   bind_mount(path, "/etc/passwd", c->newroot, 0, NULL);
   Z_e (unlink(path));

   // /etc/group
   path = cat(host_tmp, "/ch-run_group.XXXXXX");
   T_e (-1 != (fd = mkstemp(path)));
   if (c->container_gid != 0)
      T_e (1 <= dprintf(fd, "root:x:0:\n"));
   if (c->container_gid != 65534)
      T_e (1 <= dprintf(fd, "nogroup:x:65534:\n"));
   errno = 0;
   g = getgrgid(c->container_gid);
   if (g) {
      T_e (1 <= dprintf(fd, "%s:x:%u:\n", g->gr_name, c->container_gid));
   } else {
      Zfe (errno, "getgrgid(3) failed");
      VERBOSE("GID %d not found; using dummy info", c->container_gid);
      T_e (1 <= dprintf(fd, "%s:x:%u:\n", "charliegroup", c->container_gid));
   }
   Z_e (close(fd));
   bind_mount(path, "/etc/group", c->newroot, 0, NULL);
   Z_e (unlink(path));
}

/* Pivot into the container. Note that pivot_root(2) requires a complex dance
   to work, i.e., to avoid multiple undocumented error conditions. This dance
   is explained in detail in bin/ch-checkns.c. */
void pivot(struct container *c)
{
   char *nr_parent, *nr_base;

   VERBOSE("pivoting into container");
   path_split(c->newroot, &nr_parent, &nr_base);

   // Overmount / to avoid EINVAL if it’s a rootfs.
   Z_e (chdir(nr_parent));
   Z_e (mount(nr_parent, "/", NULL, MS_MOVE, NULL));
   Z_e (chroot("."));
   // Pivot into the new root. Use /dev because it’s available even in
   // extremely minimal images.
   c->newroot = cat("/", nr_base);
   Zfe (chdir(c->newroot), "can't chdir into new root");
   Zfe (syscall(SYS_pivot_root, c->newroot, path_join(c->newroot, "dev")),
        "can't pivot_root(2)");
   Zfe (chroot("."), "can't chroot(2) into new root");
   Zfe (umount2("/dev", MNT_DETACH), "can't umount old root");
}

/** Replace the current containerized process with user command.

      @param c            Container configuration.

      @param argv         Command and arguments.

    @p argv is passed unchanged to the underlying @c execvpe(3), whose first
    argument is @c argv[0] per convention.

    @note We use the *container* @c PATH to locate the user command because
    it’s inside the container image. This includes unsetting `PATH` if not set
    for the container. */
noreturn run_user_command(struct container *c, char *argv[])
{
   LOG_IDS;

   if (c->initial_dir != NULL)
      Zfe (chdir(c->initial_dir), "can't cd to %s", c->initial_dir);
   cenv_set(c, "PWD", getcwd_ch());

   VERBOSE("executing: %s", argv_to_string(argv));

   Zfe (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0), "can't set no_new_privs");
   if (verbose < LL_INFO)
      T_e (freopen("/dev/null", "w", stdout));
   if (verbose < LL_STDERR)
      T_e (freopen("/dev/null", "w", stderr));
   mem_log("usrx");
   if (cenv_get(c, "PATH"))
      Z__ (setenv("PATH", cenv_get(c, "PATH"), true));
   else
      Z__ (unsetenv("PATH"));
   execvpe(argv[0], argv, c->environ);  // only returns if error
   ERROR(errno, "can't execvpe(3): %s", argv[0]);
   exit(EXIT_ERR_CMD);
}

/* Wait for semaphore sem for up to timeout seconds. If timeout or an error,
   exit unsuccessfully. */
void sem_timedwait_relative(sem_t *sem, int timeout)
{
   struct timespec deadline;

   // sem_timedwait() requires a deadline rather than a timeout.
   Z_e (clock_gettime(CLOCK_REALTIME, &deadline));
   deadline.tv_sec += timeout;
   Zfe (sem_timedwait(sem, &deadline), "failure waiting for join lock");
}

/* Mount a tmpfs at the given path. */
void tmpfs_mount(const char *dst, const char *newroot, const char *data)
{
   char *dst_full = cat(newroot, dst);

   Zfe (mount(NULL, dst_full, "tmpfs", 0, data),
        "can't mount tmpfs at %s", dst_full);
}