Unable to start runc when setting uidmappings/gidmappings

[wangkirin]

I try to set uidmappings and gidmappings in config.json to test the user namespace, but when I start runc , there is following error information:

Timestamp: 2015-09-08 08:54:40.052944261 +0800 CST
Code: System error

Message: operation not permitted

Frames:

---
0: setupRootfs
Package: github.com/opencontainers/runc/libcontainer
File: rootfs_linux.go@39

---
1: Init
Package: github.com/opencontainers/runc/libcontainer.(*linuxStandardInit)
File: standard_init_linux.go@57

---
2: StartInitialization
Package: github.com/opencontainers/runc/libcontainer.(*LinuxFactory)
File: factory_linux.go@242

---
3: init·1
Package: main
File: start.go@59

---
4: init
Package: main
File: utils.go@177

---
5: main
Package: runtime
File: proc.go@58

---
6: goexit
Package: runtime
File: asm_amd64.s@2232
WARN[0000] exit status 1                                
FATA[0000] Container start failed: [8] System error: operation not permitted 

my config.json is :

{
  "version": "pre-draft",
  "platform": {
    "os": "linux",
    "arch": "amd64"
  },
  "process": {
    "terminal": true,
    "user": {
      "uid": 0,
      "gid": 0,
      "additionalGids": null
    },
    "args": [
      "bash"
    ],
    "env": [
      "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
      "TERM=xterm"
    ],
    "cwd": ""
  },
  "root": {
    "path": "rootfs",
    "readonly": true
  },
  "hostname": "shell",
  "mounts": [
    {
      "type": "proc",
      "source": "proc",
      "destination": "/proc",
      "options": ""
    },
    {
      "type": "tmpfs",
      "source": "tmpfs",
      "destination": "/dev",
      "options": "nosuid,strictatime,mode=755,size=65536k"
    },
    {
      "type": "devpts",
      "source": "devpts",
      "destination": "/dev/pts",
      "options": "nosuid,noexec,newinstance,ptmxmode=0666,mode=0620,gid=5"
    },
    {
      "type": "tmpfs",
      "source": "shm",
      "destination": "/dev/shm",
      "options": "nosuid,noexec,nodev,mode=1777,size=65536k"
    },
    {
      "type": "mqueue",
      "source": "mqueue",
      "destination": "/dev/mqueue",
      "options": "nosuid,noexec,nodev"
    },
    {
      "type": "sysfs",
      "source": "sysfs",
      "destination": "/sys",
      "options": "nosuid,noexec,nodev"
    },
    {
      "type": "cgroup",
      "source": "cgroup",
      "destination": "/sys/fs/cgroup",
      "options": "nosuid,noexec,nodev,relatime,ro"
    }
  ],
  "hooks": {
    "prestart": null,
    "poststop": null
  },
  "linux": {
    "uidMappings": [
      {
        "hostID": 1002,
        "containerID": 0,
        "size": 10
      }
    ],
    "gidMappings": [
      {
        "hostID": 1002,
        "containerID": 0,
        "size": 10
      }
    ],
    "rlimits": [
      {
        "type": 7,
        "hard": 1024,
        "soft": 1024
      }
    ],
    "sysctl": null,
    "resources": {
      "disableOOMKiller": false,
      "memory": {
        "limit": 0,
        "reservation": 0,
        "swap": 0,
        "kernel": 0,
        "swappiness": -1
      },
      "cpu": {
        "shares": 0,
        "quota": 0,
        "period": 0,
        "realtimeRuntime": 0,
        "realtimePeriod": 0,
        "cpus": "",
        "mems": ""
      },
      "blockIO": {
        "blkioWeight": 0,
        "blkioWeightDevice": "",
        "blkioThrottleReadBpsDevice": "",
        "blkioThrottleWriteBpsDevice": "",
        "blkioThrottleReadIopsDevice": "",
        "blkioThrottleWriteIopsDevice": ""
      },
      "hugepageLimits": null,
      "network": {
        "classId": "",
        "priorities": null
      }
    },
    "namespaces": [
      {
        "type": "pid",
        "path": ""
      },
      {
        "type": "network",
        "path": ""
      },
      {
        "type": "ipc",
        "path": ""
      },
      {
        "type": "user",
        "path": ""
      },
      {
        "type": "uts",
        "path": ""
      },
      {
        "type": "mount",
        "path": ""
      }
    ],
    "capabilities": [
      "AUDIT_WRITE",
      "KILL",
      "SETPCAP",
      "SYS_MODULE",
      "SYS_RAWIO",
      "SYS_PACCT",
      "SYS_ADMIN",
      "SYS_NICE",
      "SYS_RESOURCE",
      "SYS_TIME",
      "SYS_TTY_CONFIG",
      "MKNOD",
      "AUDIT_CONTROL",
      "MAC_OVERRIDE",
      "MAC_ADMIN",
      "NET_ADMIN",
      "SYSLOG",
      "CHOWN",
      "NET_RAW",
      "DAC_OVERRIDE",
      "FOWNER",
      "DAC_READ_SEARCH",
      "FSETID",
      "SETGID",
      "SETUID",
      "LINUX_IMMUTABLE",
      "NET_BIND_SERVICE",
      "NET_BROADCAST",
      "IPC_LOCK",
      "IPC_OWNER",
      "SYS_CHROOT"
    ],
    "devices": [
      {
        "type": 99,
        "path": "/dev/null",
        "major": 1,
        "minor": 3,
        "permissions": "rwm",
        "fileMode": 438,
        "uid": 0,
        "gid": 0
      },
      {
        "type": 99,
        "path": "/dev/random",
        "major": 1,
        "minor": 8,
        "permissions": "rwm",
        "fileMode": 438,
        "uid": 0,
        "gid": 0
      },
      {
        "type": 99,
        "path": "/dev/full",
        "major": 1,
        "minor": 7,
        "permissions": "rwm",
        "fileMode": 438,
        "uid": 0,
        "gid": 0
      },
      {
        "type": 99,
        "path": "/dev/tty",
        "major": 5,
        "minor": 0,
        "permissions": "rwm",
        "fileMode": 438,
        "uid": 0,
        "gid": 0
      },
      {
        "type": 99,
        "path": "/dev/zero",
        "major": 1,
        "minor": 5,
        "permissions": "rwm",
        "fileMode": 438,
        "uid": 0,
        "gid": 0
      },
      {
        "type": 99,
        "path": "/dev/urandom",
        "major": 1,
        "minor": 9,
        "permissions": "rwm",
        "fileMode": 438,
        "uid": 0,
        "gid": 0
      }
    ],
    "apparmorProfile": "",
    "selinuxProcessLabel": "",
    "seccomp": {
      "defaultAction": "SCMP_ACT_ALLOW",
      "syscalls": []
    },
    "rootfsPropagation": ""
  }
}

[wking]

On Mon, Sep 07, 2015 at 06:09:32PM -0700, Wang Qilin wrote:

I try to set uidmappings and gidmappings in config.json to
test the user namespace…
…
5: main
Package: runtime

File: proc.go@58

6: goexit
Package: runtime
File: asm_amd64.s@2232
WARN[0000] exit status 1
FATA[0000] Container start failed: [8] System error: operation not permitted

This is probably a duplicate of #225. Try dropping the ‘gid’ entry
from your devpts mount options, and dropping ‘ro’ from your cgroups
mount options.

[wangkirin]

@wking ，thank you! you mean #225 ? I change the config.json as your advice ,

{
      "type": "devpts",
      "source": "devpts",
      "destination": "/dev/pts",
      "options": "nosuid,noexec,newinstance,ptmxmode=0666,mode=0620"
    },
{
      "type": "cgroup",
      "source": "cgroup",
      "destination": "/sys/fs/cgroup",
      "options": "nosuid,noexec,nodev,relatime"
    }

but exists other error:

Code: System error

Message: can't create pivot_root dir , error mkdir /home/wang/runctest/rootfs/.pivot_root887033361: permission denied

Frames:
---
0: setupRootfs
Package: github.com/opencontainers/runc/libcontainer
File: rootfs_linux.go@68
---
1: Init
Package: github.com/opencontainers/runc/libcontainer.(*linuxStandardInit)
File: standard_init_linux.go@57
---
2: StartInitialization
Package: github.com/opencontainers/runc/libcontainer.(*LinuxFactory)
File: factory_linux.go@242
---
3: init·1
Package: main
File: start.go@59
---
4: init
Package: main
File: utils.go@177
---
5: main
Package: runtime
File: proc.go@58
---
6: goexit
Package: runtime
File: asm_amd64.s@2232
WARN[0000] exit status 1                                
FATA[0000] Container start failed: [8] System error: can't create pivot_root dir , error mkdir /home/wang/runctest/rootfs/.pivot_root887033361: permission denied 

[hqhq]

@wangkirin Do you have uid 1002 on your host? I think that's the problem.

[hqhq]

Read only cgroup mount should work in userns, current problem is caused by: #136

But I don't know why we can't remount /sys/fs/cgroup in userns, maybe related with that mentioned kernel fix , haven't checked yet.

[wangkirin]

@hqhq YES, 1002 is the uid of current user in my OS

[wking]

On Tue, Sep 08, 2015 at 12:25:44AM -0700, Wang Qilin wrote:

@wking ，thank you! you mean #225 ?

Yes, sorry. I've updated my inital comment to avoid more confusion
;).

Message: can't create pivot_root dir , error mkdir
/home/wang/runctest/rootfs/.pivot_root887033361: permission denied

I see a few runC issues that mention pivot_root, but none that talk
about this error. So this may be a new issue. For what it's worth, I
can't reproduce it with your altered config using a runC based on
0f85e4e and a vanilla 4.1 kernel.

[hqhq]

@wangkirin I used your config and got exactly the same error, then I found out that's because I don't have uid 1002 on my host, so I changed it to 1000 which is my current user's id, then runc start successfully.

[wangkirin]

@hqhq @wking
Inspired by @hqhq , I change the uid to 0 and the runc can start successfully, it reminds me I have changed the owner of rootfs folder to root:root. Then I change the owner of rootfs back to the
corresponding user of uid 1002 , and also change the the config.json back to uid 1002 , it works.
I think that's why the error occurs

[hqhq]

@wangkirin Still wired, runc won't request the owner of rootfs to be the user on the host, root:root should work. But I do see there are some problems about owner of rootfs in the container with use userns.

$ sudo runc start config.json.userns
root@shell:/# ls -l
total 64
drwxr-xr-x.   2 nobody nogroup 4096 Feb 18  2015 bin
drwxr-xr-x.   2 nobody nogroup 4096 Apr 10  2014 boot
drwxr-xr-x.   5 root   root     360 Sep  9 03:27 dev
drwxr-xr-x.  61 nobody nogroup 4096 Jul  6 02:45 etc
drwxr-xr-x.   2 nobody nogroup 4096 Apr 10  2014 home
drwxr-xr-x.  12 nobody nogroup 4096 Feb 18  2015 lib
drwxr-xr-x.   2 nobody nogroup 4096 Feb 18  2015 lib64
drwxr-xr-x.   2 nobody nogroup 4096 Feb 18  2015 media
drwxr-xr-x.   2 nobody nogroup 4096 Apr 10  2014 mnt
drwxr-xr-x.   2 nobody nogroup 4096 Feb 18  2015 opt
dr-xr-xr-x. 257 nobody nogroup    0 Sep  9 03:27 proc
drwxr-xr-x.   2 nobody nogroup 4096 Feb 18  2015 root
drwxr-xr-x.   7 nobody nogroup 4096 Feb 18  2015 run
drwxr-xr-x.   2 nobody nogroup 4096 Feb 21  2015 sbin
drwxr-xr-x.   2 nobody nogroup 4096 Feb 18  2015 srv
dr-xr-xr-x.  13 nobody nogroup    0 Sep  9 03:27 sys
drwxr-xr-x.   2 nobody nogroup 4096 Feb 18  2015 tmp
drwxr-xr-x.  10 nobody nogroup 4096 Feb 21  2015 usr
drwxr-xr-x.  11 nobody nogroup 4096 Feb 21  2015 var

Maybe ping @estesp see if that's a regression.

[ashahab-altiscale]

@hqhq In userns you cannot have the host root(or any unmapped user) as the owner of your files(except proc, as it's not virtualized). Any files owned by an unmapped user will be unusable by all users inside the container.

[vito]

Ran into this same issue (EPERM when mounting /proc); it ended up being because of a /proc/xen mount point. Unmounting it in a separate mount namespace (unshare -m) and then running runc from that namespace fixed it.

[cyphar]

Since this issue hasn't had any activity in the past year, I'm closing it. Please feel free to re-open it.

As for the actual issue, I believe this has been fixed (I've tested the user namespace code quite extensively in #774 -- and it works in quite a lot of cases). It was quite likely that you had some permission issue with rootfs, but our pivot_root has been fixed to no longer require using mkdir.