`Could not exchange paths: Invalid argument` on Linux with ZFS (renameat2 with RENAME_EXCHANGE)

[kpcyrd]

It seems on Linux with ZFS the renameat2 syscall isn't fully implemented, this causes xch to fail even if libxch::xch_non_atomic is used. This was found in kpcyrd/spotify-launcher#10. :)

renameat2(AT_FDCWD, "foo", AT_FDCWD, "bar", RENAME_EXCHANGE) = -1 EINVAL (Invalid argument)

Errors

From the manpage, this means one of the following:

       EINVAL An invalid flag was specified in flags.

       EINVAL Both RENAME_NOREPLACE and RENAME_EXCHANGE were specified in flags.

       EINVAL Both RENAME_WHITEOUT and RENAME_EXCHANGE were specified in flags.

       EINVAL The filesystem does not support one of the flags in flags.

I'm suspecting RENAME_EXCHANGE is not supported on ZFS.

Reproduce

On an ubuntu 22.04 system with zfs selected in the installer:

user@user-VirtualBox:~/repos/xch$ mkdir -p foo bar
user@user-VirtualBox:~/repos/xch$ target/debug/xch foo bar
error: could not swap files: Could not exchange paths: Invalid argument
user@user-VirtualBox:~/repos/xch$ target/debug/xch -n foo bar
error: could not swap files: Could not exchange paths: Invalid argument

Filesystems (ubuntu 22.04 zfs)

user@user-VirtualBox:~/repos/xch$ mount | grep zfs
rpool/ROOT/ubuntu_2carst on / type zfs (rw,relatime,xattr,posixacl)
rpool/USERDATA/root_q8v6j2 on /root type zfs (rw,relatime,xattr,posixacl)
rpool/USERDATA/user_q8v6j2 on /home/user type zfs (rw,relatime,xattr,posixacl)
rpool/ROOT/ubuntu_2carst/srv on /srv type zfs (rw,relatime,xattr,posixacl)
rpool/ROOT/ubuntu_2carst/usr/local on /usr/local type zfs (rw,relatime,xattr,posixacl)
rpool/ROOT/ubuntu_2carst/var/games on /var/games type zfs (rw,relatime,xattr,posixacl)
rpool/ROOT/ubuntu_2carst/var/log on /var/log type zfs (rw,relatime,xattr,posixacl)
rpool/ROOT/ubuntu_2carst/var/lib on /var/lib type zfs (rw,relatime,xattr,posixacl)
rpool/ROOT/ubuntu_2carst/var/mail on /var/mail type zfs (rw,relatime,xattr,posixacl)
rpool/ROOT/ubuntu_2carst/var/snap on /var/snap type zfs (rw,relatime,xattr,posixacl)
rpool/ROOT/ubuntu_2carst/var/www on /var/www type zfs (rw,relatime,xattr,posixacl)
rpool/ROOT/ubuntu_2carst/var/spool on /var/spool type zfs (rw,relatime,xattr,posixacl)
rpool/ROOT/ubuntu_2carst/var/lib/AccountsService on /var/lib/AccountsService type zfs (rw,relatime,xattr,posixacl)
rpool/ROOT/ubuntu_2carst/var/lib/NetworkManager on /var/lib/NetworkManager type zfs (rw,relatime,xattr,posixacl)
rpool/ROOT/ubuntu_2carst/var/lib/apt on /var/lib/apt type zfs (rw,relatime,xattr,posixacl)
rpool/ROOT/ubuntu_2carst/var/lib/dpkg on /var/lib/dpkg type zfs (rw,relatime,xattr,posixacl)
bpool/BOOT/ubuntu_2carst on /boot type zfs (rw,nodev,relatime,xattr,posixacl)
user@user-VirtualBox:~/repos/xch$ 

strace

user@user-VirtualBox:~/repos/xch$ strace -f target/debug/xch foo bar
execve("target/debug/xch", ["target/debug/xch", "foo", "bar"], 0x7ffc79219ff8 /* 53 vars */) = 0
brk(NULL)                               = 0x55f81cd10000
arch_prctl(0x3001 /* ARCH_??? */, 0x7ffceee65230) = -1 EINVAL (Invalid argument)
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f04b9335000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=53335, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 53335, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f04b9327000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libgcc_s.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=125488, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 127720, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f04b9307000
mmap(0x7f04b930a000, 94208, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7f04b930a000
mmap(0x7f04b9321000, 16384, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1a000) = 0x7f04b9321000
mmap(0x7f04b9325000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1d000) = 0x7f04b9325000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libm.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=940560, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 942344, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f04b9220000
mmap(0x7f04b922e000, 507904, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xe000) = 0x7f04b922e000
mmap(0x7f04b92aa000, 372736, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x8a000) = 0x7f04b92aa000
mmap(0x7f04b9305000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xe4000) = 0x7f04b9305000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P\237\2\0\0\0\0\0"..., 832) = 832
pread64(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"..., 784, 64) = 784
pread64(3, "\4\0\0\0 \0\0\0\5\0\0\0GNU\0\2\0\0\300\4\0\0\0\3\0\0\0\0\0\0\0"..., 48, 848) = 48
pread64(3, "\4\0\0\0\24\0\0\0\3\0\0\0GNU\0i8\235HZ\227\223\333\350s\360\352,\223\340."..., 68, 896) = 68
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=2216304, ...}, AT_EMPTY_PATH) = 0
pread64(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"..., 784, 64) = 784
mmap(NULL, 2260560, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f04b8ff8000
mmap(0x7f04b9020000, 1658880, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x28000) = 0x7f04b9020000
mmap(0x7f04b91b5000, 360448, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1bd000) = 0x7f04b91b5000
mmap(0x7f04b920d000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x214000) = 0x7f04b920d000
mmap(0x7f04b9213000, 52816, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f04b9213000
close(3)                                = 0
mmap(NULL, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f04b8ff5000
arch_prctl(ARCH_SET_FS, 0x7f04b8ff5900) = 0
set_tid_address(0x7f04b8ff5bd0)         = 12191
set_robust_list(0x7f04b8ff5be0, 24)     = 0
rseq(0x7f04b8ff62a0, 0x20, 0, 0x53053053) = 0
mprotect(0x7f04b920d000, 16384, PROT_READ) = 0
mprotect(0x7f04b9305000, 4096, PROT_READ) = 0
mprotect(0x7f04b9325000, 4096, PROT_READ) = 0
mprotect(0x55f81c53a000, 24576, PROT_READ) = 0
mprotect(0x7f04b936f000, 8192, PROT_READ) = 0
prlimit64(0, RLIMIT_STACK, NULL, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
munmap(0x7f04b9327000, 53335)           = 0
poll([{fd=0, events=0}, {fd=1, events=0}, {fd=2, events=0}], 3, 0) = 0 (Timeout)
rt_sigaction(SIGPIPE, {sa_handler=SIG_IGN, sa_mask=[PIPE], sa_flags=SA_RESTORER|SA_RESTART, sa_restorer=0x7f04b903a520}, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0}, 8) = 0
rt_sigaction(SIGSEGV, NULL, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0}, 8) = 0
rt_sigaction(SIGSEGV, {sa_handler=0x55f81c503020, sa_mask=[], sa_flags=SA_RESTORER|SA_ONSTACK|SA_SIGINFO, sa_restorer=0x7f04b903a520}, NULL, 8) = 0
rt_sigaction(SIGBUS, NULL, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0}, 8) = 0
rt_sigaction(SIGBUS, {sa_handler=0x55f81c503020, sa_mask=[], sa_flags=SA_RESTORER|SA_ONSTACK|SA_SIGINFO, sa_restorer=0x7f04b903a520}, NULL, 8) = 0
sigaltstack(NULL, {ss_sp=NULL, ss_flags=SS_DISABLE, ss_size=0}) = 0
mmap(NULL, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0x7f04b9332000
mprotect(0x7f04b9332000, 4096, PROT_NONE) = 0
sigaltstack({ss_sp=0x7f04b9333000, ss_flags=0, ss_size=8192}, NULL) = 0
getrandom("\xec\x72\x17\xf9\xf8\x86\x56\x36", 8, GRND_NONBLOCK) = 8
brk(NULL)                               = 0x55f81cd10000
brk(0x55f81cd31000)                     = 0x55f81cd31000
openat(AT_FDCWD, "/proc/self/maps", O_RDONLY|O_CLOEXEC) = 3
prlimit64(0, RLIMIT_STACK, NULL, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
newfstatat(3, "", {st_mode=S_IFREG|0444, st_size=0, ...}, AT_EMPTY_PATH) = 0
read(3, "55f81c490000-55f81c49b000 r--p 0"..., 1024) = 1024
read(3, "7f04b9211000 r--p 00214000 00:1a"..., 1024) = 1024
read(3, "\n7f04b9321000-7f04b9325000 r--p "..., 1024) = 1024
read(3, "x-x86-64.so.2\n7ffceee46000-7ffce"..., 1024) = 339
close(3)                                = 0
sched_getaffinity(12191, 32, [0])       = 8
renameat2(AT_FDCWD, "foo", AT_FDCWD, "bar", RENAME_EXCHANGE) = -1 EINVAL (Invalid argument)
write(2, "error: could not swap files: ", 29error: could not swap files: ) = 29
write(2, "Could not exchange paths", 24Could not exchange paths) = 24
write(2, ": ", 2: )                       = 2
write(2, "Invalid argument", 16Invalid argument)        = 16
write(2, "\n", 1
)                       = 1
sigaltstack({ss_sp=NULL, ss_flags=SS_DISABLE, ss_size=8192}, NULL) = 0
munmap(0x7f04b9332000, 12288)           = 0
exit_group(1)                           = ?
+++ exited with 1 +++
user@user-VirtualBox:~/repos/xch$ 

Thanks!

[kpcyrd]

Related github issue: openzfs/zfs#12209

[kpcyrd]

There's another report with RENAME_EXCHANGE being unavailable on ecryptfs too: kpcyrd/spotify-launcher#10 (comment)