Patch series "userfaultfd: support minor fault handling for shmem", v2. Overview ======== See my original series [1] for a detailed overview of minor fault handling in general. The feature in this series works exactly like the hugetblfs version (from userspace's perspective). I'm sending this as a separate series because: - The original minor fault handling series has a full set of R-Bs, and seems close to being merged. So, it seems reasonable to start looking at this next step, which extends the basic functionality. - shmem is different enough that this series may require some additional work before it's ready, and I don't want to delay the original series unnecessarily by bundling them together. Use Case ======== In some cases it is useful to have VM memory backed by tmpfs instead of hugetlbfs. So, this feature will be used to support the same VM live migration use case described in my original series. Additionally, Android folks (Lokesh Gidra <lokeshgidra@google.com>) hope to optimize the Android Runtime garbage collector using this feature: "The plan is to use userfaultfd for concurrently compacting the heap. With this feature, the heap can be shared-mapped at another location where the GC-thread(s) could continue the compaction operation without the need to invoke userfault ioctl(UFFDIO_COPY) each time. OTOH, if and when Java threads get faults on the heap, UFFDIO_CONTINUE can be used to resume execution. Furthermore, this feature enables updating references in the 'non-moving' portion of the heap efficiently. Without this feature, uneccessary page copying (ioctl(UFFDIO_COPY)) would be required." [1] https://lore.kernel.org/linux-fsdevel/20210301222728.176417-1-axelrasmussen@google.com/T/#t This patch (of 5): Modify the userfaultfd register API to allow registering shmem VMAs in minor mode. Modify the shmem mcopy implementation to support UFFDIO_CONTINUE in order to resolve such faults. Combine the shmem mcopy handler functions into a single shmem_mcopy_atomic_pte, which takes a mode parameter. This matches how the hugetlbfs implementation is structured, and lets us remove a good chunk of boilerplate. Link: https://lkml.kernel.org/r/20210302000133.272579-1-axelrasmussen@google.com Link: https://lkml.kernel.org/r/20210302000133.272579-2-axelrasmussen@google.com Signed-off-by: Axel Rasmussen <axelrasmussen@google.com> Cc: Alexander Viro <viro@zeniv.linux.org.uk> Cc: Andrea Arcangeli <aarcange@redhat.com> Cc: Hugh Dickins <hughd@google.com> Cc: Jerome Glisse <jglisse@redhat.com> Cc: Joe Perches <joe@perches.com> Cc: Lokesh Gidra <lokeshgidra@google.com> Cc: Mike Rapoport <rppt@linux.vnet.ibm.com> Cc: Peter Xu <peterx@redhat.com> Cc: Shaohua Li <shli@fb.com> Cc: Shuah Khan <shuah@kernel.org> Cc: Wang Qing <wangqing@vivo.com> Cc: Brian Geffon <bgeffon@google.com> Cc: Cannon Matthews <cannonmatthews@google.com> Cc: "Dr . David Alan Gilbert" <dgilbert@redhat.com> Cc: David Rientjes <rientjes@google.com> Cc: Michel Lespinasse <walken@google.com> Cc: Mina Almasry <almasrymina@google.com> Cc: Oliver Upton <oupton@google.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Stephen Rothwell <sfr@canb.auug.org.au> (cherry picked from commit 4cc6e15679966aa49afc5b114c3c83ba0ac39b05 https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git akpm) Link: https://lore.kernel.org/patchwork/patch/1388146/ Conflicts: mm/shmem.c (1. Manual rebase 2. Enclosed shmem_copy_atomic_pte() with CONFIG_USERFAULTFD to avoid compile erros when USERFAULTFD is not enabled.) Signed-off-by: Lokesh Gidra <lokeshgidra@google.com> Bug: 160737021 Bug: 169683130 Change-Id: Idcd822b2a124a089121b9ad8c65061f6979126ec
300 lines
8.7 KiB
C
300 lines
8.7 KiB
C
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
|
|
/*
|
|
* include/linux/userfaultfd.h
|
|
*
|
|
* Copyright (C) 2007 Davide Libenzi <davidel@xmailserver.org>
|
|
* Copyright (C) 2015 Red Hat, Inc.
|
|
*
|
|
*/
|
|
|
|
#ifndef _LINUX_USERFAULTFD_H
|
|
#define _LINUX_USERFAULTFD_H
|
|
|
|
#include <linux/types.h>
|
|
|
|
/*
|
|
* If the UFFDIO_API is upgraded someday, the UFFDIO_UNREGISTER and
|
|
* UFFDIO_WAKE ioctls should be defined as _IOW and not as _IOR. In
|
|
* userfaultfd.h we assumed the kernel was reading (instead _IOC_READ
|
|
* means the userland is reading).
|
|
*/
|
|
#define UFFD_API ((__u64)0xAA)
|
|
#define UFFD_API_REGISTER_MODES (UFFDIO_REGISTER_MODE_MISSING | \
|
|
UFFDIO_REGISTER_MODE_WP | \
|
|
UFFDIO_REGISTER_MODE_MINOR)
|
|
#define UFFD_API_FEATURES (UFFD_FEATURE_PAGEFAULT_FLAG_WP | \
|
|
UFFD_FEATURE_EVENT_FORK | \
|
|
UFFD_FEATURE_EVENT_REMAP | \
|
|
UFFD_FEATURE_EVENT_REMOVE | \
|
|
UFFD_FEATURE_EVENT_UNMAP | \
|
|
UFFD_FEATURE_MISSING_HUGETLBFS | \
|
|
UFFD_FEATURE_MISSING_SHMEM | \
|
|
UFFD_FEATURE_SIGBUS | \
|
|
UFFD_FEATURE_THREAD_ID | \
|
|
UFFD_FEATURE_MINOR_HUGETLBFS | \
|
|
UFFD_FEATURE_MINOR_SHMEM)
|
|
#define UFFD_API_IOCTLS \
|
|
((__u64)1 << _UFFDIO_REGISTER | \
|
|
(__u64)1 << _UFFDIO_UNREGISTER | \
|
|
(__u64)1 << _UFFDIO_API)
|
|
#define UFFD_API_RANGE_IOCTLS \
|
|
((__u64)1 << _UFFDIO_WAKE | \
|
|
(__u64)1 << _UFFDIO_COPY | \
|
|
(__u64)1 << _UFFDIO_ZEROPAGE | \
|
|
(__u64)1 << _UFFDIO_WRITEPROTECT | \
|
|
(__u64)1 << _UFFDIO_CONTINUE)
|
|
#define UFFD_API_RANGE_IOCTLS_BASIC \
|
|
((__u64)1 << _UFFDIO_WAKE | \
|
|
(__u64)1 << _UFFDIO_COPY | \
|
|
(__u64)1 << _UFFDIO_CONTINUE)
|
|
|
|
/*
|
|
* Valid ioctl command number range with this API is from 0x00 to
|
|
* 0x3F. UFFDIO_API is the fixed number, everything else can be
|
|
* changed by implementing a different UFFD_API. If sticking to the
|
|
* same UFFD_API more ioctl can be added and userland will be aware of
|
|
* which ioctl the running kernel implements through the ioctl command
|
|
* bitmask written by the UFFDIO_API.
|
|
*/
|
|
#define _UFFDIO_REGISTER (0x00)
|
|
#define _UFFDIO_UNREGISTER (0x01)
|
|
#define _UFFDIO_WAKE (0x02)
|
|
#define _UFFDIO_COPY (0x03)
|
|
#define _UFFDIO_ZEROPAGE (0x04)
|
|
#define _UFFDIO_WRITEPROTECT (0x06)
|
|
#define _UFFDIO_CONTINUE (0x07)
|
|
#define _UFFDIO_API (0x3F)
|
|
|
|
/* userfaultfd ioctl ids */
|
|
#define UFFDIO 0xAA
|
|
#define UFFDIO_API _IOWR(UFFDIO, _UFFDIO_API, \
|
|
struct uffdio_api)
|
|
#define UFFDIO_REGISTER _IOWR(UFFDIO, _UFFDIO_REGISTER, \
|
|
struct uffdio_register)
|
|
#define UFFDIO_UNREGISTER _IOR(UFFDIO, _UFFDIO_UNREGISTER, \
|
|
struct uffdio_range)
|
|
#define UFFDIO_WAKE _IOR(UFFDIO, _UFFDIO_WAKE, \
|
|
struct uffdio_range)
|
|
#define UFFDIO_COPY _IOWR(UFFDIO, _UFFDIO_COPY, \
|
|
struct uffdio_copy)
|
|
#define UFFDIO_ZEROPAGE _IOWR(UFFDIO, _UFFDIO_ZEROPAGE, \
|
|
struct uffdio_zeropage)
|
|
#define UFFDIO_WRITEPROTECT _IOWR(UFFDIO, _UFFDIO_WRITEPROTECT, \
|
|
struct uffdio_writeprotect)
|
|
#define UFFDIO_CONTINUE _IOR(UFFDIO, _UFFDIO_CONTINUE, \
|
|
struct uffdio_continue)
|
|
|
|
/* read() structure */
|
|
struct uffd_msg {
|
|
__u8 event;
|
|
|
|
__u8 reserved1;
|
|
__u16 reserved2;
|
|
__u32 reserved3;
|
|
|
|
union {
|
|
struct {
|
|
__u64 flags;
|
|
__u64 address;
|
|
union {
|
|
__u32 ptid;
|
|
} feat;
|
|
} pagefault;
|
|
|
|
struct {
|
|
__u32 ufd;
|
|
} fork;
|
|
|
|
struct {
|
|
__u64 from;
|
|
__u64 to;
|
|
__u64 len;
|
|
} remap;
|
|
|
|
struct {
|
|
__u64 start;
|
|
__u64 end;
|
|
} remove;
|
|
|
|
struct {
|
|
/* unused reserved fields */
|
|
__u64 reserved1;
|
|
__u64 reserved2;
|
|
__u64 reserved3;
|
|
} reserved;
|
|
} arg;
|
|
} __packed;
|
|
|
|
/*
|
|
* Start at 0x12 and not at 0 to be more strict against bugs.
|
|
*/
|
|
#define UFFD_EVENT_PAGEFAULT 0x12
|
|
#define UFFD_EVENT_FORK 0x13
|
|
#define UFFD_EVENT_REMAP 0x14
|
|
#define UFFD_EVENT_REMOVE 0x15
|
|
#define UFFD_EVENT_UNMAP 0x16
|
|
|
|
/* flags for UFFD_EVENT_PAGEFAULT */
|
|
#define UFFD_PAGEFAULT_FLAG_WRITE (1<<0) /* If this was a write fault */
|
|
#define UFFD_PAGEFAULT_FLAG_WP (1<<1) /* If reason is VM_UFFD_WP */
|
|
#define UFFD_PAGEFAULT_FLAG_MINOR (1<<2) /* If reason is VM_UFFD_MINOR */
|
|
|
|
struct uffdio_api {
|
|
/* userland asks for an API number and the features to enable */
|
|
__u64 api;
|
|
/*
|
|
* Kernel answers below with the all available features for
|
|
* the API, this notifies userland of which events and/or
|
|
* which flags for each event are enabled in the current
|
|
* kernel.
|
|
*
|
|
* Note: UFFD_EVENT_PAGEFAULT and UFFD_PAGEFAULT_FLAG_WRITE
|
|
* are to be considered implicitly always enabled in all kernels as
|
|
* long as the uffdio_api.api requested matches UFFD_API.
|
|
*
|
|
* UFFD_FEATURE_MISSING_HUGETLBFS means an UFFDIO_REGISTER
|
|
* with UFFDIO_REGISTER_MODE_MISSING mode will succeed on
|
|
* hugetlbfs virtual memory ranges. Adding or not adding
|
|
* UFFD_FEATURE_MISSING_HUGETLBFS to uffdio_api.features has
|
|
* no real functional effect after UFFDIO_API returns, but
|
|
* it's only useful for an initial feature set probe at
|
|
* UFFDIO_API time. There are two ways to use it:
|
|
*
|
|
* 1) by adding UFFD_FEATURE_MISSING_HUGETLBFS to the
|
|
* uffdio_api.features before calling UFFDIO_API, an error
|
|
* will be returned by UFFDIO_API on a kernel without
|
|
* hugetlbfs missing support
|
|
*
|
|
* 2) the UFFD_FEATURE_MISSING_HUGETLBFS can not be added in
|
|
* uffdio_api.features and instead it will be set by the
|
|
* kernel in the uffdio_api.features if the kernel supports
|
|
* it, so userland can later check if the feature flag is
|
|
* present in uffdio_api.features after UFFDIO_API
|
|
* succeeded.
|
|
*
|
|
* UFFD_FEATURE_MISSING_SHMEM works the same as
|
|
* UFFD_FEATURE_MISSING_HUGETLBFS, but it applies to shmem
|
|
* (i.e. tmpfs and other shmem based APIs).
|
|
*
|
|
* UFFD_FEATURE_SIGBUS feature means no page-fault
|
|
* (UFFD_EVENT_PAGEFAULT) event will be delivered, instead
|
|
* a SIGBUS signal will be sent to the faulting process.
|
|
*
|
|
* UFFD_FEATURE_THREAD_ID pid of the page faulted task_struct will
|
|
* be returned, if feature is not requested 0 will be returned.
|
|
*
|
|
* UFFD_FEATURE_MINOR_HUGETLBFS indicates that minor faults
|
|
* can be intercepted (via REGISTER_MODE_MINOR) for
|
|
* hugetlbfs-backed pages.
|
|
*/
|
|
#define UFFD_FEATURE_PAGEFAULT_FLAG_WP (1<<0)
|
|
#define UFFD_FEATURE_EVENT_FORK (1<<1)
|
|
#define UFFD_FEATURE_EVENT_REMAP (1<<2)
|
|
#define UFFD_FEATURE_EVENT_REMOVE (1<<3)
|
|
#define UFFD_FEATURE_MISSING_HUGETLBFS (1<<4)
|
|
#define UFFD_FEATURE_MISSING_SHMEM (1<<5)
|
|
#define UFFD_FEATURE_EVENT_UNMAP (1<<6)
|
|
#define UFFD_FEATURE_SIGBUS (1<<7)
|
|
#define UFFD_FEATURE_THREAD_ID (1<<8)
|
|
#define UFFD_FEATURE_MINOR_HUGETLBFS (1<<9)
|
|
#define UFFD_FEATURE_MINOR_SHMEM (1<<10)
|
|
__u64 features;
|
|
|
|
__u64 ioctls;
|
|
};
|
|
|
|
struct uffdio_range {
|
|
__u64 start;
|
|
__u64 len;
|
|
};
|
|
|
|
struct uffdio_register {
|
|
struct uffdio_range range;
|
|
#define UFFDIO_REGISTER_MODE_MISSING ((__u64)1<<0)
|
|
#define UFFDIO_REGISTER_MODE_WP ((__u64)1<<1)
|
|
#define UFFDIO_REGISTER_MODE_MINOR ((__u64)1<<2)
|
|
__u64 mode;
|
|
|
|
/*
|
|
* kernel answers which ioctl commands are available for the
|
|
* range, keep at the end as the last 8 bytes aren't read.
|
|
*/
|
|
__u64 ioctls;
|
|
};
|
|
|
|
struct uffdio_copy {
|
|
__u64 dst;
|
|
__u64 src;
|
|
__u64 len;
|
|
#define UFFDIO_COPY_MODE_DONTWAKE ((__u64)1<<0)
|
|
/*
|
|
* UFFDIO_COPY_MODE_WP will map the page write protected on
|
|
* the fly. UFFDIO_COPY_MODE_WP is available only if the
|
|
* write protected ioctl is implemented for the range
|
|
* according to the uffdio_register.ioctls.
|
|
*/
|
|
#define UFFDIO_COPY_MODE_WP ((__u64)1<<1)
|
|
__u64 mode;
|
|
|
|
/*
|
|
* "copy" is written by the ioctl and must be at the end: the
|
|
* copy_from_user will not read the last 8 bytes.
|
|
*/
|
|
__s64 copy;
|
|
};
|
|
|
|
struct uffdio_zeropage {
|
|
struct uffdio_range range;
|
|
#define UFFDIO_ZEROPAGE_MODE_DONTWAKE ((__u64)1<<0)
|
|
__u64 mode;
|
|
|
|
/*
|
|
* "zeropage" is written by the ioctl and must be at the end:
|
|
* the copy_from_user will not read the last 8 bytes.
|
|
*/
|
|
__s64 zeropage;
|
|
};
|
|
|
|
struct uffdio_writeprotect {
|
|
struct uffdio_range range;
|
|
/*
|
|
* UFFDIO_WRITEPROTECT_MODE_WP: set the flag to write protect a range,
|
|
* unset the flag to undo protection of a range which was previously
|
|
* write protected.
|
|
*
|
|
* UFFDIO_WRITEPROTECT_MODE_DONTWAKE: set the flag to avoid waking up
|
|
* any wait thread after the operation succeeds.
|
|
*
|
|
* NOTE: Write protecting a region (WP=1) is unrelated to page faults,
|
|
* therefore DONTWAKE flag is meaningless with WP=1. Removing write
|
|
* protection (WP=0) in response to a page fault wakes the faulting
|
|
* task unless DONTWAKE is set.
|
|
*/
|
|
#define UFFDIO_WRITEPROTECT_MODE_WP ((__u64)1<<0)
|
|
#define UFFDIO_WRITEPROTECT_MODE_DONTWAKE ((__u64)1<<1)
|
|
__u64 mode;
|
|
};
|
|
|
|
struct uffdio_continue {
|
|
struct uffdio_range range;
|
|
#define UFFDIO_CONTINUE_MODE_DONTWAKE ((__u64)1<<0)
|
|
__u64 mode;
|
|
|
|
/*
|
|
* Fields below here are written by the ioctl and must be at the end:
|
|
* the copy_from_user will not read past here.
|
|
*/
|
|
__s64 mapped;
|
|
};
|
|
|
|
/*
|
|
* Flags for the userfaultfd(2) system call itself.
|
|
*/
|
|
|
|
/*
|
|
* Create a userfaultfd that can handle page faults only in user mode.
|
|
*/
|
|
#define UFFD_USER_MODE_ONLY 1
|
|
|
|
#endif /* _LINUX_USERFAULTFD_H */
|